You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2016/09/14 18:32:43 UTC
[23/35] ambari git commit: AMBARI-18385: Add HDF management pack
(jluniya)
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi.py
new file mode 100644
index 0000000..362a40b
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi.py
@@ -0,0 +1,231 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys, nifi_ca_util, os, pwd, grp, signal, time, glob, socket, json
+from resource_management.core import sudo
+from resource_management import *
+from subprocess import call
+from setup_ranger_nifi import setup_ranger_nifi
+
+reload(sys)
+sys.setdefaultencoding('utf8')
+
+class Master(Script):
+ def install(self, env):
+
+ import params
+ import status_params
+
+ self.install_packages(env)
+
+ Directory([params.nifi_node_dir],
+ owner=params.nifi_user,
+ group=params.nifi_group,
+ create_parents=True,
+ recursive_ownership=True
+ )
+
+ #update the configs specified by user
+ self.configure(env, True)
+
+ Execute('touch ' + params.nifi_node_log_file, user=params.nifi_user)
+
+ def configure(self, env, isInstall=False, is_starting = False):
+ import params
+ import status_params
+ env.set_params(params)
+ env.set_params(status_params)
+
+ #create the log, pid, conf dirs if not already present
+ Directory([status_params.nifi_pid_dir, params.nifi_node_log_dir, params.nifi_internal_dir, params.nifi_database_dir, params.nifi_flowfile_repo_dir, params.nifi_content_repo_dir_default, params.nifi_provenance_repo_dir_default, params.nifi_config_dir, params.nifi_flow_config_dir, params.nifi_state_dir, params.lib_dir],
+ owner=params.nifi_user,
+ group=params.nifi_group,
+ create_parents=True,
+ recursive_ownership=True
+ )
+
+ # On some OS this folder may not exist, so we will create it before pushing files there
+ Directory(params.limits_conf_dir,
+ create_parents = True,
+ owner='root',
+ group='root'
+ )
+
+ File(os.path.join(params.limits_conf_dir, 'nifi.conf'),
+ owner='root',
+ group='root',
+ mode=0644,
+ content=Template("nifi.conf.j2")
+ )
+
+
+ ca_client_script = nifi_ca_util.get_toolkit_script('tls-toolkit.sh')
+ File(ca_client_script, mode=0755)
+
+
+ if params.nifi_ca_host and params.nifi_ssl_enabled:
+ ca_client_json = os.path.realpath(os.path.join(params.nifi_config_dir, 'nifi-certificate-authority-client.json'))
+ File(ca_client_json,
+ owner = params.nifi_user,
+ group = params.nifi_group,
+ mode = 0600)
+ ca_client_dict = nifi_ca_util.load(ca_client_json)
+ if is_starting:
+ if params.nifi_toolkit_tls_regenerate:
+ nifi_ca_util.move_keystore_truststore(ca_client_dict)
+ ca_client_dict = {}
+ else:
+ nifi_ca_util.move_keystore_truststore_if_necessary(ca_client_dict, params.nifi_ca_client_config)
+ nifi_ca_util.overlay(ca_client_dict, params.nifi_ca_client_config)
+ nifi_ca_util.dump(ca_client_json, ca_client_dict)
+ if is_starting:
+ Execute('JAVA_HOME='+params.jdk64_home+' '+ca_client_script+' client -F -f '+ca_client_json, user=params.nifi_user)
+ nifi_ca_util.update_nifi_properties(nifi_ca_util.load(ca_client_json), params.nifi_properties)
+
+ #write out nifi.properties
+ PropertiesFile(params.nifi_config_dir + '/nifi.properties',
+ properties = params.nifi_properties,
+ mode = 0400,
+ owner = params.nifi_user,
+ group = params.nifi_group)
+
+ #write out boostrap.conf
+ bootstrap_content=InlineTemplate(params.nifi_boostrap_content)
+ File(format("{params.nifi_config_dir}/bootstrap.conf"), content=bootstrap_content, owner=params.nifi_user, group=params.nifi_group, mode=0400)
+
+ #write out logback.xml
+ logback_content=InlineTemplate(params.nifi_node_logback_content)
+ File(format("{params.nifi_config_dir}/logback.xml"), content=logback_content, owner=params.nifi_user, group=params.nifi_group, mode=0400)
+
+ #write out state-management.xml
+ statemgmt_content=InlineTemplate(params.nifi_state_management_content)
+ File(format("{params.nifi_config_dir}/state-management.xml"), content=statemgmt_content, owner=params.nifi_user, group=params.nifi_group, mode=0400)
+
+ #write out authorizers file
+ authorizers_content=InlineTemplate(params.nifi_authorizers_content)
+ File(format("{params.nifi_config_dir}/authorizers.xml"), content=authorizers_content, owner=params.nifi_user, group=params.nifi_group, mode=0400)
+
+ #write out login-identity-providers.xml
+ login_identity_providers_content=InlineTemplate(params.nifi_login_identity_providers_content)
+ File(format("{params.nifi_config_dir}/login-identity-providers.xml"), content=login_identity_providers_content, owner=params.nifi_user, group=params.nifi_group, mode=0400)
+
+ #write out nifi-env in bin as 0755 (see BUG-61769)
+ env_content=InlineTemplate(params.nifi_env_content)
+ File(format("{params.bin_dir}/nifi-env.sh"), content=env_content, owner=params.nifi_user, group=params.nifi_group, mode=0755)
+
+ #write out bootstrap-notification-services.xml
+ boostrap_notification_content=InlineTemplate(params.nifi_boostrap_notification_content)
+ File(format("{params.nifi_config_dir}/bootstrap-notification-services.xml"), content=boostrap_notification_content, owner=params.nifi_user, group=params.nifi_group, mode=0400)
+
+ def stop(self, env):
+ import params
+ import status_params
+
+ Execute ('export JAVA_HOME='+params.jdk64_home+';'+params.bin_dir+'/nifi.sh stop >> ' + params.nifi_node_log_file, user=params.nifi_user)
+ if os.path.isfile(status_params.nifi_node_pid_file):
+ sudo.unlink(status_params.nifi_node_pid_file)
+
+ def start(self, env):
+ import params
+ import status_params
+ self.configure(env, is_starting = True)
+ setup_ranger_nifi(upgrade_type=None)
+
+ # Write out flow.xml.gz to internal dir only if AMS installed (must be writable by Nifi)
+ # only during first install. It is used to automate setup of Ambari metrics reporting task in Nifi
+ if params.metrics_collector_host and params.nifi_ambari_reporting_enabled and self.check_is_fresh_install(self):
+ Execute('echo "First time setup so generating flow.xml.gz" >> ' + params.nifi_node_log_file, user=params.nifi_user)
+ flow_content=InlineTemplate(params.nifi_flow_content)
+ File(format("{params.nifi_flow_config_dir}/flow.xml"), content=flow_content, owner=params.nifi_user, group=params.nifi_group, mode=0600)
+ Execute(format("cd {params.nifi_flow_config_dir}; mv flow.xml.gz flow_$(date +%d-%m-%Y).xml.gz ;"),user=params.nifi_user,ignore_failures=True)
+ Execute(format("cd {params.nifi_flow_config_dir}; gzip flow.xml;"), user=params.nifi_user)
+
+
+ Execute ('export JAVA_HOME='+params.jdk64_home+';'+params.bin_dir+'/nifi.sh start >> ' + params.nifi_node_log_file, user=params.nifi_user)
+ #If nifi pid file not created yet, wait a bit
+ if not os.path.isfile(status_params.nifi_pid_dir+'/nifi.pid'):
+ Execute ('sleep 5')
+
+
+ def status(self, env):
+ import status_params
+ check_process_status(status_params.nifi_node_pid_file)
+
+
+ def check_is_fresh_install(self, env):
+ """
+ Checks if fresh nifi install by checking if zk dir exists
+ :return:
+ """
+ import params, re
+ from resource_management.core import shell
+ from resource_management.core.exceptions import Fail
+ from resource_management.core.logger import Logger
+
+ ZK_CONNECT_ERROR = "ConnectionLoss"
+ ZK_NODE_NOT_EXIST = "Node does not exist"
+
+ zookeeper_queried = False
+ is_fresh_nifi_install = True
+
+ # For every zk server try to find nifi zk dir
+ zookeeper_server_list = params.config['clusterHostInfo']['zookeeper_hosts']
+ for zookeeper_server in zookeeper_server_list:
+ # Determine where the zkCli.sh shell script is
+ zk_command_location = os.path.join(params.stack_root, "current", "zookeeper-client", "bin", "zkCli.sh")
+ if params.stack_version_buildnum is not None:
+ zk_command_location = os.path.join(params.stack_root, params.stack_version_buildnum, "zookeeper", "bin", "zkCli.sh")
+
+ # create the ZooKeeper query command e.g.
+ # /usr/hdf/current/zookeeper-client/bin/zkCli.sh -server node:2181 ls /nifi
+ command = "{0} -server {1}:{2} ls {3}".format(
+ zk_command_location, zookeeper_server, params.zookeeper_port, params.nifi_znode)
+
+ # echo 'ls /nifi' | /usr/hdf/current/zookeeper-client/bin/zkCli.sh -server node:2181
+ #command = "echo 'ls {3}' | {0} -server {1}:{2}".format(
+ # zk_command_location, zookeeper_server, params.zookeeper_port, params.nifi_znode)
+
+ Logger.info("Running command: " + command)
+
+ code, out = shell.call(command, logoutput=True, quiet=False, timeout=20)
+ if not out or re.search(ZK_CONNECT_ERROR, out):
+ Logger.info("Unable to query Zookeeper: " + zookeeper_server + ". Skipping and trying next ZK server")
+ continue
+ elif re.search(ZK_NODE_NOT_EXIST, out):
+ Logger.info("Nifi ZNode does not exist, so must be fresh install of Nifi: " + params.nifi_znode)
+ zookeeper_queried = True
+ is_fresh_nifi_install = True
+ break
+ else:
+ Logger.info("Nifi ZNode already exists, so must not be a fresh install of Nifi: " + params.nifi_znode)
+ zookeeper_queried = True
+ is_fresh_nifi_install = False
+ break
+
+ # fail if the ZK data could not be queried
+ if not zookeeper_queried:
+ raise Fail("Unable to query for znode on on any of the following ZooKeeper hosts: {0}. Please ensure Zookeepers are started and retry".format(
+ zookeeper_server_list))
+ else:
+ return is_fresh_nifi_install
+
+
+if __name__ == "__main__":
+ Master().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_ca.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_ca.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_ca.py
new file mode 100644
index 0000000..bc343e4
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_ca.py
@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import nifi_ca_util, os, time
+
+from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.core.resources.system import Directory, Execute
+from resource_management.core.sudo import kill, read_file, path_isfile, unlink
+from resource_management.libraries.functions.check_process_status import check_process_status
+from resource_management.libraries.script.script import Script
+from resource_management.core.resources import File
+from signal import SIGTERM, SIGKILL
+
+class CertificateAuthority(Script):
+ def install(self, env):
+ import params
+ import status_params
+
+ self.install_packages(env)
+
+ #Be sure ca script is in cache
+ nifi_ca_util.get_toolkit_script('tls-toolkit.sh')
+
+ def configure(self, env):
+ import params
+ import status_params
+ env.set_params(params)
+ env.set_params(status_params)
+
+ #create the log, pid, conf dirs if not already present
+ Directory([status_params.nifi_pid_dir, params.nifi_node_log_dir, params.nifi_config_dir],
+ owner=params.nifi_user,
+ group=params.nifi_group,
+ create_parents=True
+ )
+
+ ca_json = os.path.join(params.nifi_config_dir, 'nifi-certificate-authority.json')
+ ca_dict = nifi_ca_util.load(ca_json)
+ nifi_ca_util.overlay(ca_dict, params.nifi_ca_config)
+ nifi_ca_util.dump(ca_json, ca_dict)
+
+ Directory([params.nifi_config_dir],
+ owner=params.nifi_user,
+ group=params.nifi_group,
+ create_parents=True,
+ recursive_ownership=True
+ )
+
+ def invalidate_ca_server(self, env):
+ import params
+ ca_json = os.path.join(params.nifi_config_dir, 'nifi-certificate-authority.json')
+ nifi_ca_util.move_store(nifi_ca_util.load(ca_json), 'keyStore')
+ unlink(ca_json)
+
+ def status(self, env):
+ import status_params
+ check_process_status(status_params.nifi_ca_pid_file)
+
+ def start(self, env):
+ import params
+ import status_params
+
+ self.configure(env)
+ ca_server_script = nifi_ca_util.get_toolkit_script('tls-toolkit.sh')
+ run_ca_script = os.path.join(os.path.dirname(__file__), 'run_ca.sh')
+ Directory([params.nifi_config_dir],
+ owner=params.nifi_user,
+ group=params.nifi_group,
+ create_parents=True,
+ recursive_ownership=True
+ )
+
+ File(ca_server_script, mode=0755)
+ File(run_ca_script, mode=0755)
+ Execute((run_ca_script, params.jdk64_home, ca_server_script, params.nifi_config_dir + '/nifi-certificate-authority.json', params.nifi_ca_log_file_stdout, params.nifi_ca_log_file_stderr, status_params.nifi_ca_pid_file), user=params.nifi_user)
+ if not os.path.isfile(status_params.nifi_ca_pid_file):
+ raise Exception('Expected pid file to exist')
+
+ def stop(self, env):
+ import status_params
+
+ if path_isfile(status_params.nifi_ca_pid_file):
+ try:
+ self.status(env)
+ pid = int(read_file(status_params.nifi_ca_pid_file))
+ for i in range(25):
+ kill(pid, SIGTERM)
+ time.sleep(1)
+ self.status(env)
+ kill(pid, SIGKILL)
+ time.sleep(5)
+ self.status(env)
+ except ComponentIsNotRunning:
+ unlink(status_params.nifi_ca_pid_file)
+
+if __name__ == "__main__":
+ CertificateAuthority().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_ca_util.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_ca_util.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_ca_util.py
new file mode 100644
index 0000000..6807491
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_ca_util.py
@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import json, nifi_constants, os
+from resource_management.core import sudo
+from resource_management.core.resources.system import File
+
+script_dir = os.path.dirname(__file__)
+files_dir = os.path.realpath(os.path.join(os.path.dirname(script_dir), 'files'))
+
+def load(config_json):
+ if sudo.path_isfile(config_json):
+ contents = sudo.read_file(config_json)
+ if len(contents) > 0:
+ return json.loads(contents)
+ return {}
+
+def dump(config_json, config_dict):
+ import params
+ File(config_json,
+ owner=params.nifi_user,
+ group=params.nifi_group,
+ mode=0600,
+ content=json.dumps(config_dict, sort_keys=True, indent=4)
+ )
+
+def overlay(config_dict, overlay_dict):
+ for k, v in overlay_dict.iteritems():
+ if v or k not in config_dict:
+ config_dict[k] = v
+
+def get_toolkit_script(scriptName, scriptDir = files_dir):
+ nifiToolkitDir = None
+ for dir in os.listdir(scriptDir):
+ if dir.startswith('nifi-toolkit-'):
+ nifiToolkitDir = os.path.join(scriptDir, dir)
+
+ if nifiToolkitDir is None:
+ raise Exception("Couldn't find nifi toolkit directory in " + scriptDir)
+ result = nifiToolkitDir + '/bin/' + scriptName
+ if not sudo.path_isfile(result):
+ raise Exception("Couldn't find file " + result)
+ return result
+
+def update_nifi_properties(client_dict, nifi_properties):
+ nifi_properties[nifi_constants.NIFI_SECURITY_KEYSTORE_TYPE] = client_dict['keyStoreType']
+ nifi_properties[nifi_constants.NIFI_SECURITY_KEYSTORE_PASSWD] = client_dict['keyStorePassword']
+ nifi_properties[nifi_constants.NIFI_SECURITY_KEY_PASSWD] = client_dict['keyPassword']
+ nifi_properties[nifi_constants.NIFI_SECURITY_TRUSTSTORE_TYPE] = client_dict['trustStoreType']
+ nifi_properties[nifi_constants.NIFI_SECURITY_TRUSTSTORE_PASSWD] = client_dict['trustStorePassword']
+
+def store_exists(client_dict, key):
+ if key not in client_dict:
+ return False
+ return sudo.path_isfile(client_dict[key])
+
+def different(one, two, key):
+ if key not in one:
+ return False
+ if len(one[key]) == 0:
+ return False
+ if key not in two:
+ return False
+ if len(two[key]) == 0:
+ return False
+ return one[key] != two[key]
+
+def move_keystore_truststore_if_necessary(orig_client_dict, new_client_dict):
+ if not (store_exists(new_client_dict, 'keyStore') or store_exists(new_client_dict, 'trustStore')):
+ return
+ if different(orig_client_dict, new_client_dict, 'keyStoreType'):
+ move_keystore_truststore(new_client_dict)
+ elif different(orig_client_dict, new_client_dict, 'keyStorePassword'):
+ move_keystore_truststore(new_client_dict)
+ elif different(orig_client_dict, new_client_dict, 'keyPassword'):
+ move_keystore_truststore(new_client_dict)
+ elif different(orig_client_dict, new_client_dict, 'trustStoreType'):
+ move_keystore_truststore(new_client_dict)
+ elif different(orig_client_dict, new_client_dict, 'trustStorePassword'):
+ move_keystore_truststore(new_client_dict)
+
+def move_keystore_truststore(client_dict):
+ move_store(client_dict, 'keyStore')
+ move_store(client_dict, 'trustStore')
+
+def move_store(client_dict, key):
+ if store_exists(client_dict, key):
+ num = 0
+ name = client_dict[key]
+ while sudo.path_isfile(name + '.bak.' + str(num)):
+ num += 1
+ sudo.copy(name, name + '.bak.' + str(num))
+ sudo.unlink(name)
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_constants.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_constants.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_constants.py
new file mode 100644
index 0000000..632a473
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_constants.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+NIFI_SECURITY_KEY_PASSWD = 'nifi.security.keyPasswd'
+NIFI_SECURITY_KEYSTORE_PASSWD = 'nifi.security.keystorePasswd'
+NIFI_SECURITY_KEYSTORE_TYPE = 'nifi.security.keystoreType'
+NIFI_SECURITY_TRUSTSTORE_PASSWD = 'nifi.security.truststorePasswd'
+NIFI_SECURITY_TRUSTSTORE_TYPE = 'nifi.security.truststoreType'
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/params.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/params.py
new file mode 100644
index 0000000..087cef4
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/params.py
@@ -0,0 +1,445 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+from resource_management.libraries.script.script import Script
+import sys, os, glob, socket, re
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.version import format_stack_version
+from resource_management.libraries.functions import StackFeature
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.resources.hdfs_resource import HdfsResource
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import get_kinit_path
+from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
+import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set
+
+# server configurations
+config = Script.get_config()
+stack_root = Script.get_stack_root()
+stack_version_buildnum = default("/commandParams/version", None)
+
+#nifi_install_dir = '/usr/hdf/current/nifi'
+nifi_install_dir = os.path.join(stack_root, "current", "nifi")
+if stack_version_buildnum is not None:
+ nifi_install_dir = os.path.join(stack_root, stack_version_buildnum, "nifi")
+
+# params from nifi-ambari-config
+nifi_initial_mem = config['configurations']['nifi-ambari-config']['nifi.initial_mem']
+nifi_max_mem = config['configurations']['nifi-ambari-config']['nifi.max_mem']
+nifi_ambari_reporting_frequency = config['configurations']['nifi-ambari-config']['nifi.ambari_reporting_frequency']
+nifi_ambari_reporting_enabled = config['configurations']['nifi-ambari-config']['nifi.ambari_reporting_enabled']
+
+# note: nifi.node.port and nifi.node.ssl.port must be defined in same xml file for quicklinks to work
+nifi_node_port = config['configurations']['nifi-ambari-config']['nifi.node.port']
+nifi_node_ssl_port = config['configurations']['nifi-ambari-config']['nifi.node.ssl.port']
+nifi_node_protocol_port = config['configurations']['nifi-ambari-config']['nifi.node.protocol.port']
+
+nifi_znode = config['configurations']['nifi-ambari-config']['nifi.nifi_znode']
+
+nifi_internal_dir=config['configurations']['nifi-ambari-config']['nifi.internal.dir']
+nifi_state_dir=config['configurations']['nifi-ambari-config']['nifi.state.dir']
+nifi_database_dir=config['configurations']['nifi-ambari-config']['nifi.database.dir']
+nifi_flowfile_repo_dir=config['configurations']['nifi-ambari-config']['nifi.flowfile.repository.dir']
+nifi_content_repo_dir_default=config['configurations']['nifi-ambari-config']['nifi.content.repository.dir.default']
+nifi_provenance_repo_dir_default=config['configurations']['nifi-ambari-config']['nifi.provenance.repository.dir.default']
+nifi_config_dir = config['configurations']['nifi-ambari-config']['nifi.config.dir']
+nifi_flow_config_dir = config['configurations']['nifi-ambari-config']['nifi.flow.config.dir']
+nifi_sensitive_props_key = config['configurations']['nifi-ambari-config']['nifi.sensitive.props.key']
+
+
+nifi_flow_config_dir = nifi_flow_config_dir.replace('{nifi_internal_dir}',nifi_internal_dir)
+nifi_state_dir = nifi_state_dir.replace('{nifi_internal_dir}',nifi_internal_dir)
+nifi_config_dir = nifi_config_dir.replace('{nifi_install_dir}',nifi_install_dir)
+
+master_configs = config['clusterHostInfo']
+nifi_master_hosts = master_configs['nifi_master_hosts']
+
+# detect if running in single (sandbox) box
+nifi_num_nodes = len(master_configs['nifi_master_hosts'])
+#if nifi_num_nodes > 1:
+# nifi_is_node='true'
+#else:
+# nifi_is_node='false'
+#nifi_node_hosts = ",".join(master_configs['nifi_master_hosts'])
+
+# In sandbox scenario, Ambari should still setup nifi in clustered mode for now
+nifi_is_node='true'
+
+nifi_node_dir=nifi_install_dir
+bin_dir = os.path.join(*[nifi_node_dir,'bin'])
+lib_dir = os.path.join(*[nifi_node_dir,'lib'])
+
+nifi_ca_host = None
+if 'nifi_ca_hosts' in master_configs:
+ nifi_ca_hosts = master_configs['nifi_ca_hosts']
+ if len(nifi_ca_hosts) > 0:
+ nifi_ca_host = nifi_ca_hosts[0]
+
+# params from nifi-ambari-ssl-config
+
+nifi_ssl_enabled = config['configurations']['nifi-ambari-ssl-config']['nifi.node.ssl.isenabled']
+nifi_keystore = config['configurations']['nifi-ambari-ssl-config']['nifi.security.keystore']
+nifi_keystoreType = config['configurations']['nifi-ambari-ssl-config']['nifi.security.keystoreType']
+nifi_keystorePasswd = config['configurations']['nifi-ambari-ssl-config']['nifi.security.keystorePasswd']
+nifi_keyPasswd = config['configurations']['nifi-ambari-ssl-config']['nifi.security.keyPasswd']
+nifi_truststore = config['configurations']['nifi-ambari-ssl-config']['nifi.security.truststore']
+nifi_truststoreType = config['configurations']['nifi-ambari-ssl-config']['nifi.security.truststoreType']
+nifi_truststorePasswd = config['configurations']['nifi-ambari-ssl-config']['nifi.security.truststorePasswd']
+nifi_needClientAuth = config['configurations']['nifi-ambari-ssl-config']['nifi.security.needClientAuth']
+nifi_initial_admin_id = config['configurations']['nifi-ambari-ssl-config']['nifi.initial.admin.identity']
+nifi_ssl_config_content = config['configurations']['nifi-ambari-ssl-config']['content']
+
+#property that is set to hostname regardless of whether SSL enabled
+nifi_node_host = socket.getfqdn()
+
+nifi_truststore = nifi_truststore.replace('{nifi_node_ssl_host}',nifi_node_host)
+nifi_keystore = nifi_keystore.replace('{nifi_node_ssl_host}',nifi_node_host)
+
+#populate properties whose values depend on whether SSL enabled
+nifi_keystore = nifi_keystore.replace('{{nifi_config_dir}}',nifi_config_dir)
+nifi_truststore = nifi_truststore.replace('{{nifi_config_dir}}',nifi_config_dir)
+
+if nifi_ssl_enabled:
+ nifi_node_ssl_host = nifi_node_host
+ nifi_node_port = ""
+else:
+ nifi_node_nonssl_host = nifi_node_host
+ nifi_node_ssl_port = ""
+
+nifi_ca_parent_config = config['configurations']['nifi-ambari-ssl-config']
+nifi_use_ca = nifi_ca_parent_config['nifi.toolkit.tls.token']
+nifi_toolkit_dn_prefix = nifi_ca_parent_config['nifi.toolkit.dn.prefix']
+nifi_toolkit_dn_suffix = nifi_ca_parent_config['nifi.toolkit.dn.suffix']
+nifi_toolkit_tls_regenerate = nifi_ca_parent_config['nifi.toolkit.tls.regenerate']
+nifi_ca_log_file_stdout = config['configurations']['nifi-env']['nifi_node_log_dir'] + '/nifi-ca.stdout'
+nifi_ca_log_file_stderr = config['configurations']['nifi-env']['nifi_node_log_dir'] + '/nifi-ca.stderr'
+
+nifi_ca_config = {
+ "days" : int(nifi_ca_parent_config['nifi.toolkit.tls.helper.days']),
+ "keyStore" : nifi_config_dir + '/nifi-certificate-authority-keystore.jks',
+ "token" : nifi_ca_parent_config['nifi.toolkit.tls.token'],
+ "caHostname" : nifi_ca_host,
+ "port" : int(nifi_ca_parent_config['nifi.toolkit.tls.port'])
+}
+
+if nifi_ca_host:
+ nifi_ca_config['dn'] = nifi_toolkit_dn_prefix + nifi_ca_host + nifi_toolkit_dn_suffix
+
+nifi_ca_client_config = {
+ "days" : int(nifi_ca_parent_config['nifi.toolkit.tls.helper.days']),
+ "keyStore" : nifi_keystore,
+ "keyStoreType" : nifi_keystoreType,
+ "keyStorePassword" : nifi_keystorePasswd,
+ "keyPassword" : nifi_keyPasswd,
+ "token" : nifi_ca_parent_config['nifi.toolkit.tls.token'],
+ "dn" : nifi_toolkit_dn_prefix + nifi_node_host + nifi_toolkit_dn_suffix,
+ "port" : int(nifi_ca_parent_config['nifi.toolkit.tls.port']),
+ "caHostname" : nifi_ca_host,
+ "trustStore" : nifi_truststore,
+ "trustStoreType" : nifi_truststoreType,
+ "trustStorePassword": nifi_truststorePasswd
+}
+
+# params from nifi-env
+nifi_user = config['configurations']['nifi-env']['nifi_user']
+nifi_group = config['configurations']['nifi-env']['nifi_group']
+
+nifi_node_log_dir = config['configurations']['nifi-env']['nifi_node_log_dir']
+nifi_node_log_file = os.path.join(nifi_node_log_dir,'nifi-setup.log')
+
+# limits related params
+limits_conf_dir = '/etc/security/limits.d'
+nifi_user_nofile_limit = config['configurations']['nifi-env']['nifi_user_nofile_limit']
+nifi_user_nproc_limit = config['configurations']['nifi-env']['nifi_user_nproc_limit']
+
+# params from nifi-boostrap
+nifi_env_content = config['configurations']['nifi-env']['content']
+
+
+# params from nifi-logback
+nifi_master_logback_content = config['configurations']['nifi-master-logback-env']['content']
+nifi_node_logback_content = config['configurations']['nifi-node-logback-env']['content']
+
+# params from nifi-properties-env
+nifi_master_properties_content = config['configurations']['nifi-master-properties-env']['content']
+nifi_properties = config['configurations']['nifi-properties'].copy()
+
+#kerberos params
+nifi_kerberos_authentication_expiration = config['configurations']['nifi-properties']['nifi.kerberos.spnego.authentication.expiration']
+nifi_kerberos_realm = default("/configurations/kerberos-env/realm", None)
+
+# params from nifi-flow
+nifi_flow_content = config['configurations']['nifi-flow-env']['content']
+
+# params from nifi-state-management-env
+nifi_state_management_content = config['configurations']['nifi-state-management-env']['content']
+
+# params from nifi-authorizers-env
+nifi_authorizers_content = config['configurations']['nifi-authorizers-env']['content']
+
+# params from nifi-login-identity-providers-env
+nifi_login_identity_providers_content = config['configurations']['nifi-login-identity-providers-env']['content']
+
+# params from nifi-boostrap
+nifi_boostrap_content = config['configurations']['nifi-bootstrap-env']['content']
+
+# params from nifi-bootstrap-notification-services-env
+nifi_boostrap_notification_content = config['configurations']['nifi-bootstrap-notification-services-env']['content']
+
+#autodetect jdk home
+jdk64_home=config['hostLevelParams']['java_home']
+
+#autodetect ambari server for metrics
+if 'metrics_collector_hosts' in config['clusterHostInfo']:
+ metrics_collector_host = str(config['clusterHostInfo']['metrics_collector_hosts'][0])
+ metrics_collector_port = str(get_port_from_url(config['configurations']['ams-site']['timeline.metrics.service.webapp.address']))
+else:
+ metrics_collector_host = ''
+ metrics_collector_port = ''
+
+
+#detect zookeeper_quorum
+zookeeper_port=default('/configurations/zoo.cfg/clientPort', None)
+#get comma separated list of zookeeper hosts from clusterHostInfo
+index = 0
+zookeeper_quorum=""
+for host in config['clusterHostInfo']['zookeeper_hosts']:
+ zookeeper_quorum += host + ":"+str(zookeeper_port)
+ index += 1
+ if index < len(config['clusterHostInfo']['zookeeper_hosts']):
+ zookeeper_quorum += ","
+
+
+#setup ranger configuration
+
+retryAble = default("/commandParams/command_retry_enabled", False)
+version = default("/commandParams/version", None)
+namenode_hosts = default("/clusterHostInfo/namenode_host", None)
+
+if type(namenode_hosts) is list:
+ namenode_host = namenode_hosts[0]
+else:
+ namenode_host = namenode_hosts
+
+has_namenode = not namenode_host == None
+
+
+nifi_authorizer = 'file-provider'
+
+nifi_host_name = config['hostname']
+nifi_host_port = config['configurations']['nifi-ambari-config']['nifi.node.port']
+java_home = config['hostLevelParams']['java_home']
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+smokeuser = config['configurations']['cluster-env']['smokeuser']
+smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
+smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
+kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
+
+if security_enabled:
+ _hostname_lowercase = nifi_host_name.lower()
+ nifi_properties['nifi.kerberos.service.principal'] = nifi_properties['nifi.kerberos.service.principal'].replace('_HOST',_hostname_lowercase)
+ nifi_properties['nifi.kerberos.spnego.principal'] = nifi_properties['nifi.kerberos.spnego.principal'].replace('_HOST',_hostname_lowercase)
+
+# ranger host
+# E.g., 2.3
+stack_version_unformatted = config['hostLevelParams']['stack_version']
+stack_version_formatted = format_stack_version(stack_version_unformatted)
+stack_supports_ranger_kerberos = stack_version_formatted and check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, stack_version_formatted)
+stack_supports_ranger_audit_db = stack_version_formatted and check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, stack_version_formatted)
+
+ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", [])
+has_ranger_admin = not len(ranger_admin_hosts) == 0
+xml_configurations_supported = config['configurations']['ranger-env']['xml_configurations_supported']
+
+ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
+
+# ranger nifi properties
+policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url']
+
+if 'admin-properties' in config['configurations'] and 'policymgr_external_url' in config['configurations']['admin-properties'] and policymgr_mgr_url.endswith('/'):
+ policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
+
+xa_audit_db_name = config['configurations']['admin-properties']['audit_db_name']
+xa_audit_db_user = config['configurations']['admin-properties']['audit_db_user']
+xa_db_host = config['configurations']['admin-properties']['db_host']
+repo_name = str(config['clusterName']) + '_nifi'
+
+repo_config_username = config['configurations']['ranger-nifi-plugin-properties']['REPOSITORY_CONFIG_USERNAME']
+
+ranger_env = config['configurations']['ranger-env']
+ranger_plugin_properties = config['configurations']['ranger-nifi-plugin-properties']
+policy_user = config['configurations']['ranger-nifi-plugin-properties']['policy_user']
+
+#For curl command in ranger plugin to get db connector
+jdk_location = config['hostLevelParams']['jdk_location']
+java_share_dir = '/usr/share/java'
+
+if has_ranger_admin:
+ enable_ranger_nifi = (config['configurations']['ranger-nifi-plugin-properties']['ranger-nifi-plugin-enabled'].lower() == 'yes')
+ xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password']) if stack_supports_ranger_audit_db else None
+ repo_config_password = unicode(config['configurations']['ranger-nifi-plugin-properties']['REPOSITORY_CONFIG_PASSWORD'])
+ xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower()
+ previous_jdbc_jar_name= None
+
+ if stack_supports_ranger_audit_db:
+ if xa_audit_db_flavor == 'mysql':
+ jdbc_jar_name = default("/hostLevelParams/custom_mysql_jdbc_name", None)
+ previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mysql_jdbc_name", None)
+ audit_jdbc_url = format('jdbc:mysql://{xa_db_host}/{xa_audit_db_name}')
+ jdbc_driver = "com.mysql.jdbc.Driver"
+ elif xa_audit_db_flavor == 'oracle':
+ jdbc_jar_name = default("/hostLevelParams/custom_oracle_jdbc_name", None)
+ previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_oracle_jdbc_name", None)
+ colon_count = xa_db_host.count(':')
+ if colon_count == 2 or colon_count == 0:
+ audit_jdbc_url = format('jdbc:oracle:thin:@{xa_db_host}')
+ else:
+ audit_jdbc_url = format('jdbc:oracle:thin:@//{xa_db_host}')
+ jdbc_driver = "oracle.jdbc.OracleDriver"
+ elif xa_audit_db_flavor == 'postgres':
+ jdbc_jar_name = default("/hostLevelParams/custom_postgres_jdbc_name", None)
+ previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_postgres_jdbc_name", None)
+ audit_jdbc_url = format('jdbc:postgresql://{xa_db_host}/{xa_audit_db_name}')
+ jdbc_driver = "org.postgresql.Driver"
+ elif xa_audit_db_flavor == 'mssql':
+ jdbc_jar_name = default("/hostLevelParams/custom_mssql_jdbc_name", None)
+ previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mssql_jdbc_name", None)
+ audit_jdbc_url = format('jdbc:sqlserver://{xa_db_host};databaseName={xa_audit_db_name}')
+ jdbc_driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver"
+ elif xa_audit_db_flavor == 'sqla':
+ jdbc_jar_name = default("/hostLevelParams/custom_sqlanywhere_jdbc_name", None)
+ previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_sqlanywhere_jdbc_name", None)
+ audit_jdbc_url = format('jdbc:sqlanywhere:database={xa_audit_db_name};host={xa_db_host}')
+ jdbc_driver = "sap.jdbc4.sqlanywhere.IDriver"
+
+ downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None
+ driver_curl_source = format("{jdk_location}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None
+
+ driver_curl_target = format("{stack_root}/current/nifi/ext/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None
+ previous_jdbc_jar = format("{stack_root}/current/nifi/ext/{previous_jdbc_jar_name}") if stack_supports_ranger_audit_db else None
+ sql_connector_jar = ''
+
+ ssl_keystore_password = unicode(config['configurations']['ranger-nifi-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None
+ ssl_truststore_password = unicode(config['configurations']['ranger-nifi-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None
+ credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None
+ credential_file_type = 'jceks'
+ ranger_admin_username = config['configurations']['ranger-env']['ranger_admin_username']
+ ranger_admin_password = config['configurations']['ranger-env']['ranger_admin_password']
+
+ #create ranger service's nifi client properties
+ nifi_authentication = config['configurations']['ranger-nifi-plugin-properties']['nifi.authentication']
+ ranger_id_owner_for_certificate = config['configurations']['ranger-nifi-plugin-properties']['owner.for.certificate']
+ nifi_id_owner_for_certificate = config['configurations']['ranger-nifi-policymgr-ssl']['owner.for.certificate']
+ regex = r"(CN)=([a-zA-Z0-9\.\-\* ]*)"
+ match = re.search(regex, nifi_id_owner_for_certificate)
+ common_name_for_certificate = match.group(2) if match else 'NONE'
+
+ if nifi_authentication == 'SSL':
+
+ nifi_ranger_plugin_config = {
+ 'nifi.authentication': nifi_authentication,
+ 'nifi.url': format("https://{nifi_host_name}:{nifi_node_ssl_port}/nifi-api/resources"),
+ 'nifi.ssl.keystore': config['configurations']['ranger-nifi-plugin-properties']['nifi.ssl.keystore'],
+ 'nifi.ssl.keystoreType':config['configurations']['ranger-nifi-plugin-properties']['nifi.ssl.keystoreType'],
+ 'nifi.ssl.keystorePassword': config['configurations']['ranger-nifi-plugin-properties']['nifi.ssl.keystorePassword'],
+ 'nifi.ssl.truststore': config['configurations']['ranger-nifi-plugin-properties']['nifi.ssl.truststore'],
+ 'nifi.ssl.truststoreType': config['configurations']['ranger-nifi-plugin-properties']['nifi.ssl.truststoreType'],
+ 'nifi.ssl.truststorePassword': config['configurations']['ranger-nifi-plugin-properties']['nifi.ssl.truststorePassword'],
+ 'commonNameForCertificate': common_name_for_certificate
+ }
+ else:
+ nifi_ranger_plugin_config = {
+ 'nifi.authentication': nifi_authentication,
+ 'nifi.url': format("https://{nifi_host_name}:{nifi_host_port}/nifi-api/resources"),
+ 'commonNameForCertificate': common_name_for_certificate
+ }
+
+ nifi_ranger_plugin_repo = {
+ 'isActive': 'true',
+ 'config': json.dumps(nifi_ranger_plugin_config),
+ 'description': 'nifi repo',
+ 'name': repo_name,
+ 'repositoryType': 'nifi',
+ 'assetType': '5'
+ }
+
+ # used in nifi authorizers
+ ranger_admin_identity = ranger_id_owner_for_certificate
+
+ if stack_supports_ranger_kerberos and security_enabled:
+ nifi_ranger_plugin_config['policy.download.auth.users'] = nifi_user
+ nifi_ranger_plugin_config['tag.download.auth.users'] = nifi_user
+ ranger_nifi_principal = config['configurations']['nifi-properties']['nifi.kerberos.service.principal'].replace('_HOST',_hostname_lowercase)
+ ranger_nifi_keytab = config['configurations']['nifi-properties']['nifi.kerberos.service.keytab.location']
+
+ if stack_supports_ranger_kerberos:
+ nifi_ranger_plugin_config['ambari.service.check.user'] = policy_user
+
+ nifi_ranger_plugin_repo = {
+ 'isEnabled': 'true',
+ 'configs': nifi_ranger_plugin_config,
+ 'description': 'nifi repo',
+ 'name': repo_name,
+ 'type': 'nifi'
+ }
+
+ xa_audit_db_is_enabled = False
+ ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls']
+
+ if xml_configurations_supported and stack_supports_ranger_audit_db:
+ xa_audit_db_is_enabled = config['configurations']['ranger-nifi-audit']['xasecure.audit.destination.db']
+
+ xa_audit_hdfs_is_enabled = default('/configurations/ranger-nifi-audit/xasecure.audit.destination.hdfs', False)
+
+
+ #For SQLA explicitly disable audit to DB for Ranger
+ if xa_audit_db_flavor == 'sqla':
+ xa_audit_db_is_enabled = False
+
+ nifi_authorizer = 'ranger-provider'
+
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] if has_namenode else None
+hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] if has_namenode else None
+hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] if has_namenode else None
+hdfs_site = config['configurations']['hdfs-site'] if has_namenode else None
+default_fs = config['configurations']['core-site']['fs.defaultFS'] if has_namenode else None
+hadoop_bin_dir = stack_select.get_hadoop_dir("bin") if has_namenode else None
+hadoop_conf_dir = conf_select.get_hadoop_conf_dir() if has_namenode else None
+
+import functools
+#create partial functions with common arguments for every HdfsResource call
+#to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code
+HdfsResource = functools.partial(
+ HdfsResource,
+ user=hdfs_user,
+ hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore",
+ security_enabled = security_enabled,
+ keytab = hdfs_user_keytab,
+ kinit_path_local = kinit_path_local,
+ hadoop_bin_dir = hadoop_bin_dir,
+ hadoop_conf_dir = hadoop_conf_dir,
+ principal_name = hdfs_principal_name,
+ hdfs_site = hdfs_site,
+ default_fs = default_fs,
+ immutable_paths = get_not_managed_resources()
+)
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/run_ca.sh
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/run_ca.sh b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/run_ca.sh
new file mode 100644
index 0000000..c5863fa
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/run_ca.sh
@@ -0,0 +1,63 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Launches NiFi CA server
+# $1 -> JAVA_HOME
+# $2 -> tls-toolkit.sh path
+# $3 -> config json
+# $4 -> stdout log
+# $5 -> stderr log
+# $6 -> pid file
+
+die() {
+ echo "$1"
+ exit 1
+}
+
+read -r -d '' WAIT_FOR_LOG << 'EOF'
+ STDOUT_LOG="$0"
+ until [ -f "$STDOUT_LOG" ]; do
+ echo "Waiting for $STDOUT_LOG to exist"
+ sleep 1;
+ done
+EOF
+
+read -r -d '' WAIT_FOR_CHILD << 'EOF'
+ SHELL_PID="$0"
+ PID_FILE="$1"
+ CHILD_PROCESS="$(pgrep -P "$SHELL_PID" java)"
+ while [ -z "$CHILD_PROCESS" ]; do
+ echo "Waiting for child java process to exist"
+ sleep 1;
+ CHILD_PROCESS="$(pgrep -P "$SHELL_PID" java)"
+ done
+ echo "$CHILD_PROCESS" > "$PID_FILE"
+EOF
+
+JAVA_HOME="$1" nohup "$2" server -F -f "$3" > "$4" 2> "$5" < /dev/null &
+SHELL_PID="$!"
+
+timeout 30 bash -c "$WAIT_FOR_LOG" "$4" || die "Timed out while waiting for $4"
+
+timeout 30 bash -c "$WAIT_FOR_CHILD" "$SHELL_PID" "$6" || die "Timed out while waiting for child java process to exist"
+
+#Want to wait until Jetty starts
+#See http://superuser.com/questions/270529/monitoring-a-file-until-a-string-is-found#answer-900134
+( tail -f -n +1 "$4" & ) | timeout 180 grep -q "Server Started" || die "Timed out while waiting for CA server to start"
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/service_check.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/service_check.py
new file mode 100644
index 0000000..535592f
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/service_check.py
@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+import urllib2
+import httplib
+
+from resource_management.core.logger import Logger
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.script.script import Script
+
+from ambari_commons.inet_utils import openurl
+from ambari_commons.exceptions import TimeoutError
+from resource_management.core.exceptions import Fail
+from resource_management.libraries.functions.decorator import retry
+
+class NifiServiceCheck(Script):
+ def service_check(self, env):
+ import params
+ Logger.info("Running Nifi service check")
+ for nifi_master_host in params.nifi_master_hosts:
+ url = ""
+ if params.nifi_ssl_enabled:
+ url = "https://{0}:{1}/nifi".format(nifi_master_host, params.nifi_node_ssl_port)
+ else:
+ url = "http://{0}:{1}/nifi".format(nifi_master_host, params.nifi_node_port)
+ Logger.info("Checking Nifi portal {0} status".format(url))
+ NifiServiceCheck.check_nifi_portal(url)
+
+ @staticmethod
+ @retry(times=15, sleep_time=5, max_sleep_time=20, backoff_factor=2, err_class=Fail)
+ def check_nifi_portal(url):
+ try:
+ request = urllib2.Request(url)
+ result = openurl(request, timeout=20)
+ response_code = result.getcode()
+ if response_code == 200 or response_code == 401:
+ Logger.info("Nifi portal {0} is up. Response code {1}".format(url, response_code))
+ else:
+ raise Fail("Error connecting to {0}. Response code {1}".format(url, response_code))
+ except urllib2.URLError, e:
+ if isinstance(e, urllib2.HTTPError):
+ if e.code == 401:
+ Logger.info("Nifi portal {0} is up. Response code {1}".format(url, e.code))
+ else:
+ raise Fail("Error connecting to {0}. Http status code - {1}. \n {2}".format(url, e.code, e.read()))
+ elif e.reason and "violation of protocol" in str(e.reason):
+ Logger.info("Ignore certificate validation error - {0}".format(e.reason))
+ pass
+ else:
+ raise Fail("Error connecting to {0}. Reason - {1}.".format(url, e.reason))
+ except httplib.BadStatusLine:
+ raise Fail("Error connecting to {0}. Reason - Not Reachable".format(url))
+ except TimeoutError:
+ raise Fail("Error connecting to {0}. Reason - Timeout".format(url))
+
+if __name__ == "__main__":
+ NifiServiceCheck().execute()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/setup_ranger_nifi.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/setup_ranger_nifi.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/setup_ranger_nifi.py
new file mode 100644
index 0000000..91a6885
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/setup_ranger_nifi.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from resource_management.core.logger import Logger
+from resource_management.core.resources import File
+
+def setup_ranger_nifi(upgrade_type=None):
+ import params, os
+
+ if params.has_ranger_admin and params.enable_ranger_nifi:
+
+ stack_version = None
+ if upgrade_type is not None:
+ stack_version = params.version
+
+ if params.retryAble:
+ Logger.info("nifi: Setup ranger: command retry enables thus retrying if ranger admin is down !")
+ else:
+ Logger.info("nifi: Setup ranger: command retry not enabled thus skipping if ranger admin is down !")
+
+
+ api_version=None
+ if params.stack_supports_ranger_kerberos:
+ api_version='v2'
+ from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
+ setup_ranger_plugin('nifi', 'nifi', params.previous_jdbc_jar,
+ params.downloaded_custom_connector, params.driver_curl_source,
+ params.driver_curl_target, params.java_home,
+ params.repo_name, params.nifi_ranger_plugin_repo,
+ params.ranger_env, params.ranger_plugin_properties,
+ params.policy_user, params.policymgr_mgr_url,
+ params.enable_ranger_nifi, conf_dict=params.nifi_config_dir,
+ component_user=params.nifi_user, component_group=params.nifi_group, cache_service_list=['nifi'],
+ plugin_audit_properties=params.config['configurations']['ranger-nifi-audit'], plugin_audit_attributes=params.config['configuration_attributes']['ranger-nifi-audit'],
+ plugin_security_properties=params.config['configurations']['ranger-nifi-security'], plugin_security_attributes=params.config['configuration_attributes']['ranger-nifi-security'],
+ plugin_policymgr_ssl_properties=params.config['configurations']['ranger-nifi-policymgr-ssl'], plugin_policymgr_ssl_attributes=params.config['configuration_attributes']['ranger-nifi-policymgr-ssl'],
+ component_list=[], audit_db_is_enabled=params.xa_audit_db_is_enabled,
+ credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password,
+ ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
+ stack_version_override = stack_version, skip_if_rangeradmin_down= not params.retryAble,api_version=api_version,
+ is_security_enabled = params.security_enabled,
+ is_stack_supports_ranger_kerberos = params.stack_supports_ranger_kerberos,
+ component_user_principal=params.ranger_nifi_principal if params.security_enabled else None,
+ component_user_keytab=params.ranger_nifi_keytab if params.security_enabled else None)
+
+ #change permissions of ranger xml that were written to 0400
+ File(os.path.join(params.nifi_config_dir, 'ranger-nifi-audit.xml'), owner=params.nifi_user, group=params.nifi_group, mode=0400)
+ File(os.path.join(params.nifi_config_dir, 'ranger-nifi-security.xml'), owner=params.nifi_user, group=params.nifi_group, mode=0400)
+ File(os.path.join(params.nifi_config_dir, 'ranger-policymgr-ssl.xml'), owner=params.nifi_user, group=params.nifi_group, mode=0400)
+
+ else:
+ Logger.info('Ranger admin not installed')
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/status_params.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/status_params.py
new file mode 100755
index 0000000..9748661
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/status_params.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+import sys, os
+
+config = Script.get_config()
+
+nifi_pid_dir=config['configurations']['nifi-env']['nifi_pid_dir']
+#nifi_master_pid_file=nifi_pid_dir + '/nifi-master.pid'
+nifi_node_pid_file=nifi_pid_dir + '/nifi.pid'
+nifi_ca_pid_file=nifi_pid_dir + '/nifi-ca.pid'
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/templates/nifi.conf.j2
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/templates/nifi.conf.j2 b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/templates/nifi.conf.j2
new file mode 100644
index 0000000..97f5711
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/templates/nifi.conf.j2
@@ -0,0 +1,36 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{nifi_user}} - nofile {{nifi_user_nofile_limit}}
+{{nifi_user}} - nproc {{nifi_user_nproc_limit}}
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/quicklinks/quicklinks.json b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/quicklinks/quicklinks.json
new file mode 100644
index 0000000..bd27705
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/quicklinks/quicklinks.json
@@ -0,0 +1,35 @@
+{
+ "name": "default",
+ "description": "default quick links configuration",
+ "configuration": {
+ "protocol":
+ {
+ "type":"https",
+ "checks":[
+ {
+ "property":"nifi.node.ssl.isenabled",
+ "desired":"true",
+ "site":"nifi-ambari-ssl-config"
+ }
+ ]
+ },
+
+ "links": [
+ {
+ "name": "nifi_ui",
+ "label": "Nifi UI",
+ "requires_user_name": "false",
+ "component_name": "NIFI_MASTER",
+ "url":"%@://%@:%@/nifi",
+ "port":{
+ "http_property": "nifi.node.port",
+ "http_default_port": "9090",
+ "https_property": "nifi.node.ssl.port",
+ "https_default_port": "9091",
+ "regex": "^(\\d+)$",
+ "site": "nifi-ambari-config"
+ }
+ }
+ ]
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/widgets.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/widgets.json b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/widgets.json
new file mode 100644
index 0000000..cd6791b
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/widgets.json
@@ -0,0 +1,463 @@
+{
+ "layouts": [
+ {
+ "layout_name": "default_nifi_dashboard",
+ "display_name": "Standard NiFi Dashboard",
+ "section_name": "NIFI_SUMMARY",
+ "widgetLayoutInfo": [
+ {
+ "widget_name": "FlowFiles Received Last 5 mins",
+ "description": "The number of FlowFiles received in the last 5 minutes.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "FlowFilesReceivedLast5Minutes",
+ "metric_path": "metrics/nifi/FlowFilesReceivedLast5Minutes",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "FlowFiles Received",
+ "value": "${FlowFilesReceivedLast5Minutes}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "MBs Received Last 5 mins",
+ "description": "The number of MBs received in the last 5 minutes.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "BytesReceivedLast5Minutes",
+ "metric_path": "metrics/nifi/BytesReceivedLast5Minutes",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "MBs Received",
+ "value": "${BytesReceivedLast5Minutes/1024000}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "FlowFiles Sent Last 5 mins",
+ "description": "The number of FlowFiles sent in the last 5 minutes.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "FlowFilesSentLast5Minutes",
+ "metric_path": "metrics/nifi/FlowFilesSentLast5Minutes",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "FlowFiles Sent",
+ "value": "${FlowFilesSentLast5Minutes}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "MBs Sent Last 5 mins",
+ "description": "The number of MBs sent in the last 5 minutes.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "BytesSentLast5Minutes",
+ "metric_path": "metrics/nifi/BytesSentLast5Minutes",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "MBs Sent",
+ "value": "${BytesSentLast5Minutes/1024000}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "FlowFiles Queued",
+ "description": "The number of FlowFiles queued.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "FlowFilesQueued",
+ "metric_path": "metrics/nifi/FlowFilesQueued",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "FlowFiles Queued",
+ "value": "${FlowFilesQueued}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "MBs Queued",
+ "description": "The number of MBs queued.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "BytesQueued",
+ "metric_path": "metrics/nifi/BytesQueued",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "MBs Queued",
+ "value": "${BytesQueued/1024000}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "MBs Read Last 5 mins",
+ "description": "The number of MBs read in the last 5 minutes.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "BytesReadLast5Minutes",
+ "metric_path": "metrics/nifi/BytesReadLast5Minutes",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "MBs Read",
+ "value": "${BytesReadLast5Minutes/1024000}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "MBs Written Last 5 mins",
+ "description": "The number of MBs written in the last 5 minutes.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "BytesWrittenLast5Minutes",
+ "metric_path": "metrics/nifi/BytesWrittenLast5Minutes",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "MBs Written",
+ "value": "${BytesWrittenLast5Minutes/1024000}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "Active Threads",
+ "description": "The number of active threads.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "ActiveThreads",
+ "metric_path": "metrics/nifi/ActiveThreads",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "Active Threads",
+ "value": "${ActiveThreads}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "Total Task Duration Milliseconds",
+ "description": "The total task duration in Milliseconds.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "TotalTaskDurationNanoSeconds",
+ "metric_path": "metrics/nifi/TotalTaskDurationNanoSeconds",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "Total Task Duration Milliseconds",
+ "value": "${TotalTaskDurationNanoSeconds/1000}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "JVM Heap Used (MBs)",
+ "description": "Number of MBs being used on the JVM Heap.",
+ "widget_type": "NUMBER",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "jvm.heap_used",
+ "metric_path": "metrics/jvm/heap_used",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "JVM Heap Used",
+ "value": "${jvm.heap_used/1024/1024}"
+ }
+ ],
+ "properties": {
+ "warning_threshold": "100000",
+ "error_threshold": "200000"
+ }
+ },
+ {
+ "widget_name": "JVM Heap Usage",
+ "description": "Precentage of JVM heap being used.",
+ "widget_type": "GAUGE",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "jvm.heap_usage",
+ "metric_path": "metrics/jvm/heap_usage",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "JVM Heap Usage",
+ "value": "${jvm.heap_usage}"
+ }
+ ],
+ "properties": {
+ "warning_threshold": "0.8",
+ "error_threshold": "0.9"
+ }
+ },
+ {
+ "widget_name": "JVM File Descriptor Usage",
+ "description": "Precentage of file descriptors being used.",
+ "widget_type": "GAUGE",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "jvm.file_descriptor_usage",
+ "metric_path": "metrics/jvm/file_descriptor_usage",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "JVM File Descriptor Usage",
+ "value": "${jvm.file_descriptor_usage}"
+ }
+ ],
+ "properties": {
+ "warning_threshold": "0.8",
+ "error_threshold": "0.9"
+ }
+ },
+ {
+ "widget_name": "Thread Count",
+ "description": "Number of threads in the JVM.",
+ "widget_type": "NUMBER",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "jvm.thread_count",
+ "metric_path": "metrics/jvm/thread_count",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "Thread Count",
+ "value": "${jvm.thread_count}"
+ }
+ ],
+ "properties": {
+ }
+ },
+ {
+ "widget_name": "Daemon Thread Count",
+ "description": "Number of daemon threads in the JVM.",
+ "widget_type": "NUMBER",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "jvm.daemon_thread_count",
+ "metric_path": "metrics/jvm/daemon_thread_count",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "Daemon Thread Count",
+ "value": "${jvm.daemon_thread_count}"
+ }
+ ],
+ "properties": {
+ }
+ },
+ {
+ "widget_name": "Runnable Threads",
+ "description": "Number of runnable threads in the JVM.",
+ "widget_type": "GRAPH",
+ "is_visible": false,
+ "metrics": [
+ {
+ "name": "jvm.thread_states.runnable",
+ "metric_path": "metrics/jvm/thread_states/runnable",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "Runnable Threads",
+ "value": "${jvm.thread_states.runnable}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "Blocked Threads",
+ "description": "Number of blocked threads in the JVM.",
+ "widget_type": "GRAPH",
+ "is_visible": false,
+ "metrics": [
+ {
+ "name": "jvm.thread_states.blocked",
+ "metric_path": "metrics/jvm/thread_states/blocked",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "Blocked Threads",
+ "value": "${jvm.thread_states.blocked}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "Timed-Waiting Threads",
+ "description": "Number of timed-waiting threads in the JVM.",
+ "widget_type": "GRAPH",
+ "is_visible": false,
+ "metrics": [
+ {
+ "name": "jvm.thread_states.timed_waiting",
+ "metric_path": "metrics/jvm/thread_states/timed_waiting",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "Timed-Waiting Threads",
+ "value": "${jvm.thread_states.timed_waiting}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "Terminated Threads",
+ "description": "Number of terminated threads in the JVM.",
+ "widget_type": "GRAPH",
+ "is_visible": false,
+ "metrics": [
+ {
+ "name": "jvm.thread_states.terminated",
+ "metric_path": "metrics/jvm/thread_states/terminated",
+ "service_name": "NIFI",
+ "component_name": "NIFI_MASTER"
+ }
+ ],
+ "values": [
+ {
+ "name": "Terminated Threads",
+ "value": "${jvm.thread_states.terminated}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/hooks/after_install.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/hooks/after_install.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/hooks/after_install.py
new file mode 100644
index 0000000..8f0743b
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/hooks/after_install.py
@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import sys
+import os
+from ambari_server.serverConfiguration import get_ambari_properties, get_resources_location
+from resource_management.core import sudo
+
+def main():
+ properties = get_ambari_properties()
+ if properties == -1:
+ print >> sys.stderr, "Error getting ambari properties"
+ return -1
+
+ resources_location = get_resources_location(properties)
+ views_dir = os.path.join(resources_location, "views")
+
+ for file in os.listdir(views_dir):
+ path = os.path.join(views_dir, file)
+ if os.path.isfile(path):
+ if "ambari-admin" in path or "storm-view" in path:
+ print "Keeping views jar : " + path
+ else:
+ print "Deleting views jar : " + path
+ sudo.unlink(path)
+ else:
+ print "Deleting views directory : " + path
+ sudo.rmtree(path)
+ return 0
+
+if __name__ == "__main__":
+ exit (main())
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/mpack.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/mpack.json b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/mpack.json
new file mode 100644
index 0000000..865561e
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/mpack.json
@@ -0,0 +1,30 @@
+{
+ "type" : "full-release",
+ "name" : "hdf-ambari-mpack",
+ "version": "${project.version}",
+ "description" : "HDF Ambari Management Pack",
+ "prerequisites": {
+ "min-ambari-version" : "${minAmbariVersion}",
+ "max-ambari-version" : "${maxAmbariVersion}"
+ },
+ "hooks": [
+ {
+ "name": "after-install",
+ "type": "python",
+ "script": "hooks/after_install.py"
+ }
+ ],
+ "artifacts": [
+ {
+ "name" : "hdf-service-definitions",
+ "type" : "service-definitions",
+ "source_dir": "common-services"
+ },
+ {
+ "name" : "hdf-stack-definitions",
+ "type" : "stack-definitions",
+ "source_dir": "stacks"
+ }
+ ]
+}
+