You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by mm...@apache.org on 2016/12/13 18:25:45 UTC
[3/4] incubator-metron git commit: METRON-580: Remove hard-coded
Metron version from Ambari MPack code (mmiklavc) closes
apache/incubator-metron#364
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/metron_service.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/metron_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/metron_service.py
deleted file mode 100644
index 57da2c7..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/metron_service.py
+++ /dev/null
@@ -1,76 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-
-import json
-import subprocess
-
-from resource_management.core.logger import Logger
-from resource_management.core.resources.system import Directory, File
-from resource_management.core.resources.system import Execute
-from resource_management.core.source import InlineTemplate
-from resource_management.libraries.functions import format as ambari_format
-
-
-def init_config():
- Logger.info('Loading config into ZooKeeper')
- Execute(ambari_format(
- "{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_quorum}"),
- path=ambari_format("{java_home}/bin")
- )
-
-
-def get_running_topologies():
- Logger.info('Getting Running Storm Topologies from Storm REST Server')
-
- cmd = ambari_format('curl --max-time 3 {storm_rest_addr}/api/v1/topology/summary')
- proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
- (stdout, stderr) = proc.communicate()
-
- try:
- stormjson = json.loads(stdout)
- except ValueError:
- return {}
-
- topologiesDict = {}
-
- for topology in stormjson['topologies']:
- topologiesDict[topology['name']] = topology['status']
-
- Logger.info("Topologies: " + str(topologiesDict))
- return topologiesDict
-
-
-def load_global_config(params):
- Logger.info('Create Metron Local Config Directory')
- Logger.info("Configure Metron global.json")
-
- directories = [params.metron_zookeeper_config_path]
- Directory(directories,
- mode=0755,
- owner=params.metron_user,
- group=params.metron_group
- )
-
- File("{0}/global.json".format(params.metron_zookeeper_config_path),
- owner=params.metron_user,
- content=InlineTemplate(params.global_json_template)
- )
-
- File("{0}/elasticsearch.properties".format(params.metron_zookeeper_config_path + '/..'),
- owner=params.metron_user,
- content=InlineTemplate(params.global_properties_template))
-
- init_config()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/mysql_server.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/mysql_server.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/mysql_server.py
deleted file mode 100755
index fcc83af..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/mysql_server.py
+++ /dev/null
@@ -1,63 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.libraries.script.script import Script
-
-import mysql_users
-from mysql_service import mysql_service
-from mysql_utils import mysql_configure
-from resource_management.core.resources.packaging import Package
-from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
-
-class MysqlServer(Script):
- def install(self, env):
- self.install_packages(env)
- self.configure(env)
-
- def clean(self, env):
- from params import params
- env.set_params(params)
- if params.install_mysql == 'Yes':
- mysql_users.mysql_deluser()
-
- def configure(self, env, upgrade_type=None, config_dir=None):
- from params import params
- env.set_params(params)
- mysql_configure()
-
- def start(self, env, rolling_restart=False):
- from params import params
- env.set_params(params)
- mysql_service(daemon_name=params.daemon_name, action='start')
-
- def stop(self, env, rolling_restart=False):
- from params import params
- env.set_params(params)
- mysql_service(daemon_name=params.daemon_name, action='stop')
-
- def status(self, env):
- from params import status_params
- env.set_params(status_params)
-
- mysql_service(daemon_name=status_params.daemon_name, action='status')
-
-
-if __name__ == "__main__":
- MysqlServer().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/mysql_service.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/mysql_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/mysql_service.py
deleted file mode 100755
index 2e0ce8b..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/mysql_service.py
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.core.exceptions import ComponentIsNotRunning, Fail
-from resource_management.core.resources.system import Execute
-from resource_management.libraries.functions.format import format
-
-
-def mysql_service(daemon_name=None, action='start'):
- status_cmd = format("pgrep -l '^{mysql_process_name}$'")
- cmd = ('service', daemon_name, action)
-
- if action == 'status':
- try:
- Execute(status_cmd)
- except Fail:
- raise ComponentIsNotRunning()
- elif action == 'stop':
- Execute(cmd,
- logoutput=True,
- only_if=status_cmd,
- sudo=True,
- )
- elif action == 'start':
- Execute(cmd,
- logoutput=True,
- not_if=status_cmd,
- sudo=True,
- )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/mysql_users.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/mysql_users.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/mysql_users.py
deleted file mode 100755
index b778e85..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/mysql_users.py
+++ /dev/null
@@ -1,58 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.core.resources.system import Execute, File
-from resource_management.core.source import StaticFile
-from resource_management.libraries.functions.format import format
-
-
-# Used to add metron access to the needed components
-def mysql_adduser():
- from params import params
-
- File(params.mysql_adduser_path,
- mode=0755,
- content=StaticFile('addMysqlUser.sh')
- )
-
- add_user_cmd = format("bash -x {mysql_adduser_path} {metron_user} {enrichment_metron_user_passwd!p} {mysql_host} {mysql_admin_password!p}")
- Execute(add_user_cmd,
- tries=3,
- try_sleep=5,
- logoutput=False,
- path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'
- )
-
-
-# Removes hive metron from components
-def mysql_deluser():
- from params import params
-
- File(params.mysql_deluser_path,
- mode=0755,
- content=StaticFile('removeMysqlUser.sh')
- )
-
- del_user_cmd = format("bash -x {mysql_deluser_path} {metron_user} {mysql_host} {mysql_admin_password!p}")
- Execute(del_user_cmd,
- tries=3,
- try_sleep=5,
- path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
- )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/mysql_utils.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/mysql_utils.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/mysql_utils.py
deleted file mode 100755
index 63cdb38..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/mysql_utils.py
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.core.resources.system import Execute, File
-from resource_management.core.source import StaticFile
-from resource_management.libraries.functions.format import format
-from mysql_service import mysql_service
-
-import mysql_users
-
-
-def mysql_configure():
- from params import params
-
- if params.install_mysql == 'Yes':
- mysql_service(daemon_name=params.daemon_name, action='start')
-
- replace_bind_address = ('sed', '-i', 's|^bind-address[ \t]*=.*|bind-address = 0.0.0.0|', params.mysql_configname)
- Execute(replace_bind_address,
- sudo=True,
- )
-
- mysql_users.mysql_adduser()
-
- File(params.mysql_create_geoip_path,
- mode=0755,
- content=StaticFile('createMysqlGeoIp.sh')
- )
-
- geoip_setup_cmd = format("bash -x {mysql_create_geoip_path} {geoip_ddl} {geoip_url} {mysql_admin_password!p}")
-
- Execute(geoip_setup_cmd,
- tries=3,
- try_sleep=5,
- path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
- )
-
- if params.install_mysql == 'Yes':
- mysql_service(daemon_name=params.daemon_name, action='stop')
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/__init__.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/__init__.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/__init__.py
deleted file mode 100755
index 242460e..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/__init__.py
+++ /dev/null
@@ -1,18 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/params.py
deleted file mode 100755
index 953435d..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/params.py
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-from ambari_commons import OSCheck
-from resource_management.libraries.functions.default import default
-from resource_management.libraries.functions.expect import expect
-
-if OSCheck.is_windows_family():
- from params_windows import *
-else:
- from params_linux import *
-
-java_home = config['hostLevelParams']['java_home']
-java_version = expect("/hostLevelParams/java_version", int)
-
-host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/params_linux.py
deleted file mode 100755
index 6b3052d..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/params_linux.py
+++ /dev/null
@@ -1,190 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-import functools
-import os
-
-from ambari_commons.os_check import OSCheck
-from resource_management.libraries.functions import conf_select
-from resource_management.libraries.functions import format
-from resource_management.libraries.functions import get_kinit_path
-from resource_management.libraries.functions import stack_select
-from resource_management.libraries.functions.default import default
-from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
-from resource_management.libraries.functions.is_empty import is_empty
-from resource_management.libraries.resources.hdfs_resource import HdfsResource
-from resource_management.libraries.script import Script
-
-import status_params
-
-# server configurations
-config = Script.get_config()
-tmp_dir = Script.get_tmp_dir()
-
-hostname = config['hostname']
-user_group = config['configurations']['cluster-env']['user_group']
-metron_home = status_params.metron_home
-parsers = status_params.parsers
-metron_ddl_dir = metron_home + '/ddl'
-geoip_ddl = metron_ddl_dir + '/geoip_ddl.sql'
-geoip_url = config['configurations']['metron-env']['geoip_url']
-metron_indexing_topology = status_params.metron_indexing_topology
-metron_user = config['configurations']['metron-env']['metron_user']
-metron_group = config['configurations']['metron-env']['metron_group']
-metron_config_path = metron_home + '/config'
-metron_zookeeper_config_dir = status_params.metron_zookeeper_config_dir
-metron_zookeeper_config_path = status_params.metron_zookeeper_config_path
-parsers_configured_flag_file = status_params.parsers_configured_flag_file
-enrichment_configured_flag_file = status_params.enrichment_configured_flag_file
-indexing_configured_flag_file = status_params.indexing_configured_flag_file
-global_json_template = config['configurations']['metron-env']['global-json']
-global_properties_template = config['configurations']['metron-env']['elasticsearch-properties']
-
-# Elasticsearch hosts and port management
-es_cluster_name = config['configurations']['metron-env']['es_cluster_name']
-es_hosts = config['configurations']['metron-env']['es_hosts']
-es_host_list = es_hosts.split(",")
-es_binary_port = config['configurations']['metron-env']['es_binary_port']
-es_url = ",".join([host + ":" + es_binary_port for host in es_host_list])
-es_http_port = config['configurations']['metron-env']['es_http_port']
-es_http_url = es_host_list[0] + ":" + es_http_port
-
-# install repo
-yum_repo_type = config['configurations']['metron-env']['repo_type']
-if yum_repo_type == 'local':
- repo_url = 'file:///localrepo'
-else:
- repo_url = config['configurations']['metron-env']['repo_url']
-
-# hadoop params
-stack_root = Script.get_stack_root()
-hadoop_home_dir = stack_select.get_hadoop_dir("home")
-hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
-hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
-kafka_home = os.path.join(stack_root, "current", "kafka-broker")
-kafka_bin_dir = os.path.join(kafka_home, "bin")
-
-# zookeeper
-zk_hosts = default("/clusterHostInfo/zookeeper_hosts", [])
-has_zk_host = not len(zk_hosts) == 0
-zookeeper_quorum = None
-if has_zk_host:
- if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']:
- zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort']
- else:
- zookeeper_clientPort = '2181'
- zookeeper_quorum = (':' + zookeeper_clientPort + ',').join(config['clusterHostInfo']['zookeeper_hosts'])
- # last port config
- zookeeper_quorum += ':' + zookeeper_clientPort
-
-# Storm
-storm_rest_addr = status_params.storm_rest_addr
-
-# Kafka
-kafka_hosts = default("/clusterHostInfo/kafka_broker_hosts", [])
-has_kafka_host = not len(kafka_hosts) == 0
-kafka_brokers = None
-if has_kafka_host:
- if 'port' in config['configurations']['kafka-broker']:
- kafka_broker_port = config['configurations']['kafka-broker']['port']
- else:
- kafka_broker_port = '6667'
- kafka_brokers = (':' + kafka_broker_port + ',').join(config['clusterHostInfo']['kafka_broker_hosts'])
- kafka_brokers += ':' + kafka_broker_port
-
-metron_apps_hdfs_dir = config['configurations']['metron-env']['metron_apps_hdfs_dir']
-# the double "format" is not an error - we are pulling in a jinja-templated param. This is a bit of a hack, but works
-# well enough until we find a better way via Ambari
-metron_apps_indexed_hdfs_dir = format(format(config['configurations']['metron-env']['metron_apps_indexed_hdfs_dir']))
-metron_topic_retention = config['configurations']['metron-env']['metron_topic_retention']
-
-local_grok_patterns_dir = format("{metron_home}/patterns")
-hdfs_grok_patterns_dir = format("{metron_apps_hdfs_dir}/patterns")
-
-# for create_hdfs_directory
-security_enabled = config['configurations']['cluster-env']['security_enabled']
-hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
-hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
-hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
-smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
-kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
-hdfs_site = config['configurations']['hdfs-site']
-default_fs = config['configurations']['core-site']['fs.defaultFS']
-dfs_type = default("/commandParams/dfs_type", "")
-
-# MYSQL
-if OSCheck.is_ubuntu_family():
- mysql_configname = '/etc/mysql/my.cnf'
-else:
- mysql_configname = '/etc/my.cnf'
-
-daemon_name = status_params.daemon_name
-
-install_mysql = config['configurations']['metron-env']['install_mysql']
-mysql_admin_password = config['configurations']['metron-env']['mysql_admin_password']
-
-# There will always be exactly one mysql_host
-mysql_host = config['clusterHostInfo']['metron_enrichment_mysql_server_hosts'][0]
-
-mysql_port = config['configurations']['metron-env']['metron_enrichment_db_port']
-
-mysql_adduser_path = tmp_dir + "/addMysqlUser.sh"
-mysql_deluser_path = tmp_dir + "/removeMysqlUser.sh"
-mysql_create_geoip_path = tmp_dir + "/createMysqlGeoIp.sh"
-
-enrichment_metron_user = config['configurations']['metron-env']['metron_enrichment_db_user']
-enrichment_metron_user_passwd = config['configurations']['metron-env']['metron_enrichment_db_password']
-enrichment_metron_user_passwd = unicode(enrichment_metron_user_passwd) if not is_empty(
- enrichment_metron_user_passwd) else enrichment_metron_user_passwd
-mysql_process_name = status_params.mysql_process_name
-
-# create partial functions with common arguments for every HdfsResource call
-# to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code
-HdfsResource = functools.partial(
- HdfsResource,
- user=hdfs_user,
- hdfs_resource_ignore_file="/var/lib/ambari-agent/data/.hdfs_resource_ignore",
- security_enabled=security_enabled,
- keytab=hdfs_user_keytab,
- kinit_path_local=kinit_path_local,
- hadoop_bin_dir=hadoop_bin_dir,
- hadoop_conf_dir=hadoop_conf_dir,
- principal_name=hdfs_principal_name,
- hdfs_site=hdfs_site,
- default_fs=default_fs,
- immutable_paths=get_not_managed_resources(),
- dfs_type=dfs_type
-)
-
-# HBase
-enrichment_table = status_params.enrichment_table
-enrichment_cf = status_params.enrichment_cf
-threatintel_table = status_params.threatintel_table
-threatintel_cf = status_params.threatintel_cf
-
-metron_enrichment_topology = status_params.metron_enrichment_topology
-metron_enrichment_topic = status_params.metron_enrichment_topic
-metron_enrichment_error_topic = status_params.metron_enrichment_error_topic
-metron_threat_intel_error_topic = status_params.metron_threat_intel_error_topic
-
-# ES Templates
-bro_index_path = tmp_dir + "/bro_index.template"
-snort_index_path = tmp_dir + "/snort_index.template"
-yaf_index_path = tmp_dir + "/yaf_index.template"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/params_windows.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/params_windows.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/params_windows.py
deleted file mode 100755
index 4d11b35..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/params_windows.py
+++ /dev/null
@@ -1,20 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-raise NotImplementedError
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/status_params.py
deleted file mode 100644
index 92c9d06..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/status_params.py
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from ambari_commons import OSCheck
-from resource_management.libraries.functions import format
-from resource_management.libraries.script import Script
-
-config = Script.get_config()
-
-# Parsers
-parsers = config['configurations']['metron-env']['parsers']
-metron_home = config['configurations']['metron-env']['metron_home']
-metron_zookeeper_config_dir = config['configurations']['metron-env']['metron_zookeeper_config_dir']
-metron_zookeeper_config_path = format('{metron_home}/{metron_zookeeper_config_dir}')
-parsers_configured_flag_file = metron_zookeeper_config_path + '/../metron_parsers_configured'
-
-# Enrichment
-metron_enrichment_topology = 'enrichment'
-metron_enrichment_topic = 'enrichments'
-metron_enrichment_error_topic = 'enrichments_error'
-metron_threat_intel_error_topic = 'threatintel_error'
-
-enrichment_table = 'enrichment'
-enrichment_cf = 't'
-threatintel_table = 'threatintel'
-threatintel_cf = 't'
-
-mysql_process_name = 'mysqld'
-if OSCheck.is_suse_family() or OSCheck.is_ubuntu_family():
- daemon_name = 'mysql'
-else:
- daemon_name = 'mysqld'
-
-# ing
-metron_indexing_topology = config['configurations']['metron-env']['metron_indexing_topology']
-indexing_configured_flag_file = metron_zookeeper_config_path + '/../metron_indexing_configured'
-
-# Enrichment
-enrichment_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_configured'
-
-# Storm
-storm_rest_addr = config['configurations']['metron-env']['storm_rest_addr']
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/parser_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/parser_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/parser_commands.py
deleted file mode 100755
index d43333c..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/parser_commands.py
+++ /dev/null
@@ -1,208 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-import os
-import re
-import subprocess
-import time
-
-from resource_management.core.logger import Logger
-from resource_management.core.resources.system import Execute, File
-
-import metron_service
-
-
-# Wrap major operations and functionality in this class
-class ParserCommands:
- __params = None
- __parser_list = None
- __configured = False
-
- def __init__(self, params):
- if params is None:
- raise ValueError("params argument is required for initialization")
- self.__params = params
- self.__parser_list = self.__get_parsers(params)
- self.__configured = os.path.isfile(self.__params.parsers_configured_flag_file)
-
- # get list of parsers
- def __get_parsers(self, params):
- return params.parsers.replace(' ', '').split(',')
-
- def is_configured(self):
- return self.__configured
-
- def set_configured(self):
- File(self.__params.parsers_configured_flag_file,
- content="",
- owner=self.__params.metron_user,
- mode=0775)
-
- def init_parsers(self):
- Logger.info(
- "Copying grok patterns from local directory '{0}' to HDFS '{1}'".format(self.__params.local_grok_patterns_dir,
- self.__params.hdfs_grok_patterns_dir))
- self.__params.HdfsResource(self.__params.hdfs_grok_patterns_dir,
- type="directory",
- action="create_on_execute",
- owner=self.__params.metron_user,
- mode=0775,
- source=self.__params.local_grok_patterns_dir)
-
- Logger.info("Done initializing parser configuration")
-
- def get_parser_list(self):
- return self.__parser_list
-
- def setup_repo(self):
- def local_repo():
- Logger.info("Setting up local repo")
- Execute("yum -y install createrepo")
- Execute("createrepo /localrepo")
- Execute("chmod -R o-w+r /localrepo")
- Execute("echo \"[METRON-0.3.0]\n"
- "name=Metron 0.3.0 packages\n"
- "baseurl=file:///localrepo\n"
- "gpgcheck=0\n"
- "enabled=1\" > /etc/yum.repos.d/local.repo")
-
- def remote_repo():
- print('Using remote repo')
-
- yum_repo_types = {
- 'local': local_repo,
- 'remote': remote_repo
- }
- repo_type = self.__params.yum_repo_type
- if repo_type in yum_repo_types:
- yum_repo_types[repo_type]()
- else:
- raise ValueError("Unsupported repo type '{0}'".format(repo_type))
-
- def init_kafka_topics(self):
- Logger.info('Creating Kafka topics')
- command_template = """{0}/kafka-topics.sh \
- --zookeeper {1} \
- --create \
- --topic {2} \
- --partitions {3} \
- --replication-factor {4} \
- --config retention.bytes={5}"""
- num_partitions = 1
- replication_factor = 1
- retention_gigabytes = int(self.__params.metron_topic_retention)
- retention_bytes = retention_gigabytes * 1024 * 1024 * 1024
- Logger.info("Creating main topics for parsers")
- for parser_name in self.get_parser_list():
- Logger.info("Creating topic'{0}'".format(parser_name))
- Execute(command_template.format(self.__params.kafka_bin_dir,
- self.__params.zookeeper_quorum,
- parser_name,
- num_partitions,
- replication_factor,
- retention_bytes))
- Logger.info("Creating topics for error handling")
- Execute(command_template.format(self.__params.kafka_bin_dir,
- self.__params.zookeeper_quorum,
- "parser_invalid",
- num_partitions,
- replication_factor,
- retention_bytes))
- Execute(command_template.format(self.__params.kafka_bin_dir,
- self.__params.zookeeper_quorum,
- "parser_error",
- num_partitions, replication_factor,
- retention_bytes))
- Logger.info("Done creating Kafka topics")
-
- def start_parser_topologies(self):
- Logger.info("Starting Metron parser topologies: {0}".format(self.get_parser_list()))
- start_cmd_template = """{0}/bin/start_parser_topology.sh \
- -k {1} \
- -z {2} \
- -s {3}"""
- for parser in self.get_parser_list():
- Logger.info('Starting ' + parser)
- Execute(start_cmd_template.format(self.__params.metron_home, self.__params.kafka_brokers,
- self.__params.zookeeper_quorum, parser))
-
- Logger.info('Finished starting parser topologies')
-
- def stop_parser_topologies(self):
- Logger.info('Stopping parsers')
- for parser in self.get_parser_list():
- Logger.info('Stopping ' + parser)
- stop_cmd = 'storm kill ' + parser
- Execute(stop_cmd)
- Logger.info('Done stopping parser topologies')
-
- def restart_parser_topologies(self, env):
- Logger.info('Restarting the parser topologies')
- self.stop_parser_topologies()
- attempt_count = 0
- while self.topologies_running(env):
- if attempt_count > 2:
- raise Exception("Unable to kill topologies")
- attempt_count += 1
- time.sleep(10)
- self.start_parser_topologies()
- Logger.info('Done restarting the parser topologies')
-
- def topologies_exist(self):
- cmd_open = subprocess.Popen(["storm", "list"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- (stdout, stderr) = cmd_open.communicate()
- stdout_lines = stdout.splitlines()
- if stdout_lines:
- status_lines = self.__get_status_lines(stdout_lines)
- for parser in self.get_parser_list():
- for line in status_lines:
- items = re.sub('[\s]+', ' ', line).split()
- if items and items[0] == parser:
- return True
- return False
-
- def topologies_running(self, env):
- env.set_params(self.__params)
- all_running = True
- topologies = metron_service.get_running_topologies()
- for parser in self.get_parser_list():
- parser_found = False
- is_running = False
- if parser in topologies:
- parser_found = True
- is_running = topologies[parser] in ['ACTIVE', 'REBALANCING']
- all_running &= parser_found and is_running
- return all_running
-
- def __get_status_lines(self, lines):
- status_lines = []
- do_stat = False
- skipped = 0
- for line in lines:
- if line.startswith("Topology_name"):
- do_stat = True
- if do_stat and skipped == 2:
- status_lines += [line]
- elif do_stat:
- skipped += 1
- return status_lines
-
- def __is_running(self, status):
- return status in ['ACTIVE', 'REBALANCING']
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/parser_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/parser_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/parser_master.py
deleted file mode 100755
index e6d2ac6..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/parser_master.py
+++ /dev/null
@@ -1,88 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.core.exceptions import ComponentIsNotRunning
-from resource_management.core.logger import Logger
-from resource_management.libraries.script import Script
-
-import metron_service
-from parser_commands import ParserCommands
-
-
-class ParserMaster(Script):
- def get_component_name(self):
- # TODO add this at some point - currently will cause problems with hdp-select
- # return "parser-master"
- pass
-
- def install(self, env):
- from params import params
- env.set_params(params)
- commands = ParserCommands(params)
- commands.setup_repo()
- self.install_packages(env)
-
- def configure(self, env, upgrade_type=None, config_dir=None):
- from params import params
- env.set_params(params)
- metron_service.load_global_config(params)
- commands = ParserCommands(params)
- if not commands.is_configured():
- commands.init_parsers()
- commands.init_kafka_topics()
- commands.set_configured()
-
- def start(self, env, upgrade_type=None):
- from params import params
- env.set_params(params)
- self.configure(env)
- commands = ParserCommands(params)
- commands.start_parser_topologies()
-
- def stop(self, env, upgrade_type=None):
- from params import params
- env.set_params(params)
- commands = ParserCommands(params)
- commands.stop_parser_topologies()
-
- def status(self, env):
- from params import status_params
- env.set_params(status_params)
- commands = ParserCommands(status_params)
- if not commands.topologies_running(env):
- raise ComponentIsNotRunning()
-
- def restart(self, env):
- from params import params
- env.set_params(params)
- self.configure(env)
- commands = ParserCommands(params)
- commands.restart_parser_topologies(env)
-
- def servicechecktest(self, env):
- from params import params
- env.set_params(params)
- from service_check import ServiceCheck
- service_check = ServiceCheck()
- Logger.info('Service Check Test')
- service_check.service_check(env)
-
-
-if __name__ == "__main__":
- ParserMaster().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/service_check.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/service_check.py
deleted file mode 100755
index 7dd9dfb..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/service_check.py
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-from __future__ import print_function
-
-from resource_management.libraries.script import Script
-
-from indexing_commands import IndexingCommands
-from parser_commands import ParserCommands
-
-
-class ServiceCheck(Script):
- def service_check(self, env):
- from params import params
- parsercommands = ParserCommands(params)
- indexingcommands = IndexingCommands(params)
- all_found = parsercommands.topologies_running(env) and indexingcommands.is_topology_active(env)
- if all_found:
- exit(0)
- else:
- exit(1)
-
-
-if __name__ == "__main__":
- ServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/templates/enrichment.properties.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/templates/enrichment.properties.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/templates/enrichment.properties.j2
deleted file mode 100755
index 14403de..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/templates/enrichment.properties.j2
+++ /dev/null
@@ -1,90 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-##### Kafka #####
-
-kafka.zk={{zookeeper_quorum}}
-kafka.broker={{kafka_brokers}}
-enrichment.output.topic=indexing
-enrichment.error.topic=enrichments_error
-threat.intel.error.topic=threatintel_error
-
-##### MySQL #####
-
-mysql.ip={{mysql_host}}
-mysql.port={{mysql_port}}
-mysql.username={{enrichment_metron_user}}
-mysql.password={{enrichment_metron_user_passwd}}
-
-##### Metrics #####
-
-#reporters
-org.apache.metron.metrics.reporter.graphite=true
-org.apache.metron.metrics.reporter.console=false
-org.apache.metron.metrics.reporter.jmx=false
-
-#Graphite Addresses
-
-org.apache.metron.metrics.graphite.address=localhost
-org.apache.metron.metrics.graphite.port=2023
-
-#TelemetryParserBolt
-org.apache.metron.metrics.TelemetryParserBolt.acks=true
-org.apache.metron.metrics.TelemetryParserBolt.emits=true
-org.apache.metron.metrics.TelemetryParserBolt.fails=true
-
-
-#GenericEnrichmentBolt
-org.apache.metron.metrics.GenericEnrichmentBolt.acks=true
-org.apache.metron.metrics.GenericEnrichmentBolt.emits=true
-org.apache.metron.metrics.GenericEnrichmentBolt.fails=true
-
-
-#TelemetryIndexingBolt
-org.apache.metron.metrics.TelemetryIndexingBolt.acks=true
-org.apache.metron.metrics.TelemetryIndexingBolt.emits=true
-org.apache.metron.metrics.TelemetryIndexingBolt.fails=true
-
-##### Host Enrichment #####
-
-hbase.provider.impl=org.apache.metron.hbase.HTableProvider
-enrichment.simple.hbase.table={{enrichment_table}}
-enrichment.simple.hbase.cf={{enrichment_cf}}
-org.apache.metron.enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\
-{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},\
-{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}]
-
-
-##### HBase #####
-bolt.hbase.table.name=pcap
-bolt.hbase.table.fields=t:value
-bolt.hbase.table.key.tuple.field.name=key
-bolt.hbase.table.timestamp.tuple.field.name=timestamp
-bolt.hbase.enable.batching=false
-bolt.hbase.write.buffer.size.in.bytes=2000000
-bolt.hbase.durability=SKIP_WAL
-bolt.hbase.partitioner.region.info.refresh.interval.mins=60
-
-##### Threat Intel #####
-
-threat.intel.tracker.table={{threatintel_table}}
-threat.intel.tracker.cf={{threatintel_cf}}
-threat.intel.simple.hbase.table={{threatintel_table}}
-threat.intel.simple.hbase.cf={{threatintel_cf}}
-threat.intel.ip.table=
-threat.intel.ip.cf=
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/quicklinks/quicklinks.json
deleted file mode 100755
index ee1b225..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/quicklinks/quicklinks.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
- "name": "default",
- "description": "default quick links configuration",
- "configuration": {
- "protocol":
- {
- "type":"HTTP_ONLY"
- },
-
- "links": [
- {
- "name": "storm_ui",
- "label": "Storm UI",
- "requires_user_name": "false",
- "component_name": "STORM_UI_SERVER",
- "url":"%@://%@:%@/",
- "port":{
- "http_property": "ui.port",
- "http_default_port": "8744",
- "https_property": "ui.port",
- "https_default_port": "8744",
- "regex": "^(\\d+)$",
- "site": "storm-site"
- }
- }
- ]
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/service_advisor.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/service_advisor.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/service_advisor.py
deleted file mode 100644
index edb919b..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/service_advisor.py
+++ /dev/null
@@ -1,159 +0,0 @@
-#!/usr/bin/env ambari-python-wrap
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-import os
-import fnmatch
-import imp
-import socket
-import sys
-import traceback
-
-SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
-STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../../../stacks/')
-PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py')
-
-try:
- with open(PARENT_FILE, 'rb') as fp:
- service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE))
-except Exception as e:
- traceback.print_exc()
- print "Failed to load parent"
-
-class METRON030ServiceAdvisor(service_advisor.ServiceAdvisor):
-
- def getServiceComponentLayoutValidations(self, services, hosts):
-
- componentsListList = [service["components"] for service in services["services"]]
- componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist]
-
- metronParsersHost = self.getHosts(componentsList, "METRON_PARSERS")[0]
- metronEnrichmentMaster = self.getHosts(componentsList, "METRON_ENRICHMENT_MASTER")[0]
- metronIndexingHost = self.getHosts(componentsList, "METRON_INDEXING")[0]
- metronEnrichmentMysqlServer = self.getHosts(componentsList, "METRON_ENRICHMENT_MYSQL_SERVER")[0]
-
- hbaseClientHosts = self.getHosts(componentsList, "HBASE_CLIENT")
- hdfsClientHosts = self.getHosts(componentsList, "HDFS_CLIENT")
- zookeeperClientHosts = self.getHosts(componentsList, "ZOOKEEPER_CLIENT")
-
- kafkaBrokers = self.getHosts(componentsList, "KAFKA_BROKER")
- stormSupervisors = self.getHosts(componentsList, "SUPERVISOR")
-
- items = []
-
- # Metron Must Co-locate with KAFKA_BROKER and STORM_SUPERVISOR
- if metronParsersHost not in kafkaBrokers:
- message = "Metron must be colocated with an instance of KAFKA BROKER"
- items.append({ "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'METRON_PARSERS', "host": metronParsersHost })
-
- if metronParsersHost not in stormSupervisors:
- message = "Metron must be colocated with an instance of STORM SUPERVISOR"
- items.append({ "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'METRON_PARSERS', "host": metronParsersHost })
-
- if metronParsersHost != metronEnrichmentMaster:
- message = "Metron Enrichment Master must be co-located with Metron Parsers on {0}".format(metronParsersHost)
- items.append({ "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'METRON_ENRICHMENT_MASTER', "host": metronEnrichmentMaster })
-
- if metronParsersHost != metronIndexingHost:
- message = "Metron Indexing must be co-located with Metron Parsers on {0}".format(metronParsersHost)
- items.append({ "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'METRON_INDEXING', "host": metronIndexingHost })
-
- if metronParsersHost != metronEnrichmentMysqlServer:
- message = "Metron MySQL Server must be co-located with Metron Parsers on {0}".format(metronParsersHost)
- items.append({ "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'METRON_ENRICHMENT_MYSQL_SERVER', "host": metronEnrichmentMysqlServer })
-
- # Enrichment Master also needs ZK Client, but this is already guaranteed by being colocated with Parsers Master
- if metronParsersHost not in zookeeperClientHosts:
- message = "Metron must be co-located with an instance of Zookeeper Client"
- items.append({ "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'METRON_PARSERS', "host": metronParsersHost })
-
- # Enrichment Master also needs HDFS clients, but this is already guaranteed by being colocated with Parsers Master
- if metronParsersHost not in hdfsClientHosts:
- message = "Metron must be co-located with an instance of HDFS Client"
- items.append({ "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'METRON_PARSERS', "host": metronParsersHost })
-
- if metronEnrichmentMaster not in hbaseClientHosts:
- message = "Metron Enrichment Master must be co-located with an instance of HBase Client"
- items.append({ "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'METRON_ENRICHMENT_MASTER', "host": metronEnrichmentMaster })
-
- return items
-
- def getServiceConfigurationsValidationItems(self, configurations, recommendedDefaults, services, hosts):
-
- # validate recommended properties in storm-site
- siteName = "storm-site"
- method = self.validateSTORMSiteConfigurations
- items = self.validateConfigurationsForSite(configurations, recommendedDefaults, services, hosts, siteName, method)
-
- return items
-
- def getServiceConfigurationRecommendations(self, configurations, clusterData, services, hosts):
- #Suggest mysql server hostname
- mySQLServerHost = self.getComponentHostNames(services, "METRON", "METRON_ENRICHMENT_MYSQL_SERVER")[0]
- putMetronEnvProperty = self.putProperty(configurations, "metron-env", services)
- putMetronEnvProperty("mysql_host",mySQLServerHost)
-
- #Suggest Storm Rest URL
- if "storm-site" in services["configurations"]:
- stormUIServerHost = self.getComponentHostNames(services, "STORM", "STORM_UI_SERVER")[0]
- stormUIServerPort = services["configurations"]["storm-site"]["properties"]["ui.port"]
- stormUIServerURL = stormUIServerHost + ":" + stormUIServerPort
- putMetronEnvProperty = self.putProperty(configurations, "metron-env", services)
- putMetronEnvProperty("storm_rest_addr",stormUIServerURL)
-
- storm_site = services["configurations"]["storm-site"]["properties"]
- putStormSiteProperty = self.putProperty(configurations, "storm-site", services)
-
- for property, desired_value in self.getSTORMSiteDesiredValues().iteritems():
- if property not in storm_site:
- putStormSiteProperty(property, desired_value)
- elif property == "topology.classpath" and storm_site[property] != desired_value:
- topololgyClasspath = storm_site[property]
- #check that desired values exist in topology.classpath. append them if they do not
- for path in desired_value.split(':'):
- if path not in topololgyClasspath:
- topololgyClasspath += ":" + path
- putStormSiteProperty(property,topololgyClasspath)
-
- def validateSTORMSiteConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
-
- storm_site = properties
- validationItems = []
-
- for property, desired_value in self.getSTORMSiteDesiredValues().iteritems():
- if property not in storm_site :
- message = "Metron requires this property to be set to the recommended value of " + desired_value
- item = self.getErrorItem(message) if property == "topology.classpath" else self.getWarnItem(message)
- validationItems.append({"config-name": property, "item": item})
- elif storm_site[property] != desired_value:
- topologyClasspath = storm_site[property]
- for path in desired_value.split(':'):
- if path not in topologyClasspath:
- message = "Metron requires this property to contain " + desired_value
- item = self.getErrorItem(message)
- validationItems.append({"config-name": property, "item": item})
-
- return self.toConfigurationValidationProblems(validationItems, "storm-site")
-
- def getSTORMSiteDesiredValues(self):
-
- storm_site_desired_values = {
- "topology.classpath" : "/etc/hbase/conf:/etc/hadoop/conf"
- }
-
- return storm_site_desired_values
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/themes/metron_theme.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/themes/metron_theme.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/themes/metron_theme.json
deleted file mode 100644
index b939182..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/themes/metron_theme.json
+++ /dev/null
@@ -1,219 +0,0 @@
-{
- "configuration": {
- "layouts": [
- {
- "name": "default",
- "tabs": [
- {
- "name": "metron_general",
- "display-name": "Settings",
- "layout": {
- "tab-columns": "1",
- "tab-rows": "1",
- "sections": [
- {
- "name": "section-general",
- "row-index": "0",
- "column-index": "0",
- "row-span": "1",
- "column-span": "1",
- "section-columns": "2",
- "section-rows": "1",
- "subsections": [
- {
- "name": "subsection-general-database",
- "display-name" : "Database Settings",
- "row-index": "0",
- "column-index": "0",
- "row-span": "1",
- "column-span": "1"
- },
- {
- "name": "subsection-general-indexing",
- "display-name": "Index Settings",
- "row-index": "0",
- "column-index": "1",
- "row-span": "1",
- "column-span": "1"
- }
- ]
- }
- ]
- }
- },
- {
- "name": "metron_repo",
- "display-name" : "Repository",
- "layout": {
- "tab-columns" : "1",
- "tab-rows" : "1",
- "sections" : [
- {
- "name" : "section-repo",
- "row-index": "0",
- "column-index" : "0",
- "row-span": "1",
- "column-span": "1",
- "section-columns": "1",
- "section-rows": "1",
- "subsections": [
- {
- "name": "subsection-repo",
- "row-index": "0",
- "column-index": "0",
- "row-span": "1",
- "column-span": "1"
- }
- ]
- }
- ]
- }
- }
- ]
- }
- ],
- "placement": {
- "configuration-layout": "default",
- "configs": [
- {
- "config": "metron-env/install_mysql",
- "subsection-name": "subsection-general-database"
- },
- {
- "config": "metron-env/mysql_admin_password",
- "subsection-name": "subsection-general-database",
- "depends-on": [
- {
- "configs":[
- "metron-env/install_mysql"
- ],
- "if": "${metron-env/install_mysql} === No",
- "then": {
- "property_value_attributes": {
- "visible": true
- }
- },
- "else": {
- "property_value_attributes": {
- "visible": false
- }
- }
- }
- ]
- },
- {
- "config": "metron-env/metron_enrichment_db_password",
- "subsection-name": "subsection-general-database"
- },
- {
- "config": "metron-env/metron_enrichment_db_port",
- "subsection-name": "subsection-general-database"
- },
- {
- "config": "metron-env/es_hosts",
- "subsection-name": "subsection-general-indexing"
- },
- {
- "config": "metron-env/es_binary_port",
- "subsection-name": "subsection-general-indexing"
- },
- {
- "config": "metron-env/es_http_port",
- "subsection-name": "subsection-general-indexing"
- },
- {
- "config": "metron-env/es_cluster_name",
- "subsection-name": "subsection-general-indexing"
- },
- {
- "config": "metron-env/repo_type",
- "subsection-name": "subsection-repo"
- },
- {
- "config": "metron-env/repo_url",
- "subsection-name": "subsection-repo",
- "depends-on": [
- {
- "configs":[
- "metron-env/repo_type"
- ],
- "if": "${metron-env/repo_type} === remote",
- "then": {
- "property_value_attributes": {
- "visible": true
- }
- },
- "else": {
- "property_value_attributes": {
- "visible": false
- }
- }
- }
- ]
- }
- ]
- },
- "widgets": [
- {
- "config": "metron-env/install_mysql",
- "widget": {
- "type": "toggle"
- }
- },
- {
- "config": "metron-env/mysql_admin_password",
- "widget": {
- "type": "password"
- }
- },
- {
- "config": "metron-env/metron_enrichment_db_password",
- "widget": {
- "type": "password"
- }
- },
- {
- "config": "metron-env/metron_enrichment_db_port",
- "widget": {
- "type": "text-field"
- }
- },
- {
- "config": "metron-env/es_hosts",
- "widget": {
- "type": "text-field"
- }
- },
- {
- "config": "metron-env/es_binary_port",
- "widget": {
- "type": "text-field"
- }
- },
- {
- "config": "metron-env/es_http_port",
- "widget": {
- "type": "text-field"
- }
- },
- {
- "config": "metron-env/es_cluster_name",
- "widget": {
- "type": "text-field"
- }
- },
- {
- "config": "metron-env/repo_type",
- "widget": {
- "type": "toggle"
- }
- },
- {
- "config": "metron-env/repo_url",
- "widget": {
- "type": "text-field"
- }
- }
- ]
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/64a49ada/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
new file mode 100644
index 0000000..7484d31
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
@@ -0,0 +1,262 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration supports_final="true">
+ <property>
+ <name>metron_home</name>
+ <value>/usr/metron/${metron.version}</value>
+ <description>Metron home directory</description>
+ <display-name>Metron home</display-name>
+ </property>
+ <property>
+ <name>metron_apps_hdfs_dir</name>
+ <value>/apps/metron</value>
+ <description>Metron apps HDFS dir</description>
+ <display-name>Metron apps HDFS dir</display-name>
+ </property>
+ <property>
+ <name>metron_apps_indexed_hdfs_dir</name>
+ <value>{{metron_apps_hdfs_dir}}/indexing/indexed</value>
+ <description>Indexing bolts will write to this HDFS directory</description>
+ <display-name>Metron apps indexed HDFS dir</display-name>
+ </property>
+ <property>
+ <name>metron_zookeeper_config_dir</name>
+ <value>config/zookeeper</value>
+ <description>Metron Zookeeper config dir. Relative path to Metron home.</description>
+ <display-name>Metron Zookeeper config dir</display-name>
+ </property>
+ <property>
+ <name>metron_user</name>
+ <value>metron</value>
+ <property-type>USER</property-type>
+ <description>The user for Metron</description>
+ <display-name>Metron User</display-name>
+ </property>
+ <property>
+ <name>metron_group</name>
+ <value>metron</value>
+ <property-type>GROUP</property-type>
+ <description>The group for Metron</description>
+ <display-name>Metron Group Name</display-name>
+ </property>
+ <property>
+ <name>metron_topic_retention</name>
+ <description>Kafka Retention in GB</description>
+ <value>10</value>
+ <display-name>Topic Retention</display-name>
+ </property>
+ <property>
+ <name>parsers</name>
+ <value>bro,snort,yaf</value>
+ <description>Metron parsers to deploy</description>
+ <display-name>Metron Parsers</display-name>
+ </property>
+ <property>
+ <name>metron_enrichment_db_user</name>
+ <value>metron</value>
+ <description>Database username to use to connect to the database.</description>
+ <display-name>Enrichment Database User</display-name>
+ </property>
+ <property>
+ <name>metron_enrichment_db_port</name>
+ <display-name>Metron Enrichment Database Port</display-name>
+ <value>3306</value>
+ <description>Database port to use to connect to the database.</description>
+ </property>
+ <property>
+ <name>metron_enrichment_db_password</name>
+ <value></value>
+ <property-type>PASSWORD</property-type>
+ <display-name>Metron Enrichment Database Password</display-name>
+ <description>Password to use against database</description>
+ <value-attributes>
+ <type>password</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>mysql_admin_password</name>
+ <value></value>
+ <property-type>PASSWORD</property-type>
+ <display-name>MySQL root user password</display-name>
+ <description>Password to use to add Metron user to MySQL</description>
+ <value-attributes>
+ <type>password</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>metron_indexing_topology</name>
+ <value>indexing</value>
+ <description>The Storm topology name for Indexing</description>
+ <display-name>Indexing Topology Name</display-name>
+ </property>
+ <property>
+ <name>es_cluster_name</name>
+ <value>metron</value>
+ <description>Name of Elasticsearch Cluster</description>
+ <display-name>Elasticsearch Cluster Name</display-name>
+ </property>
+ <property>
+ <name>geoip_url</name>
+ <value>http://geolite.maxmind.com/download/geoip/database/GeoLiteCity_CSV/GeoLiteCity-latest.tar.xz</value>
+ <description>Location of the GeoIP data to load.</description>
+ <display-name>GEOIP Load Datafile URL</display-name>
+ </property>
+ <property require-input="true">
+ <name>es_hosts</name>
+ <value></value>
+ <description>Comma delimited list of Elasticsearch Hosts. (eshost1,eshost2)</description>
+ <display-name>Elasticsearch Hosts</display-name>
+ </property>
+ <property>
+ <name>es_binary_port</name>
+ <value>9300</value>
+ <description>Elasticsearch binary port. (9300)</description>
+ <display-name>Elasticsearch Binary Port</display-name>
+ </property>
+ <property>
+ <name>es_http_port</name>
+ <value>9200</value>
+ <description>Elasticsearch HTTP port. (9200)</description>
+ <display-name>Elasticsearch HTTP port</display-name>
+ </property>
+ <property require-input = "true">
+ <name>storm_rest_addr</name>
+ <display-name>Storm Rest Server Address</display-name>
+ <description>URL of Storm UI (storm.ui.hostname:8744)</description>
+ <value></value>
+ </property>
+ <property>
+ <name>repo_type</name>
+ <display-name>Repository Type</display-name>
+ <description>Type of Repository: Local or Remote</description>
+ <value>local</value>
+ <value-attributes>
+ <overridable>false</overridable>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>local</value>
+ <label>Local</label>
+ </entry>
+ <entry>
+ <value>remote</value>
+ <label>Remote</label>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ </property>
+ <property>
+ <name>repo_url</name>
+ <display-name>Repository URL</display-name>
+ <value></value>
+ </property>
+ <property>
+ <name>install_mysql</name>
+ <description>Install New MySQL Instance for Enrichments</description>
+ <display-name>Install MySQL</display-name>
+ <value>Yes</value>
+ <value-attributes>
+ <overridable>false</overridable>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>Yes</value>
+ <label>Yes</label>
+ </entry>
+ <entry>
+ <value>No</value>
+ <label>No</label>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ </property>
+ <property>
+ <name>mysql_host</name>
+ <display-name>MySQL Host Address</display-name>
+ <value></value>
+ </property>
+ <property>
+ <name>global-json</name>
+ <display-name>global.json template</display-name>
+ <description>This is the jinja template for global.json file</description>
+ <value>
+{
+"es.clustername": "{{ es_cluster_name }}",
+"es.ip": "{{ es_url }}",
+"es.date.format": "yyyy.MM.dd.HH"
+}
+ </value>
+ <value-attributes>
+ <type>content</type>
+ </value-attributes>
+ </property>
+ <property>
+ <name>elasticsearch-properties</name>
+ <description>The template for the elasticsearch.properties file.</description>
+ <display-name>elasticsearch.properties template</display-name>
+ <value>
+##### Storm #####
+indexing.workers=1
+indexing.executors=0
+##### Kafka #####
+kafka.zk={{ zookeeper_quorum }}
+kafka.broker={{ kafka_brokers }}
+kafka.start=WHERE_I_LEFT_OFF
+##### Indexing #####
+index.input.topic=indexing
+index.error.topic=indexing_error
+writer.class.name=org.apache.metron.elasticsearch.writer.ElasticsearchWriter
+##### Metrics #####
+#reporters
+org.apache.metron.metrics.reporter.graphite=true
+org.apache.metron.metrics.reporter.console=false
+org.apache.metron.metrics.reporter.jmx=false
+#Graphite Addresses
+org.apache.metron.metrics.graphite.address=localhost
+org.apache.metron.metrics.graphite.port=2023
+#TelemetryParserBolt
+org.apache.metron.metrics.TelemetryParserBolt.acks=true
+org.apache.metron.metrics.TelemetryParserBolt.emits=true
+org.apache.metron.metrics.TelemetryParserBolt.fails=true
+##### HDFS #####
+bolt.hdfs.batch.size=5000
+bolt.hdfs.field.delimiter=|
+bolt.hdfs.rotation.policy=org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy
+bolt.hdfs.rotation.policy.units=DAYS
+bolt.hdfs.rotation.policy.count=1
+bolt.hdfs.file.rotation.size.in.mb=5
+bolt.hdfs.file.system.url={{ default_fs }}
+bolt.hdfs.wip.file.path=/paloalto/wip
+bolt.hdfs.finished.file.path=/paloalto/rotated
+bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec
+index.hdfs.output={{ metron_apps_indexed_hdfs_dir }}
+ </value>
+ <value-attributes>
+ <type>content</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+</configuration>