You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by le...@apache.org on 2016/09/26 14:05:05 UTC
[2/7] incubator-metron git commit: METRON-427 Create Ambari
Management Pack for Metron Installation closes apache/incubator-metron#266
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboardindex.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboardindex.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboardindex.py
deleted file mode 100755
index f0903ac..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboardindex.py
+++ /dev/null
@@ -1,95 +0,0 @@
-#!/usr/bin/python
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-from elasticsearch import Elasticsearch
-from elasticsearch.helpers import bulk
-import cPickle as pickle
-import argparse, sys, os.path
-import errno
-import os
-
-
-class DashboardIndex(object):
-
- def __init__(self, host='localhost', port=9200, url_prefix='', timeout=10, **kwargs):
- """
- :arg host: hostname of the node (default: localhost)
- :arg port: port to use (integer, default: 9200)
- :arg url_prefix: optional url prefix for elasticsearch
- :arg timeout: default timeout in seconds (float, default: 10)
- """
- self.es = Elasticsearch([{'host':host,'port': port, 'url_prefix': url_prefix, 'timeout':timeout}])
-
- def get(self):
- """
- Get .kibana index from Elasticsearch
- """
- dotkibana = self.es.search(index='.kibana', size = 100)
- return dotkibana['hits']['hits']
-
- def load(self,filespec):
- """
- Save Index data on local filesystem
- :args filespec: path/filename for saved file
- """
- data=[]
- with open(filespec,'rb') as fp:
- data = pickle.load(fp)
- return data
-
- def save(self,filename,data):
- """
- Save Index data on local filesystem
- :args filespec: path/filename for saved file
- """
- with open(filename,'wb') as fp:
- pickle.dump(data,fp)
-
- def put(self,data):
- """
- Bulk write data to Elasticsearch
- :args data: data to be written (note: index name is specified in data)
- """
- bulk(self.es,data)
-
- def main(self,args):
-
- if args.save:
- print("running save with host:%s on port %d, filespec: %s" % (args.hostname, args.port, args.filespec))
- self.save(filename=args.filespec,data=di.get())
- else:
- """
- Loads Kibana Dashboard definition from disk and replaces .kibana on index
- :args filespec: path/filename for saved file
- """
- if not os.path.isfile(args.filespec):
- raise IOError(
- errno.ENOENT, os.strerror(errno.ENOENT), args.filespec)
- self.es.indices.delete(index='.kibana', ignore=[400, 404])
- self.put(data=di.load(filespec=args.filespec))
-
-if __name__ == '__main__':
-
- parser = argparse.ArgumentParser()
- parser.add_argument("hostname", help="ES Hostname or IP", type=str)
- parser.add_argument("port", help="ES Port", type=int)
- parser.add_argument("filespec", help="file to be pushed from or saved to", type=str)
- parser.add_argument("-s","--save", help="run in SAVE mode - .kibana will be read and saved to filespec",action="store_true")
- args = parser.parse_args()
- di = DashboardIndex(host=args.hostname,port=args.port)
- di.main(args)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py
deleted file mode 100755
index 7913c8c..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py
+++ /dev/null
@@ -1,146 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-kibana_master
-
-"""
-
-
-from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
-from resource_management.core.resources.system import Execute
-from resource_management.libraries.script import Script
-from resource_management.libraries.functions.format import format
-from resource_management.core.logger import Logger
-from resource_management.core.resources.system import Directory
-from resource_management.core.resources.system import File
-from resource_management.core.source import InlineTemplate
-
-import errno
-import os
-
-class Kibana(Script):
-
- def install(self, env):
-
- import params
- env.set_params(params)
-
- Logger.info("Install Kibana Master")
-
- #TODO: Figure this out for all supported OSes
- Execute('rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch')
- Execute("echo \"[kibana-4.x]\n"
- "name=Kibana repository for 4.5.x packages\n"
- "baseurl=http://packages.elastic.co/kibana/4.5/centos\n"
- "gpgcheck=1\n"
- "gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch\n"
- "enabled=1\" > /etc/yum.repos.d/kibana.repo")
-
- self.install_packages(env)
- self.loadtemplate(env)
-
- def configure(self, env):
-
- import params
- env.set_params(params)
-
- Logger.info("Configure Kibana for Metron")
-
- directories = [params.log_dir, params.pid_dir, params.conf_dir]
- Directory(directories,
- # recursive=True,
- mode=0755,
- owner=params.kibana_user,
- group=params.kibana_user
- )
-
- File("{}/kibana.yml".format(params.conf_dir),
- owner=params.kibana_user,
- content=InlineTemplate(params.kibana_yml_template)
- )
-
- def stop(self, env):
-
- import params
- env.set_params(params)
-
- Logger.info("Stop Kibana Master")
-
- Execute("service kibana stop")
-
- def start(self, env):
-
- import params
- env.set_params(params)
-
- self.configure(env)
-
- Logger.info("Start the Master")
-
- Execute("service kibana start")
-
- def restart(self,env):
-
- import params
- env.set_params(params)
-
- self.configure(env)
-
- Logger.info("Restarting the Master")
-
- Execute("service kibana restart")
-
- def status(self, env):
-
- import params
- env.set_params(params)
-
- Logger.info("Status of the Master")
-
- Execute("service kibana status")
-
- @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
- def loadtemplate(self,env):
-
- from dashboard.dashboardindex import DashboardIndex
-
- import params
- env.set_params(params)
-
- hostname = format("{es_host}")
- port = int(format("{es_port}"))
-
- Logger.info("Connecting to Elasticsearch on host: %s, port: %s" % (hostname,port))
- di = DashboardIndex(host=hostname,port=port)
-
- #Loads Kibana Dashboard definition from disk and replaces .kibana on index
- templateFile = os.path.join(os.path.dirname(os.path.abspath(__file__)),'dashboard','dashboard.p')
- if not os.path.isfile(templateFile):
- raise IOError(
- errno.ENOENT, os.strerror(errno.ENOENT), templateFile)
-
- Logger.info("Deleting .kibana index from Elasticsearch")
-
- di.es.indices.delete(index='.kibana', ignore=[400, 404])
-
- Logger.info("Loading .kibana index from %s" % templateFile)
-
- di.put(data=di.load(filespec=templateFile))
-
-
-if __name__ == "__main__":
- Kibana().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py
deleted file mode 100755
index 03fae0b..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py
+++ /dev/null
@@ -1,47 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Kibana Params configurations
-
-"""
-
-
-from resource_management.libraries.script import Script
-from resource_management.libraries.functions import format
-from urlparse import urlparse
-
-# server configurations
-config = Script.get_config()
-
-kibana_home = '/usr/share/kibana/'
-kibana_bin = '/usr/share/kibana/bin/'
-
-conf_dir = "/opt/kibana/config/"
-kibana_user = config['configurations']['kibana-env']['kibana_user']
-user_group = config['configurations']['kibana-env']['user_group']
-log_dir = config['configurations']['kibana-env']['kibana_log_dir']
-pid_dir = config['configurations']['kibana-env']['kibana_pid_dir']
-pid_file = format("{pid_dir}/kibanasearch.pid")
-es_url = config['configurations']['kibana-env']['kibana_es_url']
-parsed = urlparse(es_url)
-es_host = parsed.netloc.split(':')[0]
-es_port = parsed.netloc.split(':')[1]
-kibana_port = config['configurations']['kibana-env']['kibana_server_port']
-hostname = config['hostname']
-java64_home = config['hostLevelParams']['java_home']
-kibana_yml_template = config['configurations']['kibana-site']['content']
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/quicklinks/quicklinks.json
deleted file mode 100755
index 448e102..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/quicklinks/quicklinks.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
- "name": "default",
- "description": "default quick links configuration",
- "configuration": {
- "protocol":
- {
- "type":"HTTP_ONLY"
- },
-
- "links": [
- {
- "name": "metron_ui",
- "label": "Metron UI",
- "requires_user_name": "false",
- "component_name": "KIBANA_MASTER",
- "url":"%@://%@:%@/",
- "port":{
- "http_property": "kibana_server_port",
- "http_default_port": "5601",
- "https_property": "kibana_server_port",
- "https_default_port": "5601",
- "regex": "^(\\d+)$",
- "site": "kibana-env"
- }
- }
- ]
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml
deleted file mode 100755
index 71966ea..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml
+++ /dev/null
@@ -1,57 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<configuration supports_final="true">
- <property>
- <name>metron_home</name>
- <value>/usr/metron/0.2.0BETA</value>
- <description>Metron home directory</description>
- <display-name>Metron home</display-name>
- </property>
- <property>
- <name>parsers</name>
- <value>bro,snort,yaf</value>
- <description>Metron parsers to deploy</description>
- <display-name>Metron parsers</display-name>
- </property>
- <property>
- <name>metron_apps_hdfs_dir</name>
- <value>/apps/metron</value>
- <description>Metron apps HDFS dir</description>
- <display-name>Metron apps HDFS dir</display-name>
- </property>
- <property>
- <name>metron_zookeeper_config_dir</name>
- <value>config/zookeeper</value>
- <description>Metron Zookeeper config dir. Relative path to Metron home.</description>
- <display-name>Metron Zookeeper config dir</display-name>
- </property>
- <property>
- <name>metron_user</name>
- <value>metron</value>
- <property-type>USER</property-type>
- <description>The user for Metron</description>
- <display-name>Metron User</display-name>
- </property>
- <property>
- <name>metron_group</name>
- <value>metron</value>
- <description>The group for Metron</description>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml
deleted file mode 100755
index f1e5ab7..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml
+++ /dev/null
@@ -1,108 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<metainfo>
- <schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>PARSERS</name>
- <displayName>Metron Parsers</displayName>
- <comment>Parsers</comment>
- <version>0.2.0BETA</version>
-
- <components>
- <component>
- <name>PARSER_MASTER</name>
- <displayName>Metron Parser Topologies</displayName>
- <category>MASTER</category>
- <cardinality>1</cardinality>
- <versionAdvertised>true</versionAdvertised>
- <dependencies>
- <dependency>
- <name>HDFS/HDFS_CLIENT</name>
- <scope>host</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- <dependency>
- <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
- <scope>cluster</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- <dependency>
- <name>KAFKA/KAFKA_BROKER</name>
- <scope>cluster</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- </dependencies>
- <commandScript>
- <script>scripts/parser_master.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- <configuration-dependencies>
- <config-type>metron-parsers</config-type>
- </configuration-dependencies>
- </component>
- </components>
-
- <osSpecifics>
- <osSpecific>
- <osFamily>any</osFamily>
- <packages>
- <package>
- <name>metron-common</name>
- </package>
- <package>
- <name>metron-parsers</name>
- </package>
- </packages>
- </osSpecific>
- </osSpecifics>
-
- <commandScript>
- <script>scripts/service_check.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>300</timeout>
- </commandScript>
-
- <requiredServices>
- <service>HDFS</service>
- <service>KAFKA</service>
- <service>STORM</service>
- <service>ZOOKEEPER</service>
- </requiredServices>
-
- <configuration-dependencies>
- <config-type>metron-parsers</config-type>
- </configuration-dependencies>
-
- <quickLinksConfigurations>
- <quickLinksConfiguration>
- <fileName>quicklinks.json</fileName>
- <default>true</default>
- </quickLinksConfiguration>
- </quickLinksConfigurations>
-
- </service>
- </services>
-</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py
deleted file mode 100755
index 6b33e55..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py
+++ /dev/null
@@ -1,235 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-import os
-import re
-import subprocess
-import time
-
-from resource_management.core.logger import Logger
-from resource_management.core.resources.system import Execute, File
-from resource_management.core.source import Template
-from resource_management.libraries.functions import format
-
-
-# Wrap major operations and functionality in this class
-class Commands:
- __params = None
- __parser_list = None
- __configured = False
-
- def __init__(self, params):
- if params is None:
- raise ValueError("params argument is required for initialization")
- self.__params = params
- self.__parser_list = self.__get_parsers(params)
- self.__configured = os.path.isfile(self.__params.configured_flag_file)
-
- # get list of parsers
- def __get_parsers(self, params):
- return params.parsers.replace(' ', '').split(',')
-
- def is_configured(self):
- return self.__configured
-
- def set_configured(self):
- File(self.__params.configured_flag_file,
- content="",
- owner=self.__params.metron_user,
- mode=0775)
-
- # Possible storm topology status states
- # http://storm.apache.org/releases/0.10.0/javadocs/backtype/storm/generated/TopologyStatus.html
- class StormStatus:
- ACTIVE = "ACTIVE"
- INACTIVE = "INACTIVE"
- KILLED = "KILLED"
- REBALANCING = "REBALANCING"
-
- def init_parsers(self):
- Logger.info(
- "Copying grok patterns from local directory '{}' to HDFS '{}'".format(self.__params.local_grok_patterns_dir,
- self.__params.metron_apps_dir))
- self.__params.HdfsResource(self.__params.metron_apps_dir,
- type="directory",
- action="create_on_execute",
- owner=self.__params.metron_user,
- mode=0775,
- source=self.__params.local_grok_patterns_dir)
-
- Logger.info("Creating global.json file")
- File(self.__params.metron_zookeeper_config_path + '/global.json',
- content=Template("metron-global.json"),
- owner=self.__params.metron_user,
- mode=0775)
- Logger.info("Done initializing parser configuration")
-
- def get_parser_list(self):
- return self.__parser_list
-
- def setup_repo(self):
- def local_repo():
- Logger.info("Setting up local repo")
- Execute("yum -y install createrepo")
- Execute("createrepo /localrepo")
- Execute("chmod -R o-w+r /localrepo")
- Execute("echo \"[METRON-0.2.0BETA]\n"
- "name=Metron 0.2.0BETA packages\n"
- "baseurl=file:///localrepo\n"
- "gpgcheck=0\n"
- "enabled=1\" > /etc/yum.repos.d/local.repo")
-
- def remote_repo():
- print('Using remote repo')
-
- yum_repo_types = {
- 'local': local_repo,
- 'remote': remote_repo
- }
- repo_type = self.__params.yum_repo_type
- if repo_type in yum_repo_types:
- yum_repo_types[repo_type]()
- else:
- raise ValueError("Unsupported repo type '{}'".format(repo_type))
-
- def init_kafka_topics(self):
- Logger.info('Creating Kafka topics')
- command_template = """{}/kafka-topics.sh \
- --zookeeper {} \
- --create \
- --topic {} \
- --partitions {} \
- --replication-factor {} \
- --config retention.bytes={}"""
- num_partitions = 1
- replication_factor = 1
- retention_gigabytes = 10
- retention_bytes = retention_gigabytes * 1024 * 1024 * 1024
- Logger.info("Creating main topics for parsers")
- for parser_name in self.get_parser_list():
- Logger.info("Creating topic'{}'".format(parser_name))
- Execute(command_template.format(self.__params.kafka_bin_dir,
- self.__params.zookeeper_quorum,
- parser_name,
- num_partitions,
- replication_factor,
- retention_bytes))
- Logger.info("Creating topics for error handling")
- Execute(command_template.format(self.__params.kafka_bin_dir,
- self.__params.zookeeper_quorum,
- "parser_invalid",
- num_partitions,
- replication_factor,
- retention_bytes))
- Execute(command_template.format(self.__params.kafka_bin_dir,
- self.__params.zookeeper_quorum,
- "parser_error",
- num_partitions, replication_factor,
- retention_bytes))
- Logger.info("Done creating Kafka topics")
-
- def init_parser_config(self):
- Logger.info('Loading parser config into ZooKeeper')
- Execute(format(
- "{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_quorum}"),
- path=format("{java_home}/bin")
- )
-
- def start_parser_topologies(self):
- Logger.info("Starting Metron parser topologies: {}".format(self.get_parser_list()))
- start_cmd_template = """{}/bin/start_parser_topology.sh \
- -k {} \
- -z {} \
- -s {}"""
- for parser in self.get_parser_list():
- Logger.info('Starting ' + parser)
- Execute(start_cmd_template.format(self.__params.metron_home, self.__params.kafka_brokers,
- self.__params.zookeeper_quorum, parser))
-
- Logger.info('Finished starting parser topologies')
-
- def stop_parser_topologies(self):
- Logger.info('Stopping parsers')
- for parser in self.get_parser_list():
- Logger.info('Stopping ' + parser)
- stop_cmd = 'storm kill ' + parser
- Execute(stop_cmd)
- Logger.info('Done stopping parser topologies')
-
- def restart_parser_topologies(self):
- Logger.info('Restarting the parser topologies')
- self.stop_parser_topologies()
- attempt_count = 0
- while self.topologies_exist():
- if attempt_count > 2:
- raise Exception("Unable to kill topologies")
- attempt_count += 1
- time.sleep(10)
- self.start_parser_topologies()
- Logger.info('Done restarting the parser topologies')
-
- def topologies_exist(self):
- cmd_open = subprocess.Popen(["storm", "list"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- (stdout, stderr) = cmd_open.communicate()
- stdout_lines = stdout.splitlines()
- if stdout_lines:
- status_lines = self.__get_status_lines(stdout_lines)
- for parser in self.get_parser_list():
- for line in status_lines:
- items = re.sub('[\s]+', ' ', line).split()
- if items and items[0] == parser:
- return True
- return False
-
- def topologies_running(self):
- cmd_open = subprocess.Popen(["storm", "list"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- (stdout, stderr) = cmd_open.communicate()
- stdout_lines = stdout.splitlines()
- all_running = False
- if stdout_lines:
- all_running = True
- status_lines = self.__get_status_lines(stdout_lines)
- for parser in self.get_parser_list():
- parser_found = False
- is_running = False
- for line in status_lines:
- items = re.sub('[\s]+', ' ', line).split()
- if items and items[0] == parser:
- status = items[1]
- parser_found = True
- is_running = self.__is_running(status)
- all_running &= parser_found and is_running
- return all_running
-
- def __get_status_lines(self, lines):
- status_lines = []
- do_stat = False
- skipped = 0
- for line in lines:
- if line.startswith("Topology_name"):
- do_stat = True
- if do_stat and skipped == 2:
- status_lines += [line]
- elif do_stat:
- skipped += 1
- return status_lines
-
- def __is_running(self, status):
- return status in [self.StormStatus.ACTIVE, self.StormStatus.REBALANCING]
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py
deleted file mode 100755
index 242460e..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py
+++ /dev/null
@@ -1,18 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params.py
deleted file mode 100755
index 953435d..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params.py
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-from ambari_commons import OSCheck
-from resource_management.libraries.functions.default import default
-from resource_management.libraries.functions.expect import expect
-
-if OSCheck.is_windows_family():
- from params_windows import *
-else:
- from params_linux import *
-
-java_home = config['hostLevelParams']['java_home']
-java_version = expect("/hostLevelParams/java_version", int)
-
-host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py
deleted file mode 100755
index de7dfc9..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py
+++ /dev/null
@@ -1,114 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-import functools
-import os
-
-from resource_management.libraries.functions import conf_select
-from resource_management.libraries.functions import format
-from resource_management.libraries.functions import get_kinit_path
-from resource_management.libraries.functions import stack_select
-from resource_management.libraries.functions.default import default
-from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
-from resource_management.libraries.resources.hdfs_resource import HdfsResource
-from resource_management.libraries.script import Script
-
-# server configurations
-config = Script.get_config()
-
-hostname = config['hostname']
-#print(config['configurations'])
-metron_home = config['configurations']['metron-parsers']['metron_home']
-parsers = config['configurations']['metron-parsers']['parsers']
-metron_user = config['configurations']['metron-parsers']['metron_user']
-metron_group = config['configurations']['metron-parsers']['metron_group']
-metron_zookeeper_config_dir = config['configurations']['metron-parsers']['metron_zookeeper_config_dir']
-#metron_zookeeper_config_path = metron_home + "/" + metron_zookeeper_config_dir
-metron_zookeeper_config_path = format('{metron_home}/{metron_zookeeper_config_dir}')
-configured_flag_file = metron_zookeeper_config_path + '/../metron_is_configured'
-yum_repo_type = 'local'
-
-# hadoop params
-stack_root = Script.get_stack_root()
-hadoop_home_dir = stack_select.get_hadoop_dir("home")
-hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
-hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
-kafka_home = os.path.join(stack_root, "current", "kafka-broker")
-kafka_bin_dir = os.path.join(kafka_home, "bin")
-
-# zookeeper
-zk_hosts = default("/clusterHostInfo/zookeeper_hosts", [])
-has_zk_host = not len(zk_hosts) == 0
-zookeeper_quorum = None
-if has_zk_host:
- if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']:
- zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort']
- else:
- zookeeper_clientPort = '2181'
- zookeeper_quorum = (':' + zookeeper_clientPort + ',').join(config['clusterHostInfo']['zookeeper_hosts'])
- # last port config
- zookeeper_quorum += ':' + zookeeper_clientPort
-
-# Kafka
-kafka_hosts = default("/clusterHostInfo/kafka_broker_hosts", [])
-has_kafka_host = not len(kafka_hosts) == 0
-kafka_brokers = None
-if has_kafka_host:
- if 'port' in config['configurations']['kafka-broker']:
- kafka_broker_port = config['configurations']['kafka-broker']['port']
- else:
- kafka_broker_port = '6667'
- kafka_brokers = (':' + kafka_broker_port + ',').join(config['clusterHostInfo']['kafka_broker_hosts'])
- kafka_brokers += ':' + kafka_broker_port
-
-metron_apps_dir = config['configurations']['metron-parsers']['metron_apps_hdfs_dir']
-
-local_grok_patterns_dir = format("{metron_home}/patterns")
-hdfs_grok_patterns_dir = format("{metron_apps_dir}/patterns")
-
-# for create_hdfs_directory
-security_enabled = config['configurations']['cluster-env']['security_enabled']
-hostname = config["hostname"]
-hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
-hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
-hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
-smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
-kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
-hdfs_site = config['configurations']['hdfs-site']
-default_fs = config['configurations']['core-site']['fs.defaultFS']
-dfs_type = default("/commandParams/dfs_type", "")
-
-# create partial functions with common arguments for every HdfsResource call
-# to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code
-HdfsResource = functools.partial(
- HdfsResource,
- user=hdfs_user,
- hdfs_resource_ignore_file="/var/lib/ambari-agent/data/.hdfs_resource_ignore",
- security_enabled=security_enabled,
- keytab=hdfs_user_keytab,
- kinit_path_local=kinit_path_local,
- hadoop_bin_dir=hadoop_bin_dir,
- hadoop_conf_dir=hadoop_conf_dir,
- principal_name=hdfs_principal_name,
- hdfs_site=hdfs_site,
- default_fs=default_fs,
- immutable_paths=get_not_managed_resources(),
- dfs_type=dfs_type
-)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_windows.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_windows.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_windows.py
deleted file mode 100755
index 4d11b35..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_windows.py
+++ /dev/null
@@ -1,20 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-raise NotImplementedError
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py
deleted file mode 100755
index ee83414..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-parsers = "bro,yaf,snort"
-configured_flag_file = ""
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py
deleted file mode 100755
index 4a5b910..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py
+++ /dev/null
@@ -1,77 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.core.exceptions import ComponentIsNotRunning
-from resource_management.core.logger import Logger
-from resource_management.libraries.script import Script
-
-from commands import Commands
-
-
-class ParserMaster(Script):
- def get_component_name(self):
- # TODO add this at some point - currently will cause problems with hdp-select
- # return "parser-master"
- pass
-
- def install(self, env):
- from params import params
- env.set_params(params)
- commands = Commands(params)
- commands.setup_repo()
- Logger.info('Install RPM packages')
- self.install_packages(env)
-
- def configure(self, env, upgrade_type=None, config_dir=None):
- from params import params
- env.set_params(params)
-
- def start(self, env, upgrade_type=None):
- from params import params
- env.set_params(params)
- commands = Commands(params)
- if not commands.is_configured():
- commands.init_parsers()
- commands.init_kafka_topics()
- commands.init_parser_config()
- commands.set_configured()
- commands.start_parser_topologies()
-
- def stop(self, env, upgrade_type=None):
- from params import params
- env.set_params(params)
- commands = Commands(params)
- commands.stop_parser_topologies()
-
- def status(self, env):
- from params import status_params
- env.set_params(status_params)
- commands = Commands(status_params)
- if not commands.topologies_running():
- raise ComponentIsNotRunning()
-
- def restart(self, env):
- from params import params
- env.set_params(params)
- commands = Commands(params)
- commands.restart_parser_topologies()
-
-
-if __name__ == "__main__":
- ParserMaster().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py
deleted file mode 100755
index 6ef7e4a..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-from __future__ import print_function
-
-from resource_management.libraries.script import Script
-
-from commands import Commands
-
-
-class ServiceCheck(Script):
- def service_check(self, env):
- from params import params
- env.set_params(params)
- commands = Commands(params)
- all_found = commands.topologies_running()
- if all_found:
- exit(0)
- else:
- exit(1)
-
-
-if __name__ == "__main__":
- ServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json
deleted file mode 100755
index 0967ef4..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json
+++ /dev/null
@@ -1 +0,0 @@
-{}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/quicklinks/quicklinks.json
deleted file mode 100755
index ee1b225..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/quicklinks/quicklinks.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
- "name": "default",
- "description": "default quick links configuration",
- "configuration": {
- "protocol":
- {
- "type":"HTTP_ONLY"
- },
-
- "links": [
- {
- "name": "storm_ui",
- "label": "Storm UI",
- "requires_user_name": "false",
- "component_name": "STORM_UI_SERVER",
- "url":"%@://%@:%@/",
- "port":{
- "http_property": "ui.port",
- "http_default_port": "8744",
- "https_property": "ui.port",
- "https_default_port": "8744",
- "regex": "^(\\d+)$",
- "site": "storm-site"
- }
- }
- ]
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py
deleted file mode 100755
index 84b466a..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py
+++ /dev/null
@@ -1,68 +0,0 @@
-#!/usr/bin/env ambari-python-wrap
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-import imp
-import os
-import traceback
-
-SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
-STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../stacks/')
-PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py')
-
-try:
- with open(PARENT_FILE, 'rb') as fp:
- service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE))
-except Exception as e:
- traceback.print_exc()
- print("Failed to load parent service_advisor file '{}'".format(PARENT_FILE))
-
-
-class TODO_PARSERS020BETAServiceAdvisor(service_advisor.ServiceAdvisor):
- # colocate Metron Parser Master with KAFKA_BROKERs
- def TODO_colocateService(self, hostsComponentsMap, serviceComponents):
- parsersMasterComponent = [component for component in serviceComponents if
- component["StackServiceComponents"]["component_name"] == "PARSER_MASTER"][0]
- if not self.isComponentHostsPopulated(parsersMasterComponent):
- for hostName in hostsComponentsMap.keys():
- hostComponents = hostsComponentsMap[hostName]
- if ({"name": "KAFKA_BROKER"} in hostComponents) and {"name": "PARSER_MASTER"} not in hostComponents:
- hostsComponentsMap[hostName].append({"name": "PARSER_MASTER"})
- if ({"name": "KAFKA_BROKER"} not in hostComponents) and {"name": "PARSER_MASTER"} in hostComponents:
- hostsComponentsMap[hostName].remove({"name": "PARSER_MASTER"})
-
- def TODO_getServiceComponentLayoutValidations(self, services, hosts):
- componentsListList = [service["components"] for service in services["services"]]
- componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist]
- hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
- hostsCount = len(hostsList)
-
- masterHosts = self.getHosts(componentsList, "PARSER_MASTER")
- expectedMasterHosts = set(self.getHosts(componentsList, "KAFKA_BROKER"))
-
- items = []
-
- mismatchHosts = sorted(expectedMasterHosts.symmetric_difference(set(masterHosts)))
- if len(mismatchHosts) > 0:
- hostsString = ', '.join(mismatchHosts)
- message = "Metron Parsers Master must be installed on Kafka Brokers. " \
- "The following {0} host(s) do not satisfy the colocation recommendation: {1}".format(
- len(mismatchHosts), hostsString)
- items.append(
- {"type": 'host-component', "level": 'WARN', "message": message, "component-name": 'PARSER_MASTER'})
-
- return items
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/metainfo.xml
deleted file mode 100755
index e0292cf..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/metainfo.xml
+++ /dev/null
@@ -1,27 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<metainfo>
- <schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>PARSERS</name>
- <version>0.2.0BETA</version>
- <extends>common-services/PARSERS/0.2.0BETA</extends>
- </service>
- </services>
-</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json
deleted file mode 100755
index c861646..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
- "_comment": "Record format:",
- "_comment": "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]",
- "general_deps": {
- "_comment": "dependencies for Metron Parsers",
- "PARSER_MASTER-START": [
- "NAMENODE-START",
- "ZOOKEEPER_SERVER-START",
- "KAFKA_BROKER-START",
- "STORM_REST_API-START"
- ],
- "_comment": "<SERVICE_NAME>_SERVICE_CHECK-SERVICE_CHECK",
- "PARSERS_SERVICE_CHECK-SERVICE_CHECK": [
- "PARSER_MASTER-START"
- ]
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/ELASTICSEARCH/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/ELASTICSEARCH/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/ELASTICSEARCH/metainfo.xml
deleted file mode 100755
index 361f5f5..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/ELASTICSEARCH/metainfo.xml
+++ /dev/null
@@ -1,41 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<metainfo>
- <schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>ELASTICSEARCH</name>
- <displayName>Elasticsearch</displayName>
- <comment>Indexing and Search</comment>
- <version>2.3.3</version>
- <extends>common-services/ELASTICSEARCH/2.3.3</extends>
- <osSpecifics>
- <osSpecific>
- <osFamily>any</osFamily>
- <packages>
- <package>
- <name>elasticsearch-2.3.3</name>
- </package>
- </packages>
- </osSpecific>
- </osSpecifics>
- </service>
- </services>
-</metainfo>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/metainfo.xml
deleted file mode 100755
index e406780..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/metainfo.xml
+++ /dev/null
@@ -1,26 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<metainfo>
- <schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>INDEXING</name>
- <version>0.2.0BETA</version>
- <extends>common-services/INDEXING/0.2.0BETA</extends>
- </service>
- </services>
-</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/role_command_order.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/role_command_order.json b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/role_command_order.json
deleted file mode 100755
index f58c59c..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/role_command_order.json
+++ /dev/null
@@ -1,9 +0,0 @@
-{
- "_comment" : "Record format:",
- "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]",
- "general_deps" : {
- "_comment" : "dependencies for all cases",
- "INDEXING_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START"],
- "INDEXING_SERVICE_CHECK-SERVICE_CHECK" : ["INDEXING_MASTER-START"]
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/KIBANA/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/KIBANA/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/KIBANA/metainfo.xml
deleted file mode 100755
index 494f71b..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/KIBANA/metainfo.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-<?xml version="1.0"?>
-<!--
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-<metainfo>
- <schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>KIBANA</name>
- <extends>common-services/KIBANA/4.5.1</extends>
- </service>
- </services>
-</metainfo>
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index 7f78806..6b8f195 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -220,6 +220,7 @@ This package installs the Metron Enrichment files
%dir %{metron_home}/config
%dir %{metron_home}/config/zookeeper
%dir %{metron_home}/config/zookeeper/enrichments
+%dir %{metron_home}/ddl
%dir %{metron_home}/flux
%dir %{metron_home}/flux/enrichment
%{metron_home}/bin/latency_summarizer.sh
@@ -229,6 +230,7 @@ This package installs the Metron Enrichment files
%{metron_home}/config/zookeeper/enrichments/snort.json
%{metron_home}/config/zookeeper/enrichments/websphere.json
%{metron_home}/config/zookeeper/enrichments/yaf.json
+%{metron_home}/ddl/geoip_ddl.sql
%{metron_home}/flux/enrichment/remote.yaml
%exclude %{metron_home}/flux/enrichment/test.yaml
%attr(0644,root,root) %{metron_home}/lib/metron-enrichment-%{full_version}-uber.jar
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/docker/rpm-docker/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/rpm-docker/pom.xml b/metron-deployment/packaging/docker/rpm-docker/pom.xml
new file mode 100644
index 0000000..92414c2
--- /dev/null
+++ b/metron-deployment/packaging/docker/rpm-docker/pom.xml
@@ -0,0 +1,218 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>metron-rpm</artifactId>
+ <packaging>pom</packaging>
+ <name>metron-rpm</name>
+ <version>0.2.0BETA</version>
+ <parent>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-deployment</artifactId>
+ <version>0.2.0BETA</version>
+ <relativePath>../../..</relativePath>
+ </parent>
+ <description>RPM Builder for Apache Metron</description>
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+ <rpm.docker.tag>rpm-docker</rpm.docker.tag>
+ <metron_dir>${parent.relativePath}/..</metron_dir>
+ </properties>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.1</version>
+ <executions>
+ <execution>
+ <phase>validate</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <tasks>
+ <echo>******** Displaying value of property ********</echo>
+ <echo>${metron_dir}</echo>
+ </tasks>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-clean-plugin</artifactId>
+ <version>3.0.0</version>
+ <configuration>
+ <filesets>
+ <fileset>
+ <directory>BUILD</directory>
+ <followSymlinks>false</followSymlinks>
+ </fileset>
+ <fileset>
+ <directory>BUILDROOT</directory>
+ <followSymlinks>false</followSymlinks>
+ </fileset>
+ <fileset>
+ <directory>RPMS</directory>
+ <followSymlinks>false</followSymlinks>
+ </fileset>
+ <fileset>
+ <directory>RPMS/noarch</directory>
+ <followSymlinks>false</followSymlinks>
+ </fileset>
+ <fileset>
+ <directory>SOURCES</directory>
+ <followSymlinks>false</followSymlinks>
+ </fileset>
+ <fileset>
+ <directory>SRPMS</directory>
+ <followSymlinks>false</followSymlinks>
+ </fileset>
+ </filesets>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>1.5.0</version>
+ <executions>
+ <execution>
+ <id>docker-build</id>
+ <phase>package</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <executable>docker</executable>
+ <arguments>
+ <argument>build</argument>
+ <argument>-f</argument>
+ <argument>Dockerfile</argument>
+ <argument>-t</argument>
+ <argument>${rpm.docker.tag}</argument>
+ <argument>.</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ <execution>
+ <id>rpm-build</id>
+ <phase>package</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <executable>docker</executable>
+ <arguments>
+ <argument>run</argument>
+ <argument>-v</argument>
+ <argument>${project.basedir}:/root</argument>
+ <argument>${rpm.docker.tag}:latest</argument>
+ <argument>/bin/bash</argument>
+ <argument>-c</argument>
+ <argument>./build.sh ${project.version}</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>3.0.1</version>
+ <executions>
+ <execution>
+ <id>copy-rpm-sources</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.basedir}/SOURCES</outputDirectory>
+ <resources>
+ <resource>
+ <directory>${metron_dir}/metron-platform/metron-common/target/</directory>
+ <includes>
+ <include>*.tar.gz</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>${metron_dir}/metron-platform/metron-data-management/target/</directory>
+ <includes>
+ <include>*.tar.gz</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>${metron_dir}/metron-platform/metron-elasticsearch/target/</directory>
+ <includes>
+ <include>*.tar.gz</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>${metron_dir}/metron-platform/metron-enrichment/target/</directory>
+ <includes>
+ <include>*.tar.gz</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>${metron_dir}/metron-platform/metron-indexing/target/</directory>
+ <includes>
+ <include>*.tar.gz</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>${metron_dir}/metron-platform/metron-parsers/target/</directory>
+ <includes>
+ <include>*.tar.gz</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>${metron_dir}/metron-platform/metron-pcap-backend/target/</directory>
+ <includes>
+ <include>*.tar.gz</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>${metron_dir}/metron-platform/metron-solr/target/</directory>
+ <includes>
+ <include>*.tar.gz</include>
+ </includes>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ <execution>
+ <id>copy-rpms-target</id>
+ <phase>package</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.basedir}/target/RPMS</outputDirectory>
+ <resources>
+ <resource>
+ <directory>RPMS</directory>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/pom.xml b/metron-deployment/pom.xml
index 24b8498..d4b05f1 100644
--- a/metron-deployment/pom.xml
+++ b/metron-deployment/pom.xml
@@ -29,166 +29,21 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<rpm.docker.tag>rpm-docker</rpm.docker.tag>
</properties>
-
- <build>
- <plugins>
- <plugin>
- <artifactId>maven-clean-plugin</artifactId>
- <version>3.0.0</version>
- <configuration>
- <filesets>
- <fileset>
- <directory>packaging/docker/rpm-docker/BUILD</directory>
- <followSymlinks>false</followSymlinks>
- </fileset>
- <fileset>
- <directory>packaging/docker/rpm-docker/BUILDROOT</directory>
- <followSymlinks>false</followSymlinks>
- </fileset>
- <fileset>
- <directory>packaging/docker/rpm-docker/RPMS</directory>
- <followSymlinks>false</followSymlinks>
- </fileset>
- <fileset>
- <directory>packaging/docker/rpm-docker/RPMS/noarch</directory>
- <followSymlinks>false</followSymlinks>
- </fileset>
- <fileset>
- <directory>packaging/docker/rpm-docker/SOURCES</directory>
- <followSymlinks>false</followSymlinks>
- </fileset>
- <fileset>
- <directory>packaging/docker/rpm-docker/SRPMS</directory>
- <followSymlinks>false</followSymlinks>
- </fileset>
- </filesets>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>exec-maven-plugin</artifactId>
- <version>1.5.0</version>
- <executions>
- <execution>
- <id>docker-build</id>
- <phase>package</phase>
- <goals>
- <goal>exec</goal>
- </goals>
- <configuration>
- <executable>docker</executable>
- <arguments>
- <argument>build</argument>
- <argument>-f</argument>
- <argument>packaging/docker/rpm-docker/Dockerfile</argument>
- <argument>-t</argument>
- <argument>${rpm.docker.tag}</argument>
- <argument>.</argument>
- </arguments>
- </configuration>
- </execution>
- <execution>
- <id>rpm-build</id>
- <phase>package</phase>
- <goals>
- <goal>exec</goal>
- </goals>
- <configuration>
- <executable>docker</executable>
- <arguments>
- <argument>run</argument>
- <argument>-v</argument>
- <argument>${project.basedir}/packaging/docker/rpm-docker:/root</argument>
- <argument>${rpm.docker.tag}:latest</argument>
- <argument>/bin/bash</argument>
- <argument>-c</argument>
- <argument>./build.sh ${project.version}</argument>
- </arguments>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <artifactId>maven-resources-plugin</artifactId>
- <version>3.0.1</version>
- <executions>
- <execution>
- <id>copy-rpm-sources</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>copy-resources</goal>
- </goals>
- <configuration>
- <outputDirectory>${project.basedir}/packaging/docker/rpm-docker/SOURCES</outputDirectory>
- <resources>
- <resource>
- <directory>${project.basedir}/../metron-platform/metron-common/target/</directory>
- <includes>
- <include>*.tar.gz</include>
- </includes>
- </resource>
- <resource>
- <directory>${project.basedir}/../metron-platform/metron-data-management/target/</directory>
- <includes>
- <include>*.tar.gz</include>
- </includes>
- </resource>
- <resource>
- <directory>${project.basedir}/../metron-platform/metron-elasticsearch/target/</directory>
- <includes>
- <include>*.tar.gz</include>
- </includes>
- </resource>
- <resource>
- <directory>${project.basedir}/../metron-platform/metron-enrichment/target/</directory>
- <includes>
- <include>*.tar.gz</include>
- </includes>
- </resource>
- <resource>
- <directory>${project.basedir}/../metron-platform/metron-indexing/target/</directory>
- <includes>
- <include>*.tar.gz</include>
- </includes>
- </resource>
- <resource>
- <directory>${project.basedir}/../metron-platform/metron-parsers/target/</directory>
- <includes>
- <include>*.tar.gz</include>
- </includes>
- </resource>
- <resource>
- <directory>${project.basedir}/../metron-platform/metron-pcap-backend/target/</directory>
- <includes>
- <include>*.tar.gz</include>
- </includes>
- </resource>
- <resource>
- <directory>${project.basedir}/../metron-platform/metron-solr/target/</directory>
- <includes>
- <include>*.tar.gz</include>
- </includes>
- </resource>
- </resources>
- </configuration>
- </execution>
- <execution>
- <id>copy-rpms-target</id>
- <phase>package</phase>
- <goals>
- <goal>copy-resources</goal>
- </goals>
- <configuration>
- <outputDirectory>${project.basedir}/target/RPMS</outputDirectory>
- <resources>
- <resource>
- <directory>packaging/docker/rpm-docker/RPMS</directory>
- </resource>
- </resources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
+ <profiles>
+ <profile>
+ <id>build-rpms</id>
+ <modules>
+ <module>packaging/docker/rpm-docker</module>
+ </modules>
+ </profile>
+ <profile>
+ <id>default</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
+ <modules>
+ <module>packaging/ambari/metron-mpack</module>
+ </modules>
+ </profile>
+ </profiles>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/roles/ambari_common/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_common/defaults/main.yml b/metron-deployment/roles/ambari_common/defaults/main.yml
index 8612cfa..eda4a06 100644
--- a/metron-deployment/roles/ambari_common/defaults/main.yml
+++ b/metron-deployment/roles/ambari_common/defaults/main.yml
@@ -18,4 +18,4 @@
hadoop_logrotate_frequency: daily
hadoop_logrotate_retention: 30
-rhel_ambari_install_url: "http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.2.2.0/ambari.repo"
\ No newline at end of file
+rhel_ambari_install_url: "http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.4.0.1/ambari.repo"
\ No newline at end of file