You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by le...@apache.org on 2016/11/11 21:31:35 UTC
[1/4] incubator-metron git commit: METRON-533 Up the version for
release closes apache/incubator-metron#353
Repository: incubator-metron
Updated Branches:
refs/heads/master 66df134ce -> d7784d437
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/parser_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/parser_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/parser_commands.py
new file mode 100755
index 0000000..d43333c
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/parser_commands.py
@@ -0,0 +1,208 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+import re
+import subprocess
+import time
+
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute, File
+
+import metron_service
+
+
+# Wrap major operations and functionality in this class
+class ParserCommands:
+ __params = None
+ __parser_list = None
+ __configured = False
+
+ def __init__(self, params):
+ if params is None:
+ raise ValueError("params argument is required for initialization")
+ self.__params = params
+ self.__parser_list = self.__get_parsers(params)
+ self.__configured = os.path.isfile(self.__params.parsers_configured_flag_file)
+
+ # get list of parsers
+ def __get_parsers(self, params):
+ return params.parsers.replace(' ', '').split(',')
+
+ def is_configured(self):
+ return self.__configured
+
+ def set_configured(self):
+ File(self.__params.parsers_configured_flag_file,
+ content="",
+ owner=self.__params.metron_user,
+ mode=0775)
+
+ def init_parsers(self):
+ Logger.info(
+ "Copying grok patterns from local directory '{0}' to HDFS '{1}'".format(self.__params.local_grok_patterns_dir,
+ self.__params.hdfs_grok_patterns_dir))
+ self.__params.HdfsResource(self.__params.hdfs_grok_patterns_dir,
+ type="directory",
+ action="create_on_execute",
+ owner=self.__params.metron_user,
+ mode=0775,
+ source=self.__params.local_grok_patterns_dir)
+
+ Logger.info("Done initializing parser configuration")
+
+ def get_parser_list(self):
+ return self.__parser_list
+
+ def setup_repo(self):
+ def local_repo():
+ Logger.info("Setting up local repo")
+ Execute("yum -y install createrepo")
+ Execute("createrepo /localrepo")
+ Execute("chmod -R o-w+r /localrepo")
+ Execute("echo \"[METRON-0.3.0]\n"
+ "name=Metron 0.3.0 packages\n"
+ "baseurl=file:///localrepo\n"
+ "gpgcheck=0\n"
+ "enabled=1\" > /etc/yum.repos.d/local.repo")
+
+ def remote_repo():
+ print('Using remote repo')
+
+ yum_repo_types = {
+ 'local': local_repo,
+ 'remote': remote_repo
+ }
+ repo_type = self.__params.yum_repo_type
+ if repo_type in yum_repo_types:
+ yum_repo_types[repo_type]()
+ else:
+ raise ValueError("Unsupported repo type '{0}'".format(repo_type))
+
+ def init_kafka_topics(self):
+ Logger.info('Creating Kafka topics')
+ command_template = """{0}/kafka-topics.sh \
+ --zookeeper {1} \
+ --create \
+ --topic {2} \
+ --partitions {3} \
+ --replication-factor {4} \
+ --config retention.bytes={5}"""
+ num_partitions = 1
+ replication_factor = 1
+ retention_gigabytes = int(self.__params.metron_topic_retention)
+ retention_bytes = retention_gigabytes * 1024 * 1024 * 1024
+ Logger.info("Creating main topics for parsers")
+ for parser_name in self.get_parser_list():
+ Logger.info("Creating topic'{0}'".format(parser_name))
+ Execute(command_template.format(self.__params.kafka_bin_dir,
+ self.__params.zookeeper_quorum,
+ parser_name,
+ num_partitions,
+ replication_factor,
+ retention_bytes))
+ Logger.info("Creating topics for error handling")
+ Execute(command_template.format(self.__params.kafka_bin_dir,
+ self.__params.zookeeper_quorum,
+ "parser_invalid",
+ num_partitions,
+ replication_factor,
+ retention_bytes))
+ Execute(command_template.format(self.__params.kafka_bin_dir,
+ self.__params.zookeeper_quorum,
+ "parser_error",
+ num_partitions, replication_factor,
+ retention_bytes))
+ Logger.info("Done creating Kafka topics")
+
+ def start_parser_topologies(self):
+ Logger.info("Starting Metron parser topologies: {0}".format(self.get_parser_list()))
+ start_cmd_template = """{0}/bin/start_parser_topology.sh \
+ -k {1} \
+ -z {2} \
+ -s {3}"""
+ for parser in self.get_parser_list():
+ Logger.info('Starting ' + parser)
+ Execute(start_cmd_template.format(self.__params.metron_home, self.__params.kafka_brokers,
+ self.__params.zookeeper_quorum, parser))
+
+ Logger.info('Finished starting parser topologies')
+
+ def stop_parser_topologies(self):
+ Logger.info('Stopping parsers')
+ for parser in self.get_parser_list():
+ Logger.info('Stopping ' + parser)
+ stop_cmd = 'storm kill ' + parser
+ Execute(stop_cmd)
+ Logger.info('Done stopping parser topologies')
+
+ def restart_parser_topologies(self, env):
+ Logger.info('Restarting the parser topologies')
+ self.stop_parser_topologies()
+ attempt_count = 0
+ while self.topologies_running(env):
+ if attempt_count > 2:
+ raise Exception("Unable to kill topologies")
+ attempt_count += 1
+ time.sleep(10)
+ self.start_parser_topologies()
+ Logger.info('Done restarting the parser topologies')
+
+ def topologies_exist(self):
+ cmd_open = subprocess.Popen(["storm", "list"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ (stdout, stderr) = cmd_open.communicate()
+ stdout_lines = stdout.splitlines()
+ if stdout_lines:
+ status_lines = self.__get_status_lines(stdout_lines)
+ for parser in self.get_parser_list():
+ for line in status_lines:
+ items = re.sub('[\s]+', ' ', line).split()
+ if items and items[0] == parser:
+ return True
+ return False
+
+ def topologies_running(self, env):
+ env.set_params(self.__params)
+ all_running = True
+ topologies = metron_service.get_running_topologies()
+ for parser in self.get_parser_list():
+ parser_found = False
+ is_running = False
+ if parser in topologies:
+ parser_found = True
+ is_running = topologies[parser] in ['ACTIVE', 'REBALANCING']
+ all_running &= parser_found and is_running
+ return all_running
+
+ def __get_status_lines(self, lines):
+ status_lines = []
+ do_stat = False
+ skipped = 0
+ for line in lines:
+ if line.startswith("Topology_name"):
+ do_stat = True
+ if do_stat and skipped == 2:
+ status_lines += [line]
+ elif do_stat:
+ skipped += 1
+ return status_lines
+
+ def __is_running(self, status):
+ return status in ['ACTIVE', 'REBALANCING']
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/parser_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/parser_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/parser_master.py
new file mode 100755
index 0000000..e6d2ac6
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/parser_master.py
@@ -0,0 +1,88 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.core.logger import Logger
+from resource_management.libraries.script import Script
+
+import metron_service
+from parser_commands import ParserCommands
+
+
+class ParserMaster(Script):
+ def get_component_name(self):
+ # TODO add this at some point - currently will cause problems with hdp-select
+ # return "parser-master"
+ pass
+
+ def install(self, env):
+ from params import params
+ env.set_params(params)
+ commands = ParserCommands(params)
+ commands.setup_repo()
+ self.install_packages(env)
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ from params import params
+ env.set_params(params)
+ metron_service.load_global_config(params)
+ commands = ParserCommands(params)
+ if not commands.is_configured():
+ commands.init_parsers()
+ commands.init_kafka_topics()
+ commands.set_configured()
+
+ def start(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ self.configure(env)
+ commands = ParserCommands(params)
+ commands.start_parser_topologies()
+
+ def stop(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ commands = ParserCommands(params)
+ commands.stop_parser_topologies()
+
+ def status(self, env):
+ from params import status_params
+ env.set_params(status_params)
+ commands = ParserCommands(status_params)
+ if not commands.topologies_running(env):
+ raise ComponentIsNotRunning()
+
+ def restart(self, env):
+ from params import params
+ env.set_params(params)
+ self.configure(env)
+ commands = ParserCommands(params)
+ commands.restart_parser_topologies(env)
+
+ def servicechecktest(self, env):
+ from params import params
+ env.set_params(params)
+ from service_check import ServiceCheck
+ service_check = ServiceCheck()
+ Logger.info('Service Check Test')
+ service_check.service_check(env)
+
+
+if __name__ == "__main__":
+ ParserMaster().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/service_check.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/service_check.py
new file mode 100755
index 0000000..7dd9dfb
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/service_check.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from __future__ import print_function
+
+from resource_management.libraries.script import Script
+
+from indexing_commands import IndexingCommands
+from parser_commands import ParserCommands
+
+
+class ServiceCheck(Script):
+ def service_check(self, env):
+ from params import params
+ parsercommands = ParserCommands(params)
+ indexingcommands = IndexingCommands(params)
+ all_found = parsercommands.topologies_running(env) and indexingcommands.is_topology_active(env)
+ if all_found:
+ exit(0)
+ else:
+ exit(1)
+
+
+if __name__ == "__main__":
+ ServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/templates/enrichment.properties.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/templates/enrichment.properties.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/templates/enrichment.properties.j2
new file mode 100755
index 0000000..bab2f52
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/templates/enrichment.properties.j2
@@ -0,0 +1,88 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+##### Kafka #####
+
+kafka.zk={{zookeeper_quorum}}
+kafka.broker={{kafka_brokers}}
+enrichment.output.topic=indexing
+
+##### MySQL #####
+
+mysql.ip={{mysql_host}}
+mysql.port={{mysql_port}}
+mysql.username={{enrichment_metron_user}}
+mysql.password={{enrichment_metron_user_passwd}}
+
+##### Metrics #####
+
+#reporters
+org.apache.metron.metrics.reporter.graphite=true
+org.apache.metron.metrics.reporter.console=false
+org.apache.metron.metrics.reporter.jmx=false
+
+#Graphite Addresses
+
+org.apache.metron.metrics.graphite.address=localhost
+org.apache.metron.metrics.graphite.port=2023
+
+#TelemetryParserBolt
+org.apache.metron.metrics.TelemetryParserBolt.acks=true
+org.apache.metron.metrics.TelemetryParserBolt.emits=true
+org.apache.metron.metrics.TelemetryParserBolt.fails=true
+
+
+#GenericEnrichmentBolt
+org.apache.metron.metrics.GenericEnrichmentBolt.acks=true
+org.apache.metron.metrics.GenericEnrichmentBolt.emits=true
+org.apache.metron.metrics.GenericEnrichmentBolt.fails=true
+
+
+#TelemetryIndexingBolt
+org.apache.metron.metrics.TelemetryIndexingBolt.acks=true
+org.apache.metron.metrics.TelemetryIndexingBolt.emits=true
+org.apache.metron.metrics.TelemetryIndexingBolt.fails=true
+
+##### Host Enrichment #####
+
+hbase.provider.impl=org.apache.metron.hbase.HTableProvider
+enrichment.simple.hbase.table={{enrichment_table}}
+enrichment.simple.hbase.cf={{enrichment_cf}}
+org.apache.metron.enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\
+{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},\
+{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}]
+
+
+##### HBase #####
+bolt.hbase.table.name=pcap
+bolt.hbase.table.fields=t:value
+bolt.hbase.table.key.tuple.field.name=key
+bolt.hbase.table.timestamp.tuple.field.name=timestamp
+bolt.hbase.enable.batching=false
+bolt.hbase.write.buffer.size.in.bytes=2000000
+bolt.hbase.durability=SKIP_WAL
+bolt.hbase.partitioner.region.info.refresh.interval.mins=60
+
+##### Threat Intel #####
+
+threat.intel.tracker.table={{threatintel_table}}
+threat.intel.tracker.cf={{threatintel_cf}}
+threat.intel.simple.hbase.table={{threatintel_table}}
+threat.intel.simple.hbase.cf={{threatintel_cf}}
+threat.intel.ip.table=
+threat.intel.ip.cf=
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/quicklinks/quicklinks.json
new file mode 100755
index 0000000..ee1b225
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/quicklinks/quicklinks.json
@@ -0,0 +1,28 @@
+{
+ "name": "default",
+ "description": "default quick links configuration",
+ "configuration": {
+ "protocol":
+ {
+ "type":"HTTP_ONLY"
+ },
+
+ "links": [
+ {
+ "name": "storm_ui",
+ "label": "Storm UI",
+ "requires_user_name": "false",
+ "component_name": "STORM_UI_SERVER",
+ "url":"%@://%@:%@/",
+ "port":{
+ "http_property": "ui.port",
+ "http_default_port": "8744",
+ "https_property": "ui.port",
+ "https_default_port": "8744",
+ "regex": "^(\\d+)$",
+ "site": "storm-site"
+ }
+ }
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/service_advisor.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/service_advisor.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/service_advisor.py
new file mode 100644
index 0000000..6700278
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/service_advisor.py
@@ -0,0 +1,141 @@
+#!/usr/bin/env ambari-python-wrap
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import os
+import fnmatch
+import imp
+import socket
+import sys
+import traceback
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../../../stacks/')
+PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py')
+
+try:
+ with open(PARENT_FILE, 'rb') as fp:
+ service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE))
+except Exception as e:
+ traceback.print_exc()
+ print "Failed to load parent"
+
+class METRON030ServiceAdvisor(service_advisor.ServiceAdvisor):
+
+ def getServiceComponentLayoutValidations(self, services, hosts):
+
+ componentsListList = [service["components"] for service in services["services"]]
+ componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist]
+
+ metronParsersHost = self.getHosts(componentsList, "METRON_PARSERS")[0]
+ metronEnrichmentMaster = self.getHosts(componentsList, "METRON_ENRICHMENT_MASTER")[0]
+ metronIndexingHost = self.getHosts(componentsList, "METRON_INDEXING")[0]
+ metronEnrichmentMysqlServer = self.getHosts(componentsList, "METRON_ENRICHMENT_MYSQL_SERVER")[0]
+
+ kafkaBrokers = self.getHosts(componentsList, "KAFKA_BROKER")
+ stormSupervisors = self.getHosts(componentsList,"SUPERVISOR")
+
+ items = []
+
+ #Metron Must Co-locate with KAFKA_BROKER and STORM_SUPERVISOR
+ if metronParsersHost not in kafkaBrokers:
+ message = "Metron must be colocated with an instance of KAFKA BROKER"
+ items.append({ "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'METRON_PARSERS', "host": metronParsersHost })
+
+ if metronParsersHost not in stormSupervisors:
+ message = "Metron must be colocated with an instance of STORM SUPERVISOR"
+ items.append({ "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'METRON_PARSERS', "host": metronParsersHost })
+
+ if metronParsersHost != metronEnrichmentMaster:
+ message = "Metron Enrichment Master must be co-located with Metron Parsers on {0}".format(metronParsersHost)
+ items.append({ "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'METRON_ENRICHMENT_MASTER', "host": metronEnrichmentMaster })
+
+ if metronParsersHost != metronIndexingHost:
+ message = "Metron Indexing must be co-located with Metron Parsers on {0}".format(metronParsersHost)
+ items.append({ "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'METRON_INDEXING', "host": metronIndexingHost })
+
+ if metronParsersHost != metronEnrichmentMysqlServer:
+ message = "Metron MySQL Server must be co-located with Metron Parsers on {0}".format(metronParsersHost)
+ items.append({ "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'METRON_ENRICHMENT_MYSQL_SERVER', "host": metronEnrichmentMysqlServer })
+
+ return items
+
+ def getServiceConfigurationsValidationItems(self, configurations, recommendedDefaults, services, hosts):
+
+ # validate recommended properties in storm-site
+ siteName = "storm-site"
+ method = self.validateSTORMSiteConfigurations
+ items = self.validateConfigurationsForSite(configurations, recommendedDefaults, services, hosts, siteName, method)
+
+ return items
+
+ def getServiceConfigurationRecommendations(self, configurations, clusterData, services, hosts):
+ #Suggest mysql server hostname
+ mySQLServerHost = self.getComponentHostNames(services, "METRON", "METRON_ENRICHMENT_MYSQL_SERVER")[0]
+ putMetronEnvProperty = self.putProperty(configurations, "metron-env", services)
+ putMetronEnvProperty("mysql_host",mySQLServerHost)
+
+ #Suggest Storm Rest URL
+ if "storm-site" in services["configurations"]:
+ stormUIServerHost = self.getComponentHostNames(services, "STORM", "STORM_UI_SERVER")[0]
+ stormUIServerPort = services["configurations"]["storm-site"]["properties"]["ui.port"]
+ stormUIServerURL = stormUIServerHost + ":" + stormUIServerPort
+ putMetronEnvProperty = self.putProperty(configurations, "metron-env", services)
+ putMetronEnvProperty("storm_rest_addr",stormUIServerURL)
+
+ storm_site = services["configurations"]["storm-site"]["properties"]
+ putStormSiteProperty = self.putProperty(configurations, "storm-site", services)
+
+ for property, desired_value in self.getSTORMSiteDesiredValues().iteritems():
+ if property not in storm_site:
+ putStormSiteProperty(property, desired_value)
+ elif property == "topology.classpath" and storm_site[property] != desired_value:
+ topololgyClasspath = storm_site[property]
+ #check that desired values exist in topology.classpath. append them if they do not
+ for path in desired_value.split(':'):
+ if path not in topololgyClasspath:
+ topololgyClasspath += ":" + path
+ putStormSiteProperty(property,topololgyClasspath)
+
+ def validateSTORMSiteConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
+
+ storm_site = properties
+ validationItems = []
+
+ for property, desired_value in self.getSTORMSiteDesiredValues().iteritems():
+ if property not in storm_site :
+ message = "Metron requires this property to be set to the recommended value of " + desired_value
+ item = self.getErrorItem(message) if property == "topology.classpath" else self.getWarnItem(message)
+ validationItems.append({"config-name": property, "item": item})
+ elif storm_site[property] != desired_value:
+ topologyClasspath = storm_site[property]
+ for path in desired_value.split(':'):
+ if path not in topologyClasspath:
+ message = "Metron requires this property to contain " + desired_value
+ item = self.getErrorItem(message)
+ validationItems.append({"config-name": property, "item": item})
+
+ return self.toConfigurationValidationProblems(validationItems, "storm-site")
+
+ def getSTORMSiteDesiredValues(self):
+
+ storm_site_desired_values = {
+ "topology.classpath" : "/etc/hbase/conf:/etc/hadoop/conf"
+ }
+
+ return storm_site_desired_values
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/themes/metron_theme.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/themes/metron_theme.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/themes/metron_theme.json
new file mode 100644
index 0000000..b939182
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/themes/metron_theme.json
@@ -0,0 +1,219 @@
+{
+ "configuration": {
+ "layouts": [
+ {
+ "name": "default",
+ "tabs": [
+ {
+ "name": "metron_general",
+ "display-name": "Settings",
+ "layout": {
+ "tab-columns": "1",
+ "tab-rows": "1",
+ "sections": [
+ {
+ "name": "section-general",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1",
+ "section-columns": "2",
+ "section-rows": "1",
+ "subsections": [
+ {
+ "name": "subsection-general-database",
+ "display-name" : "Database Settings",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1"
+ },
+ {
+ "name": "subsection-general-indexing",
+ "display-name": "Index Settings",
+ "row-index": "0",
+ "column-index": "1",
+ "row-span": "1",
+ "column-span": "1"
+ }
+ ]
+ }
+ ]
+ }
+ },
+ {
+ "name": "metron_repo",
+ "display-name" : "Repository",
+ "layout": {
+ "tab-columns" : "1",
+ "tab-rows" : "1",
+ "sections" : [
+ {
+ "name" : "section-repo",
+ "row-index": "0",
+ "column-index" : "0",
+ "row-span": "1",
+ "column-span": "1",
+ "section-columns": "1",
+ "section-rows": "1",
+ "subsections": [
+ {
+ "name": "subsection-repo",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1"
+ }
+ ]
+ }
+ ]
+ }
+ }
+ ]
+ }
+ ],
+ "placement": {
+ "configuration-layout": "default",
+ "configs": [
+ {
+ "config": "metron-env/install_mysql",
+ "subsection-name": "subsection-general-database"
+ },
+ {
+ "config": "metron-env/mysql_admin_password",
+ "subsection-name": "subsection-general-database",
+ "depends-on": [
+ {
+ "configs":[
+ "metron-env/install_mysql"
+ ],
+ "if": "${metron-env/install_mysql} === No",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "metron-env/metron_enrichment_db_password",
+ "subsection-name": "subsection-general-database"
+ },
+ {
+ "config": "metron-env/metron_enrichment_db_port",
+ "subsection-name": "subsection-general-database"
+ },
+ {
+ "config": "metron-env/es_hosts",
+ "subsection-name": "subsection-general-indexing"
+ },
+ {
+ "config": "metron-env/es_binary_port",
+ "subsection-name": "subsection-general-indexing"
+ },
+ {
+ "config": "metron-env/es_http_port",
+ "subsection-name": "subsection-general-indexing"
+ },
+ {
+ "config": "metron-env/es_cluster_name",
+ "subsection-name": "subsection-general-indexing"
+ },
+ {
+ "config": "metron-env/repo_type",
+ "subsection-name": "subsection-repo"
+ },
+ {
+ "config": "metron-env/repo_url",
+ "subsection-name": "subsection-repo",
+ "depends-on": [
+ {
+ "configs":[
+ "metron-env/repo_type"
+ ],
+ "if": "${metron-env/repo_type} === remote",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "widgets": [
+ {
+ "config": "metron-env/install_mysql",
+ "widget": {
+ "type": "toggle"
+ }
+ },
+ {
+ "config": "metron-env/mysql_admin_password",
+ "widget": {
+ "type": "password"
+ }
+ },
+ {
+ "config": "metron-env/metron_enrichment_db_password",
+ "widget": {
+ "type": "password"
+ }
+ },
+ {
+ "config": "metron-env/metron_enrichment_db_port",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "metron-env/es_hosts",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "metron-env/es_binary_port",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "metron-env/es_http_port",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "metron-env/es_cluster_name",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "metron-env/repo_type",
+ "widget": {
+ "type": "toggle"
+ }
+ },
+ {
+ "config": "metron-env/repo_url",
+ "widget": {
+ "type": "text-field"
+ }
+ }
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json
index 303e84f..0ffdd49 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json
@@ -62,7 +62,7 @@
},
{
"service_name" : "METRON",
- "service_version" : "0.2.1BETA",
+ "service_version" : "0.3.0",
"applicable_stacks" : [
{
"stack_name" : "HDP",
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/docker/rpm-docker/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/rpm-docker/pom.xml b/metron-deployment/packaging/docker/rpm-docker/pom.xml
index 30e2670..2b5cdc1 100644
--- a/metron-deployment/packaging/docker/rpm-docker/pom.xml
+++ b/metron-deployment/packaging/docker/rpm-docker/pom.xml
@@ -21,11 +21,11 @@
<artifactId>metron-rpm</artifactId>
<packaging>pom</packaging>
<name>metron-rpm</name>
- <version>0.2.1BETA</version>
+ <version>0.3.0</version>
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-deployment</artifactId>
- <version>0.2.1BETA</version>
+ <version>0.3.0</version>
<relativePath>../../..</relativePath>
</parent>
<description>RPM Builder for Apache Metron</description>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/playbooks/docker_probe_install.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/playbooks/docker_probe_install.yml b/metron-deployment/playbooks/docker_probe_install.yml
index cc4511e..d1bd6b1 100644
--- a/metron-deployment/playbooks/docker_probe_install.yml
+++ b/metron-deployment/playbooks/docker_probe_install.yml
@@ -30,7 +30,7 @@
- hosts: sensors
vars:
- metron_version: 0.2.1BETA
+ metron_version: 0.3.0
metron_directory: /usr/metron/{{ metron_version }}
bro_version: "2.4.1"
fixbuf_version: "1.7.1"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/pom.xml b/metron-deployment/pom.xml
index 717b1ce..e95ae1e 100644
--- a/metron-deployment/pom.xml
+++ b/metron-deployment/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>Metron</artifactId>
- <version>0.2.1BETA</version>
+ <version>0.3.0</version>
</parent>
<description>Building and deploying Metron</description>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/roles/metron_pcapservice/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_pcapservice/defaults/main.yml b/metron-deployment/roles/metron_pcapservice/defaults/main.yml
index 5ab28b1..aec60c9 100644
--- a/metron-deployment/roles/metron_pcapservice/defaults/main.yml
+++ b/metron-deployment/roles/metron_pcapservice/defaults/main.yml
@@ -15,7 +15,7 @@
# limitations under the License.
#
---
-metron_version: 0.2.1BETA
+metron_version: 0.3.0
metron_directory: /usr/metron/{{ metron_version }}
pcapservice_jar_name: metron-api-{{ metron_version }}.jar
pcapservice_jar_src: "{{ playbook_dir }}/../../metron-platform/metron-api/target/{{ pcapservice_jar_name }}"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-platform/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/README.md b/metron-platform/README.md
index 360997c..da6ccb5 100644
--- a/metron-platform/README.md
+++ b/metron-platform/README.md
@@ -17,7 +17,7 @@ limitations under the License.
# Current Build
-The latest build of metron-platform is 0.2.1BETA.
+The latest build of metron-platform is 0.3.0.
We are still in the process of merging/porting additional features from our production code base into this open source release. This release will be followed by a number of additional beta releases until the port is complete. We will also work on getting additional documentation and user/developer guides to the community as soon as we can. At this time we offer no support for the beta software, but will try to respond to requests as promptly as we can.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-platform/elasticsearch-shaded/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/elasticsearch-shaded/pom.xml b/metron-platform/elasticsearch-shaded/pom.xml
index 570a5bb..cfa9c9d 100644
--- a/metron-platform/elasticsearch-shaded/pom.xml
+++ b/metron-platform/elasticsearch-shaded/pom.xml
@@ -17,7 +17,7 @@
<parent>
<artifactId>metron-platform</artifactId>
<groupId>org.apache.metron</groupId>
- <version>0.2.1BETA</version>
+ <version>0.3.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>elasticsearch-shaded</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-platform/metron-api/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-api/pom.xml b/metron-platform/metron-api/pom.xml
index 9260d8e..db9645d 100644
--- a/metron-platform/metron-api/pom.xml
+++ b/metron-platform/metron-api/pom.xml
@@ -17,7 +17,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.2.1BETA</version>
+ <version>0.3.0</version>
</parent>
<artifactId>metron-api</artifactId>
<description>Metron API</description>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-platform/metron-common/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/pom.xml b/metron-platform/metron-common/pom.xml
index 0e33363..c05111f 100644
--- a/metron-platform/metron-common/pom.xml
+++ b/metron-platform/metron-common/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.2.1BETA</version>
+ <version>0.3.0</version>
</parent>
<artifactId>metron-common</artifactId>
<name>metron-common</name>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-platform/metron-data-management/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/pom.xml b/metron-platform/metron-data-management/pom.xml
index 48d37e2..a2e08a6 100644
--- a/metron-platform/metron-data-management/pom.xml
+++ b/metron-platform/metron-data-management/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.2.1BETA</version>
+ <version>0.3.0</version>
</parent>
<artifactId>metron-data-management</artifactId>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-platform/metron-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/pom.xml b/metron-platform/metron-elasticsearch/pom.xml
index efe6c2f..1786725 100644
--- a/metron-platform/metron-elasticsearch/pom.xml
+++ b/metron-platform/metron-elasticsearch/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.2.1BETA</version>
+ <version>0.3.0</version>
</parent>
<artifactId>metron-elasticsearch</artifactId>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-platform/metron-enrichment/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/pom.xml b/metron-platform/metron-enrichment/pom.xml
index 6c8f69c..690fb1d 100644
--- a/metron-platform/metron-enrichment/pom.xml
+++ b/metron-platform/metron-enrichment/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.2.1BETA</version>
+ <version>0.3.0</version>
</parent>
<artifactId>metron-enrichment</artifactId>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-platform/metron-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-hbase/pom.xml b/metron-platform/metron-hbase/pom.xml
index 7d0a587..4d6c8ba 100644
--- a/metron-platform/metron-hbase/pom.xml
+++ b/metron-platform/metron-hbase/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.2.1BETA</version>
+ <version>0.3.0</version>
</parent>
<artifactId>metron-hbase</artifactId>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-platform/metron-indexing/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/pom.xml b/metron-platform/metron-indexing/pom.xml
index 619d755..1e3a4f8 100644
--- a/metron-platform/metron-indexing/pom.xml
+++ b/metron-platform/metron-indexing/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.2.1BETA</version>
+ <version>0.3.0</version>
</parent>
<artifactId>metron-indexing</artifactId>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-platform/metron-integration-test/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/pom.xml b/metron-platform/metron-integration-test/pom.xml
index 8bae273..6632e0b 100644
--- a/metron-platform/metron-integration-test/pom.xml
+++ b/metron-platform/metron-integration-test/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.2.1BETA</version>
+ <version>0.3.0</version>
</parent>
<artifactId>metron-integration-test</artifactId>
<description>Metron Integration Test</description>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-platform/metron-management/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-management/pom.xml b/metron-platform/metron-management/pom.xml
index c0d30b8..8767499 100644
--- a/metron-platform/metron-management/pom.xml
+++ b/metron-platform/metron-management/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.2.1BETA</version>
+ <version>0.3.0</version>
</parent>
<artifactId>metron-management</artifactId>
<name>metron-management</name>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-platform/metron-parsers/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/pom.xml b/metron-platform/metron-parsers/pom.xml
index 23d0307..b6a321d 100644
--- a/metron-platform/metron-parsers/pom.xml
+++ b/metron-platform/metron-parsers/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.2.1BETA</version>
+ <version>0.3.0</version>
</parent>
<artifactId>metron-parsers</artifactId>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-platform/metron-pcap-backend/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap-backend/pom.xml b/metron-platform/metron-pcap-backend/pom.xml
index b2042c4..dfed40c 100644
--- a/metron-platform/metron-pcap-backend/pom.xml
+++ b/metron-platform/metron-pcap-backend/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.2.1BETA</version>
+ <version>0.3.0</version>
</parent>
<artifactId>metron-pcap-backend</artifactId>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-platform/metron-pcap/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap/pom.xml b/metron-platform/metron-pcap/pom.xml
index 3161041..353292f 100644
--- a/metron-platform/metron-pcap/pom.xml
+++ b/metron-platform/metron-pcap/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.2.1BETA</version>
+ <version>0.3.0</version>
</parent>
<artifactId>metron-pcap</artifactId>
<description>Metron Pcap</description>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-platform/metron-solr/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/pom.xml b/metron-platform/metron-solr/pom.xml
index ce9dba9..7431041 100644
--- a/metron-platform/metron-solr/pom.xml
+++ b/metron-platform/metron-solr/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.2.1BETA</version>
+ <version>0.3.0</version>
</parent>
<artifactId>metron-solr</artifactId>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-platform/metron-test-utilities/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-test-utilities/pom.xml b/metron-platform/metron-test-utilities/pom.xml
index 04cfe14..3984265 100644
--- a/metron-platform/metron-test-utilities/pom.xml
+++ b/metron-platform/metron-test-utilities/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.2.1BETA</version>
+ <version>0.3.0</version>
</parent>
<artifactId>metron-test-utilities</artifactId>
<description>Metron Test Utilities</description>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-platform/metron-writer/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/pom.xml b/metron-platform/metron-writer/pom.xml
index c49061f..c8dd742 100644
--- a/metron-platform/metron-writer/pom.xml
+++ b/metron-platform/metron-writer/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.2.1BETA</version>
+ <version>0.3.0</version>
</parent>
<artifactId>metron-writer</artifactId>
<name>metron-writer</name>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-platform/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/pom.xml b/metron-platform/pom.xml
index 551a07a..1d611f4 100644
--- a/metron-platform/pom.xml
+++ b/metron-platform/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>Metron</artifactId>
- <version>0.2.1BETA</version>
+ <version>0.3.0</version>
</parent>
<description>Stream analytics for Metron</description>
<url>https://metron.incubator.apache.org/</url>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ade2582..1db3c41 100644
--- a/pom.xml
+++ b/pom.xml
@@ -19,7 +19,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.metron</groupId>
<artifactId>Metron</artifactId>
- <version>0.2.1BETA</version>
+ <version>0.3.0</version>
<packaging>pom</packaging>
<name>Metron</name>
<description>Metron Top Level Project</description>
[4/4] incubator-metron git commit: METRON-533 Up the version for
release closes apache/incubator-metron#353
Posted by le...@apache.org.
METRON-533 Up the version for release closes apache/incubator-metron#353
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/d7784d43
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/d7784d43
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/d7784d43
Branch: refs/heads/master
Commit: d7784d437121088425555e1100936dc6b13498f3
Parents: 66df134
Author: justinleet <ju...@gmail.com>
Authored: Fri Nov 11 16:31:17 2016 -0500
Committer: justinjleet <ju...@gmail.com>
Committed: Fri Nov 11 16:31:17 2016 -0500
----------------------------------------------------------------------
metron-analytics/metron-maas-common/pom.xml | 2 +-
metron-analytics/metron-maas-service/pom.xml | 2 +-
.../metron-profiler-client/README.md | 2 +-
metron-analytics/metron-profiler-client/pom.xml | 2 +-
metron-analytics/metron-profiler-common/pom.xml | 2 +-
metron-analytics/metron-profiler/README.md | 4 +-
metron-analytics/metron-profiler/pom.xml | 2 +-
metron-analytics/pom.xml | 2 +-
metron-deployment/amazon-ec2/conf/defaults.yml | 2 +-
.../inventory/full-dev-platform/group_vars/all | 2 +-
.../inventory/metron_example/group_vars/all | 2 +-
.../inventory/quick-dev-platform/group_vars/all | 2 +-
.../packaging/ambari/metron-mpack/pom.xml | 2 +-
.../METRON/0.2.1BETA/metainfo.xml | 26 --
.../METRON/0.2.1BETA/role_command_order.json | 14 -
.../addon-services/METRON/0.3.0/metainfo.xml | 26 ++
.../METRON/0.3.0/role_command_order.json | 14 +
.../0.2.1BETA/configuration/metron-env.xml | 256 -----------------
.../METRON/0.2.1BETA/metainfo.xml | 288 -------------------
.../0.2.1BETA/package/files/addMysqlUser.sh | 51 ----
.../0.2.1BETA/package/files/bro_index.template | 217 --------------
.../0.2.1BETA/package/files/createMysqlGeoIp.sh | 53 ----
.../0.2.1BETA/package/files/removeMysqlUser.sh | 43 ---
.../package/files/snort_index.template | 183 ------------
.../0.2.1BETA/package/files/yaf_index.template | 205 -------------
.../package/scripts/enrichment_commands.py | 177 ------------
.../package/scripts/enrichment_master.py | 87 ------
.../package/scripts/indexing_commands.py | 142 ---------
.../package/scripts/indexing_master.py | 119 --------
.../0.2.1BETA/package/scripts/metron_service.py | 76 -----
.../0.2.1BETA/package/scripts/mysql_server.py | 63 ----
.../0.2.1BETA/package/scripts/mysql_service.py | 46 ---
.../0.2.1BETA/package/scripts/mysql_users.py | 58 ----
.../0.2.1BETA/package/scripts/mysql_utils.py | 56 ----
.../package/scripts/params/__init__.py | 18 --
.../0.2.1BETA/package/scripts/params/params.py | 32 ---
.../package/scripts/params/params_linux.py | 186 ------------
.../package/scripts/params/params_windows.py | 20 --
.../package/scripts/params/status_params.py | 57 ----
.../package/scripts/parser_commands.py | 208 --------------
.../0.2.1BETA/package/scripts/parser_master.py | 88 ------
.../0.2.1BETA/package/scripts/service_check.py | 41 ---
.../package/templates/enrichment.properties.j2 | 88 ------
.../METRON/0.2.1BETA/quicklinks/quicklinks.json | 28 --
.../METRON/0.2.1BETA/service_advisor.py | 141 ---------
.../METRON/0.2.1BETA/themes/metron_theme.json | 219 --------------
.../METRON/0.3.0/configuration/metron-env.xml | 256 +++++++++++++++++
.../common-services/METRON/0.3.0/metainfo.xml | 288 +++++++++++++++++++
.../METRON/0.3.0/package/files/addMysqlUser.sh | 51 ++++
.../0.3.0/package/files/bro_index.template | 217 ++++++++++++++
.../0.3.0/package/files/createMysqlGeoIp.sh | 53 ++++
.../0.3.0/package/files/removeMysqlUser.sh | 43 +++
.../0.3.0/package/files/snort_index.template | 183 ++++++++++++
.../0.3.0/package/files/yaf_index.template | 205 +++++++++++++
.../package/scripts/enrichment_commands.py | 177 ++++++++++++
.../0.3.0/package/scripts/enrichment_master.py | 87 ++++++
.../0.3.0/package/scripts/indexing_commands.py | 142 +++++++++
.../0.3.0/package/scripts/indexing_master.py | 119 ++++++++
.../0.3.0/package/scripts/metron_service.py | 76 +++++
.../0.3.0/package/scripts/mysql_server.py | 63 ++++
.../0.3.0/package/scripts/mysql_service.py | 46 +++
.../METRON/0.3.0/package/scripts/mysql_users.py | 58 ++++
.../METRON/0.3.0/package/scripts/mysql_utils.py | 56 ++++
.../0.3.0/package/scripts/params/__init__.py | 18 ++
.../0.3.0/package/scripts/params/params.py | 32 +++
.../package/scripts/params/params_linux.py | 186 ++++++++++++
.../package/scripts/params/params_windows.py | 20 ++
.../package/scripts/params/status_params.py | 57 ++++
.../0.3.0/package/scripts/parser_commands.py | 208 ++++++++++++++
.../0.3.0/package/scripts/parser_master.py | 88 ++++++
.../0.3.0/package/scripts/service_check.py | 41 +++
.../package/templates/enrichment.properties.j2 | 88 ++++++
.../METRON/0.3.0/quicklinks/quicklinks.json | 28 ++
.../METRON/0.3.0/service_advisor.py | 141 +++++++++
.../METRON/0.3.0/themes/metron_theme.json | 219 ++++++++++++++
.../metron-mpack/src/main/resources/mpack.json | 2 +-
.../packaging/docker/rpm-docker/pom.xml | 4 +-
.../playbooks/docker_probe_install.yml | 2 +-
metron-deployment/pom.xml | 2 +-
.../roles/metron_pcapservice/defaults/main.yml | 2 +-
metron-platform/README.md | 2 +-
metron-platform/elasticsearch-shaded/pom.xml | 2 +-
metron-platform/metron-api/pom.xml | 2 +-
metron-platform/metron-common/pom.xml | 2 +-
metron-platform/metron-data-management/pom.xml | 2 +-
metron-platform/metron-elasticsearch/pom.xml | 2 +-
metron-platform/metron-enrichment/pom.xml | 2 +-
metron-platform/metron-hbase/pom.xml | 2 +-
metron-platform/metron-indexing/pom.xml | 2 +-
metron-platform/metron-integration-test/pom.xml | 2 +-
metron-platform/metron-management/pom.xml | 2 +-
metron-platform/metron-parsers/pom.xml | 2 +-
metron-platform/metron-pcap-backend/pom.xml | 2 +-
metron-platform/metron-pcap/pom.xml | 2 +-
metron-platform/metron-solr/pom.xml | 2 +-
metron-platform/metron-test-utilities/pom.xml | 2 +-
metron-platform/metron-writer/pom.xml | 2 +-
metron-platform/pom.xml | 2 +-
pom.xml | 2 +-
99 files changed, 3325 insertions(+), 3325 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-analytics/metron-maas-common/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-maas-common/pom.xml b/metron-analytics/metron-maas-common/pom.xml
index 35b62c0..0bc7f68 100644
--- a/metron-analytics/metron-maas-common/pom.xml
+++ b/metron-analytics/metron-maas-common/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-analytics</artifactId>
- <version>0.2.1BETA</version>
+ <version>0.3.0</version>
</parent>
<artifactId>metron-maas-common</artifactId>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-analytics/metron-maas-service/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-maas-service/pom.xml b/metron-analytics/metron-maas-service/pom.xml
index d2db00b..546e67f 100644
--- a/metron-analytics/metron-maas-service/pom.xml
+++ b/metron-analytics/metron-maas-service/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-analytics</artifactId>
- <version>0.2.1BETA</version>
+ <version>0.3.0</version>
</parent>
<artifactId>metron-maas-service</artifactId>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-analytics/metron-profiler-client/README.md
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-client/README.md b/metron-analytics/metron-profiler-client/README.md
index dc4adb0..8a55739 100644
--- a/metron-analytics/metron-profiler-client/README.md
+++ b/metron-analytics/metron-profiler-client/README.md
@@ -63,7 +63,7 @@ These instructions step through the process of using the Stellar Client API on a
To validate that everything is working, login to the server hosting Metron. We will use the Stellar Shell to replicate the execution environment of Stellar running in a Storm topology, like Metron's Parser or Enrichment topology. Replace 'node1:2181' with the URL to a Zookeeper Broker.
```
-[root@node1 0.2.1BETA]# bin/stellar -z node1:2181
+[root@node1 0.3.0]# bin/stellar -z node1:2181
Stellar, Go!
Please note that functions are loading lazily in the background and will be unavailable until loaded fully.
{es.clustername=metron, es.ip=node1, es.port=9300, es.date.format=yyyy.MM.dd.HH}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-analytics/metron-profiler-client/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-client/pom.xml b/metron-analytics/metron-profiler-client/pom.xml
index 0912284..b8aadc4 100644
--- a/metron-analytics/metron-profiler-client/pom.xml
+++ b/metron-analytics/metron-profiler-client/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-analytics</artifactId>
- <version>0.2.1BETA</version>
+ <version>0.3.0</version>
</parent>
<artifactId>metron-profiler-client</artifactId>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-analytics/metron-profiler-common/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-common/pom.xml b/metron-analytics/metron-profiler-common/pom.xml
index c7920ef..c3560b4 100644
--- a/metron-analytics/metron-profiler-common/pom.xml
+++ b/metron-analytics/metron-profiler-common/pom.xml
@@ -18,7 +18,7 @@
<parent>
<artifactId>metron-analytics</artifactId>
<groupId>org.apache.metron</groupId>
- <version>0.2.1BETA</version>
+ <version>0.3.0</version>
</parent>
<artifactId>metron-profiler-common</artifactId>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-analytics/metron-profiler/README.md
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/README.md b/metron-analytics/metron-profiler/README.md
index eb4e068..3a52133 100644
--- a/metron-analytics/metron-profiler/README.md
+++ b/metron-analytics/metron-profiler/README.md
@@ -41,7 +41,7 @@ This section will describe the steps required to get your first profile running.
1. Upload the profile definition to Zookeeper.
```
- $ cd /usr/metron/0.2.1BETA/
+ $ cd /usr/metron/0.3.0/
$ bin/zk_load_configs.sh -m PUSH -i config/zookeeper/ -z node1:2181
```
@@ -171,7 +171,7 @@ The Profiler runs as an independent Storm topology. The configuration for the P
After altering the configuration, start the Profiler.
```
-$ /usr/metron/0.2.1BETA/start_profiler_topology.sh
+$ /usr/metron/0.3.0/start_profiler_topology.sh
```
## Examples
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-analytics/metron-profiler/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/pom.xml b/metron-analytics/metron-profiler/pom.xml
index 0df1f47..428fdfb 100644
--- a/metron-analytics/metron-profiler/pom.xml
+++ b/metron-analytics/metron-profiler/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-analytics</artifactId>
- <version>0.2.1BETA</version>
+ <version>0.3.0</version>
</parent>
<artifactId>metron-profiler</artifactId>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-analytics/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/pom.xml b/metron-analytics/pom.xml
index da2f015..07d192c 100644
--- a/metron-analytics/pom.xml
+++ b/metron-analytics/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>Metron</artifactId>
- <version>0.2.1BETA</version>
+ <version>0.3.0</version>
</parent>
<description>Stream analytics for Metron</description>
<url>https://metron.incubator.apache.org/</url>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/amazon-ec2/conf/defaults.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/amazon-ec2/conf/defaults.yml b/metron-deployment/amazon-ec2/conf/defaults.yml
index 0e7f8dc..f4baa5a 100644
--- a/metron-deployment/amazon-ec2/conf/defaults.yml
+++ b/metron-deployment/amazon-ec2/conf/defaults.yml
@@ -75,7 +75,7 @@ num_partitions: 3
retention_in_gb: 25
# metron variables
-metron_version: 0.2.1BETA
+metron_version: 0.3.0
metron_directory: /usr/metron/{{ metron_version }}
pcapservice_port: 8081
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/inventory/full-dev-platform/group_vars/all
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/full-dev-platform/group_vars/all b/metron-deployment/inventory/full-dev-platform/group_vars/all
index 33456dc..2d07879 100644
--- a/metron-deployment/inventory/full-dev-platform/group_vars/all
+++ b/metron-deployment/inventory/full-dev-platform/group_vars/all
@@ -47,7 +47,7 @@ threatintel_hbase_table: threatintel
enrichment_hbase_table: enrichment
# metron
-metron_version: 0.2.1BETA
+metron_version: 0.3.0
metron_directory: /usr/metron/{{ metron_version }}
bro_version: "2.4.1"
fixbuf_version: "1.7.1"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/inventory/metron_example/group_vars/all
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/metron_example/group_vars/all b/metron-deployment/inventory/metron_example/group_vars/all
index e6052af..38f1440 100644
--- a/metron-deployment/inventory/metron_example/group_vars/all
+++ b/metron-deployment/inventory/metron_example/group_vars/all
@@ -54,7 +54,7 @@ num_partitions: 3
retention_in_gb: 25
# metron variables
-metron_version: 0.2.1BETA
+metron_version: 0.3.0
metron_directory: /usr/metron/{{ metron_version }}
pcapservice_port: 8081
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/inventory/quick-dev-platform/group_vars/all
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/quick-dev-platform/group_vars/all b/metron-deployment/inventory/quick-dev-platform/group_vars/all
index 2b1bada..e27e528 100644
--- a/metron-deployment/inventory/quick-dev-platform/group_vars/all
+++ b/metron-deployment/inventory/quick-dev-platform/group_vars/all
@@ -46,7 +46,7 @@ threatintel_hbase_table: threatintel
enrichment_hbase_table: enrichment
# metron
-metron_version: 0.2.1BETA
+metron_version: 0.3.0
metron_directory: /usr/metron/{{ metron_version }}
bro_version: "2.4.1"
fixbuf_version: "1.7.1"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/pom.xml b/metron-deployment/packaging/ambari/metron-mpack/pom.xml
index 174a840..eb2be17 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/pom.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/pom.xml
@@ -26,7 +26,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-deployment</artifactId>
- <version>0.2.1BETA</version>
+ <version>0.3.0</version>
<relativePath>../../..</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.1BETA/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.1BETA/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.1BETA/metainfo.xml
deleted file mode 100755
index 7fac89a..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.1BETA/metainfo.xml
+++ /dev/null
@@ -1,26 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<metainfo>
- <schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>METRON</name>
- <version>0.2.1BETA</version>
- <extends>common-services/METRON/0.2.1BETA</extends>
- </service>
- </services>
-</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.1BETA/role_command_order.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.1BETA/role_command_order.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.1BETA/role_command_order.json
deleted file mode 100755
index 0f4cd39..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.1BETA/role_command_order.json
+++ /dev/null
@@ -1,14 +0,0 @@
-{
- "_comment" : "Record format:",
- "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]",
- "general_deps" : {
- "_comment" : "dependencies for all cases",
- "METRON_INDEXING-INSTALL" : ["METRON_PARSERS-INSTALL"],
- "METRON_PARSERS-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START"],
- "METRON_ENRICHMENT_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START", "ENRICHMENT_MYSQL_SERVER-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START"],
- "METRON_ENRICHMENT_MYSQL_SERVER-STOP" : ["METRON_ENRICHMENT_MASTER-STOP"],
- "METRON_ENRICHMENT_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_ENRICHMENT_MASTER-START"],
- "METRON_INDEXING-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START","METRON_PARSERS-START"],
- "METRON_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_PARSERS-START","METRON_INDEXING-START"]
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.3.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.3.0/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.3.0/metainfo.xml
new file mode 100755
index 0000000..86231ec
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.3.0/metainfo.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>METRON</name>
+ <version>0.3.0</version>
+ <extends>common-services/METRON/0.3.0</extends>
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.3.0/role_command_order.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.3.0/role_command_order.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.3.0/role_command_order.json
new file mode 100755
index 0000000..0f4cd39
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.3.0/role_command_order.json
@@ -0,0 +1,14 @@
+{
+ "_comment" : "Record format:",
+ "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]",
+ "general_deps" : {
+ "_comment" : "dependencies for all cases",
+ "METRON_INDEXING-INSTALL" : ["METRON_PARSERS-INSTALL"],
+ "METRON_PARSERS-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START"],
+ "METRON_ENRICHMENT_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START", "ENRICHMENT_MYSQL_SERVER-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START"],
+ "METRON_ENRICHMENT_MYSQL_SERVER-STOP" : ["METRON_ENRICHMENT_MASTER-STOP"],
+ "METRON_ENRICHMENT_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_ENRICHMENT_MASTER-START"],
+ "METRON_INDEXING-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START","METRON_PARSERS-START"],
+ "METRON_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_PARSERS-START","METRON_INDEXING-START"]
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/configuration/metron-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/configuration/metron-env.xml
deleted file mode 100644
index a6ea541..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/configuration/metron-env.xml
+++ /dev/null
@@ -1,256 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<configuration supports_final="true">
- <property>
- <name>metron_home</name>
- <value>/usr/metron/0.2.1BETA</value>
- <description>Metron home directory</description>
- <display-name>Metron home</display-name>
- </property>
- <property>
- <name>metron_apps_hdfs_dir</name>
- <value>/apps/metron</value>
- <description>Metron apps HDFS dir</description>
- <display-name>Metron apps HDFS dir</display-name>
- </property>
- <property>
- <name>metron_zookeeper_config_dir</name>
- <value>config/zookeeper</value>
- <description>Metron Zookeeper config dir. Relative path to Metron home.</description>
- <display-name>Metron Zookeeper config dir</display-name>
- </property>
- <property>
- <name>metron_user</name>
- <value>metron</value>
- <property-type>USER</property-type>
- <description>The user for Metron</description>
- <display-name>Metron User</display-name>
- </property>
- <property>
- <name>metron_group</name>
- <value>metron</value>
- <property-type>GROUP</property-type>
- <description>The group for Metron</description>
- <display-name>Metron Group Name</display-name>
- </property>
- <property>
- <name>metron_topic_retention</name>
- <description>Kafka Retention in GB</description>
- <value>10</value>
- <display-name>Topic Retention</display-name>
- </property>
- <property>
- <name>parsers</name>
- <value>bro,snort,yaf</value>
- <description>Metron parsers to deploy</description>
- <display-name>Metron Parsers</display-name>
- </property>
- <property>
- <name>metron_enrichment_db_user</name>
- <value>metron</value>
- <description>Database username to use to connect to the database.</description>
- <display-name>Enrichment Database User</display-name>
- </property>
- <property>
- <name>metron_enrichment_db_port</name>
- <display-name>Metron Enrichment Database Port</display-name>
- <value>3306</value>
- <description>Database port to use to connect to the database.</description>
- </property>
- <property>
- <name>metron_enrichment_db_password</name>
- <value></value>
- <property-type>PASSWORD</property-type>
- <display-name>Metron Enrichment Database Password</display-name>
- <description>Password to use against database</description>
- <value-attributes>
- <type>password</type>
- <overridable>false</overridable>
- </value-attributes>
- <on-ambari-upgrade add="true"/>
- </property>
- <property>
- <name>mysql_admin_password</name>
- <value></value>
- <property-type>PASSWORD</property-type>
- <display-name>MySQL root user password</display-name>
- <description>Password to use to add Metron user to MySQL</description>
- <value-attributes>
- <type>password</type>
- <overridable>false</overridable>
- </value-attributes>
- <on-ambari-upgrade add="true"/>
- </property>
- <property>
- <name>metron_indexing_topology</name>
- <value>indexing</value>
- <description>The Storm topology name for Indexing</description>
- <display-name>Indexing Topology Name</display-name>
- </property>
- <property>
- <name>es_cluster_name</name>
- <value>metron</value>
- <description>Name of Elasticsearch Cluster</description>
- <display-name>Elasticsearch Cluster Name</display-name>
- </property>
- <property>
- <name>geoip_url</name>
- <value>http://geolite.maxmind.com/download/geoip/database/GeoLiteCity_CSV/GeoLiteCity-latest.tar.xz</value>
- <description>Location of the GeoIP data to load.</description>
- <display-name>GEOIP Load Datafile URL</display-name>
- </property>
- <property require-input="true">
- <name>es_hosts</name>
- <value></value>
- <description>Comma delimited list of Elasticsearch Hosts. (eshost1,eshost2)</description>
- <display-name>Elasticsearch Hosts</display-name>
- </property>
- <property>
- <name>es_binary_port</name>
- <value>9300</value>
- <description>Elasticsearch binary port. (9300)</description>
- <display-name>Elasticsearch Binary Port</display-name>
- </property>
- <property>
- <name>es_http_port</name>
- <value>9200</value>
- <description>Elasticsearch HTTP port. (9200)</description>
- <display-name>Elasticsearch HTTP port</display-name>
- </property>
- <property require-input = "true">
- <name>storm_rest_addr</name>
- <display-name>Storm Rest Server Address</display-name>
- <description>URL of Storm UI (storm.ui.hostname:8744)</description>
- <value></value>
- </property>
- <property>
- <name>repo_type</name>
- <display-name>Repository Type</display-name>
- <description>Type of Repository: Local or Remote</description>
- <value>local</value>
- <value-attributes>
- <overridable>false</overridable>
- <type>value-list</type>
- <entries>
- <entry>
- <value>local</value>
- <label>Local</label>
- </entry>
- <entry>
- <value>remote</value>
- <label>Remote</label>
- </entry>
- </entries>
- <selection-cardinality>1</selection-cardinality>
- </value-attributes>
- </property>
- <property>
- <name>repo_url</name>
- <display-name>Repository URL</display-name>
- <value></value>
- </property>
- <property>
- <name>install_mysql</name>
- <description>Install New MySQL Instance for Enrichments</description>
- <display-name>Install MySQL</display-name>
- <value>Yes</value>
- <value-attributes>
- <overridable>false</overridable>
- <type>value-list</type>
- <entries>
- <entry>
- <value>Yes</value>
- <label>Yes</label>
- </entry>
- <entry>
- <value>No</value>
- <label>No</label>
- </entry>
- </entries>
- <selection-cardinality>1</selection-cardinality>
- </value-attributes>
- </property>
- <property>
- <name>mysql_host</name>
- <display-name>MySQL Host Address</display-name>
- <value></value>
- </property>
- <property>
- <name>global-json</name>
- <display-name>global.json template</display-name>
- <description>This is the jinja template for global.json file</description>
- <value>
-{
-"es.clustername": "{{ es_cluster_name }}",
-"es.ip": "{{ es_url }}",
-"es.date.format": "yyyy.MM.dd.HH"
-}
- </value>
- <value-attributes>
- <type>content</type>
- </value-attributes>
- </property>
- <property>
- <name>elasticsearch-properties</name>
- <description>The template for the elasticsearch.properties file.</description>
- <display-name>elasticsearch.properties template</display-name>
- <value>
-##### Storm #####
-indexing.workers=1
-indexing.executors=0
-##### Kafka #####
-kafka.zk={{ zookeeper_quorum }}
-kafka.broker={{ kafka_brokers }}
-kafka.start=WHERE_I_LEFT_OFF
-##### Indexing #####
-index.input.topic=indexing
-index.error.topic=indexing_error
-writer.class.name=org.apache.metron.elasticsearch.writer.ElasticsearchWriter
-##### Metrics #####
-#reporters
-org.apache.metron.metrics.reporter.graphite=true
-org.apache.metron.metrics.reporter.console=false
-org.apache.metron.metrics.reporter.jmx=false
-#Graphite Addresses
-org.apache.metron.metrics.graphite.address=localhost
-org.apache.metron.metrics.graphite.port=2023
-#TelemetryParserBolt
-org.apache.metron.metrics.TelemetryParserBolt.acks=true
-org.apache.metron.metrics.TelemetryParserBolt.emits=true
-org.apache.metron.metrics.TelemetryParserBolt.fails=true
-##### HDFS #####
-bolt.hdfs.batch.size=5000
-bolt.hdfs.field.delimiter=|
-bolt.hdfs.rotation.policy=org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy
-bolt.hdfs.rotation.policy.units=DAYS
-bolt.hdfs.rotation.policy.count=1
-bolt.hdfs.file.rotation.size.in.mb=5
-bolt.hdfs.file.system.url={{ default_fs }}
-bolt.hdfs.wip.file.path=/paloalto/wip
-bolt.hdfs.finished.file.path=/paloalto/rotated
-bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec
-index.hdfs.output={{ metron_apps_enrichment_dir }}
- </value>
- <value-attributes>
- <type>content</type>
- </value-attributes>
- <on-ambari-upgrade add="true"/>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/metainfo.xml
deleted file mode 100644
index f94d76c..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/metainfo.xml
+++ /dev/null
@@ -1,288 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<metainfo>
- <schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>METRON</name>
- <displayName>Metron</displayName>
- <comment>A scalable advanced security analytics framework built on Hadoop</comment>
- <version>0.2.1BETA</version>
- <components>
-
- <component>
- <name>METRON_PARSERS</name>
- <displayName>Metron Parsers</displayName>
- <category>MASTER</category>
- <cardinality>1</cardinality>
- <versionAdvertised>false</versionAdvertised>
- <reassignAllowed>false</reassignAllowed>
- <clientsToUpdateConfigs></clientsToUpdateConfigs>
- <dependencies>
- <dependency>
- <name>HDFS/HDFS_CLIENT</name>
- <scope>host</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- <dependency>
- <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
- <scope>cluster</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- <dependency>
- <name>STORM/SUPERVISOR</name>
- <scope>host</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- <dependency>
- <name>KAFKA/KAFKA_BROKER</name>
- <scope>host</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- </dependencies>
- <commandScript>
- <script>scripts/parser_master.py</script>
- <scriptType>PYTHON</scriptType>
- </commandScript>
- </component>
-
- <component>
- <name>METRON_ENRICHMENT_MASTER</name>
- <displayName>Metron Enrichment</displayName>
- <category>MASTER</category>
- <cardinality>1</cardinality>
- <versionAdvertised>false</versionAdvertised>
- <dependencies>
- <dependency>
- <name>HDFS/HDFS_CLIENT</name>
- <scope>host</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- <dependency>
- <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
- <scope>cluster</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- <dependency>
- <name>KAFKA/KAFKA_BROKER</name>
- <scope>host</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- </dependencies>
- <commandScript>
- <script>scripts/enrichment_master.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
-
- <component>
- <name>METRON_ENRICHMENT_MYSQL_SERVER</name>
- <displayName>MySQL Server</displayName>
- <category>MASTER</category>
- <cardinality>1</cardinality>
- <versionAdvertised>false</versionAdvertised>
- <clientsToUpdateConfigs></clientsToUpdateConfigs>
- <commandScript>
- <script>scripts/mysql_server.py</script>
- <scriptType>PYTHON</scriptType>
- </commandScript>
- <customCommands>
- <customCommand>
- <name>CLEAN</name>
- <commandScript>
- <script>scripts/mysql_server.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </customCommand>
- </customCommands>
- </component>
-
- <component>
- <name>METRON_INDEXING</name>
- <displayName>Metron Indexing</displayName>
- <category>MASTER</category>
- <cardinality>1</cardinality>
- <versionAdvertised>false</versionAdvertised>
- <reassignAllowed>false</reassignAllowed>
- <clientsToUpdateConfigs></clientsToUpdateConfigs>
- <dependencies>
- <dependency>
- <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
- <scope>cluster</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- <dependency>
- <name>STORM/SUPERVISOR</name>
- <scope>host</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- <dependency>
- <name>KAFKA/KAFKA_BROKER</name>
- <scope>host</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- </dependencies>
- <commandScript>
- <script>scripts/indexing_master.py</script>
- <scriptType>PYTHON</scriptType>
- </commandScript>
- <customCommands>
- <customCommand>
- <name>ELASTICSEARCH_TEMPLATE_INSTALL</name>
- <commandScript>
- <script>scripts/indexing_master.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </customCommand>
- <customCommand>
- <name>ELASTICSEARCH_TEMPLATE_DELETE</name>
- <commandScript>
- <script>scripts/indexing_master.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </customCommand>
- </customCommands>
- </component>
- </components>
-
- <themes>
- <theme>
- <fileName>metron_theme.json</fileName>
- <default>true</default>
- </theme>
- </themes>
-
- <osSpecifics>
- <osSpecific>
- <osFamily>any</osFamily>
- <packages>
- <package>
- <name>metron-common</name>
- </package>
- <package>
- <name>metron-data-management</name>
- </package>
- <package>
- <name>metron-parsers</name>
- </package>
- <package>
- <name>metron-enrichment</name>
- </package>
- <package>
- <name>metron-indexing</name>
- </package>
- <package>
- <name>metron-elasticsearch</name>
- </package>
- <package>
- <name>metron-pcap</name>
- </package>
- </packages>
- </osSpecific>
- <osSpecific>
- <osFamily>redhat6,redhat7</osFamily>
- <packages>
- <package>
- <name>expect</name>
- <skipUpgrade>true</skipUpgrade>
- </package>
- </packages>
- </osSpecific>
- <osSpecific>
- <osFamily>redhat7</osFamily>
- <packages>
- <package>
- <name>mysql-community-release</name>
- <skipUpgrade>true</skipUpgrade>
- </package>
- <package>
- <name>mysql-community-server</name>
- <skipUpgrade>true</skipUpgrade>
- </package>
- <package>MySQL-python</package>
- </packages>
- </osSpecific>
- <osSpecific>
- <osFamily>redhat6</osFamily>
- <packages>
- <package>
- <name>mysql-server</name>
- <skipUpgrade>true</skipUpgrade>
- </package>
- <package>
- <name>mysql</name>
- <skipUpgrade>true</skipUpgrade>
- </package>
- </packages>
- </osSpecific>
- </osSpecifics>
- <commandScript>
- <script>scripts/service_check.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>300</timeout>
- </commandScript>
-
- <requiredServices>
- <service>HDFS</service>
- <service>KAFKA</service>
- <service>STORM</service>
- <service>ZOOKEEPER</service>
- <service>HBASE</service>
- </requiredServices>
-
- <configuration-dependencies>
- <config-type>metron-env</config-type>
- <config-type>storm-env</config-type>
- <config-type>storm-site</config-type>
- <config-type>kafka-broker</config-type>
- <config-type>kafka-env</config-type>
- </configuration-dependencies>
- <restartRequiredAfterChange>true</restartRequiredAfterChange>
- <quickLinksConfigurations>
- <quickLinksConfiguration>
- <fileName>quicklinks.json</fileName>
- <default>true</default>
- </quickLinksConfiguration>
- </quickLinksConfigurations>
- </service>
- </services>
-</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/addMysqlUser.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/addMysqlUser.sh b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/addMysqlUser.sh
deleted file mode 100755
index 8a6a788..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/addMysqlUser.sh
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/bin/sh
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#
-mysqldbuser=$1
-mysqldbpasswd=$2
-mysqldbhost=$3
-mysqlqdminpassword=$4
-myhostname=$(hostname -f)
-
-
-echo "Adding user ${mysqldbuser}@${mysqldbhost} and ${mysqldbuser}@localhost"
-expect <<EOF
-log_user 0
-# start mysql process using password prompt
-spawn mysql -u root -p
-expect "password:"
-send "${mysqlqdminpassword}\r"
-# echo all output until the end
-expect "mysql>"
-send "CREATE USER '${mysqldbuser}'@'${mysqldbhost}' IDENTIFIED BY '${mysqldbpasswd}';\r"
-expect "mysql>"
-send "CREATE USER '${mysqldbuser}'@'localhost' IDENTIFIED BY '${mysqldbpasswd}';\r"
-expect "mysql>"
-send "GRANT ALL PRIVILEGES ON GEO.* TO '${mysqldbuser}'@'%' IDENTIFIED BY '${mysqldbpasswd}';\r"
-log_user 1
-expect "mysql>"
-send "GRANT ALL PRIVILEGES ON GEO.* TO '${mysqldbuser}'@'${mysqldbhost}';\r"
-expect "mysql>"
-send "GRANT ALL PRIVILEGES ON GEO.* TO '${mysqldbuser}'@'localhost';\r"
-expect "mysql>"
-send "flush privileges;\r"
-send "\q"
-EOF
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/bro_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/bro_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/bro_index.template
deleted file mode 100644
index 951d967..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/bro_index.template
+++ /dev/null
@@ -1,217 +0,0 @@
-{
- "template": "bro_index*",
- "mappings": {
- "bro_doc": {
- "_timestamp": {
- "enabled": true
- },
- "dynamic_templates": [
- {
- "geo_location_point": {
- "match": "enrichments:geo:*:location_point",
- "match_mapping_type": "*",
- "mapping": {
- "type": "geo_point"
- }
- }
- },
- {
- "geo_country": {
- "match": "enrichments:geo:*:country",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_city": {
- "match": "enrichments:geo:*:city",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_location_id": {
- "match": "enrichments:geo:*:locID",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_dma_code": {
- "match": "enrichments:geo:*:dmaCode",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_postal_code": {
- "match": "enrichments:geo:*:postalCode",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_latitude": {
- "match": "enrichments:geo:*:latitude",
- "match_mapping_type": "*",
- "mapping": {
- "type": "float"
- }
- }
- },
- {
- "geo_longitude": {
- "match": "enrichments:geo:*:longitude",
- "match_mapping_type": "*",
- "mapping": {
- "type": "float"
- }
- }
- },
- {
- "timestamps": {
- "match": "*:ts",
- "match_mapping_type": "*",
- "mapping": {
- "type": "date",
- "format": "epoch_millis"
- }
- }
- }
- ],
- "properties": {
- "timestamp": {
- "type": "date",
- "format": "epoch_millis"
- },
- "source:type": {
- "type": "string",
- "index": "not_analyzed"
- },
- "ip_dst_addr": {
- "type": "ip"
- },
- "ip_dst_port": {
- "type": "integer"
- },
- "ip_src_addr": {
- "type": "ip"
- },
- "ip_src_port": {
- "type": "integer"
- },
- "status_code": {
- "type": "integer"
- },
- "method": {
- "type": "string",
- "index": "not_analyzed"
- },
- "protocol": {
- "type": "string",
- "index": "not_analyzed"
- },
- "request_body_len": {
- "type": "integer"
- },
- "uri": {
- "type": "string",
- "index": "not_analyzed"
- },
- "uid": {
- "type": "string",
- "index": "not_analyzed"
- },
- "referrer": {
- "type": "string",
- "index": "not_analyzed"
- },
- "trans_depth": {
- "type": "integer"
- },
- "host": {
- "type": "string",
- "index": "not_analyzed"
- },
- "status_msg": {
- "type": "string",
- "index": "not_analyzed"
- },
- "response_body_len": {
- "type": "integer"
- },
- "user_agent": {
- "type": "string"
- },
- "query": {
- "type": "string",
- "index": "not_analyzed"
- },
- "answers": {
- "type": "ip"
- },
- "AA": {
- "type": "boolean"
- },
- "TC": {
- "type": "boolean"
- },
- "RA": {
- "type": "boolean"
- },
- "RD": {
- "type": "boolean"
- },
- "rejected": {
- "type": "boolean"
- },
- "qclass_name": {
- "type": "string",
- "index": "not_analyzed"
- },
- "proto": {
- "type": "string",
- "index": "not_analyzed"
- },
- "rcode": {
- "type": "integer"
- },
- "rcode_name": {
- "type": "string",
- "index": "not_analyzed"
- },
- "trans_id": {
- "type": "integer"
- },
- "Z": {
- "type": "integer"
- },
- "qclass": {
- "type": "integer"
- },
- "qtype": {
- "type": "integer"
- },
- "qtype_name": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/createMysqlGeoIp.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/createMysqlGeoIp.sh b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/createMysqlGeoIp.sh
deleted file mode 100755
index 40f11e1..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/createMysqlGeoIp.sh
+++ /dev/null
@@ -1,53 +0,0 @@
-#!/bin/sh
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#
-from ../scripts/params import params
-
-geoipscript=$1
-geoipurl=$2
-mysqlqdminpassword=$3
-
-# Download and extract the actual GeoIP files
-mkdir -p /tmp/geoip
-mkdir -p /var/lib/mysql-files/
-
-# Stage the GeoIP data
-pushd /tmp/geoip
-curl -O ${geoipurl}
-tar xf GeoLiteCity-latest.tar.xz
-cp /tmp/geoip/*/*.csv /var/lib/mysql-files/
-popd
-
-# Load MySQL with the GeoIP data
-expect <<EOF
-log_user 0
-# start mysql process using password prompt
-spawn mysql -u root -p
-expect "password:"
-send "${mysqlqdminpassword}\r"
-# echo all output until the end
-log_user 1
-expect "mysql>"
-send "source ${geoipscript}\r"
-expect "mysql>"
-send "show databases;\r"
-send "\q"
-EOF
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/removeMysqlUser.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/removeMysqlUser.sh b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/removeMysqlUser.sh
deleted file mode 100755
index 3fc7afb..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/removeMysqlUser.sh
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/usr/bin/env bash
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#
-
-mysqldbuser=$1
-userhost=$2
-mysqlqdminpassword=$3
-myhostname=$(hostname -f)
-sudo_prefix="/var/lib/ambari-agent/ambari-sudo.sh -H -E"
-
-echo "Removing user $mysqldbuser@$userhost"
-expect <<EOF
-log_user 0
-# start mysql process using password prompt
-spawn mysql -u root -p
-expect "password:"
-send "${mysqlqdminpassword}\r"
-# echo all output until the end
-log_user 1
-expect "mysql>"
-send "DROP USER '${mysqldbuser}'@'${userhost}';\r"
-expect "mysql>"
-send "flush privileges;;\r"
-send "\q"
-EOF
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/snort_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/snort_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/snort_index.template
deleted file mode 100644
index bf943df..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/snort_index.template
+++ /dev/null
@@ -1,183 +0,0 @@
-{
- "template": "snort_index*",
- "mappings": {
- "snort_doc": {
- "_timestamp": {
- "enabled": true
- },
- "dynamic_templates": [
- {
- "geo_location_point": {
- "match": "enrichments:geo:*:location_point",
- "match_mapping_type": "*",
- "mapping": {
- "type": "geo_point"
- }
- }
- },
- {
- "geo_country": {
- "match": "enrichments:geo:*:country",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_city": {
- "match": "enrichments:geo:*:city",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_location_id": {
- "match": "enrichments:geo:*:locID",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_dma_code": {
- "match": "enrichments:geo:*:dmaCode",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_postal_code": {
- "match": "enrichments:geo:*:postalCode",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_latitude": {
- "match": "enrichments:geo:*:latitude",
- "match_mapping_type": "*",
- "mapping": {
- "type": "float"
- }
- }
- },
- {
- "geo_longitude": {
- "match": "enrichments:geo:*:longitude",
- "match_mapping_type": "*",
- "mapping": {
- "type": "float"
- }
- }
- },
- {
- "timestamps": {
- "match": "*:ts",
- "match_mapping_type": "*",
- "mapping": {
- "type": "date",
- "format": "epoch_millis"
- }
- }
- }
- ],
- "properties": {
- "timestamp": {
- "type": "date",
- "format": "epoch_millis"
- },
- "source:type": {
- "type": "string",
- "index": "not_analyzed"
- },
- "ip_dst_addr": {
- "type": "ip"
- },
- "ip_dst_port": {
- "type": "integer"
- },
- "ip_src_addr": {
- "type": "ip"
- },
- "ip_src_port": {
- "type": "integer"
- },
- "dgmlen": {
- "type": "integer"
- },
- "ethdst": {
- "type": "string",
- "index": "not_analyzed"
- },
- "ethlen": {
- "type": "string",
- "index": "not_analyzed"
- },
- "ethsrc": {
- "type": "string",
- "index": "not_analyzed"
- },
- "id": {
- "type": "integer"
- },
- "iplen": {
- "type": "integer"
- },
- "is_alert": {
- "type": "boolean"
- },
- "msg": {
- "type": "string"
- },
- "protocol": {
- "type": "string",
- "index": "not_analyzed"
- },
- "sig_generator": {
- "type": "string",
- "index": "not_analyzed"
- },
- "sig_id": {
- "type": "integer"
- },
- "sig_rev": {
- "type": "string"
- },
- "tcpack": {
- "type": "string"
- },
- "tcpflags": {
- "type": "string"
- },
- "tcpseq": {
- "type": "string"
- },
- "tcpwindow": {
- "type": "string"
- },
- "threat:triage:level": {
- "type": "double"
- },
- "tos": {
- "type": "integer"
- },
- "ttl": {
- "type": "integer"
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/yaf_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/yaf_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/yaf_index.template
deleted file mode 100644
index 7743afc..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/yaf_index.template
+++ /dev/null
@@ -1,205 +0,0 @@
-{
- "template": "yaf_index*",
- "mappings": {
- "yaf_doc": {
- "_timestamp": {
- "enabled": true
- },
- "dynamic_templates": [
- {
- "geo_location_point": {
- "match": "enrichments:geo:*:location_point",
- "match_mapping_type": "*",
- "mapping": {
- "type": "geo_point"
- }
- }
- },
- {
- "geo_country": {
- "match": "enrichments:geo:*:country",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_city": {
- "match": "enrichments:geo:*:city",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_location_id": {
- "match": "enrichments:geo:*:locID",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_dma_code": {
- "match": "enrichments:geo:*:dmaCode",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_postal_code": {
- "match": "enrichments:geo:*:postalCode",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_latitude": {
- "match": "enrichments:geo:*:latitude",
- "match_mapping_type": "*",
- "mapping": {
- "type": "float"
- }
- }
- },
- {
- "geo_longitude": {
- "match": "enrichments:geo:*:longitude",
- "match_mapping_type": "*",
- "mapping": {
- "type": "float"
- }
- }
- },
- {
- "timestamps": {
- "match": "*:ts",
- "match_mapping_type": "*",
- "mapping": {
- "type": "date",
- "format": "epoch_millis"
- }
- }
- }
- ],
- "properties": {
- "timestamp": {
- "type": "date",
- "format": "epoch_millis"
- },
- "source:type": {
- "type": "string",
- "index": "not_analyzed"
- },
- "ip_dst_addr": {
- "type": "ip"
- },
- "ip_dst_port": {
- "type": "integer"
- },
- "ip_src_addr": {
- "type": "ip"
- },
- "ip_src_port": {
- "type": "integer"
- },
- "start_time": {
- "type": "date",
- "format": "epoch_millis"
- },
- "end_time": {
- "type": "date",
- "format": "epoch_millis"
- },
- "duration": {
- "type": "double"
- },
- "rtt": {
- "type": "double"
- },
- "proto": {
- "type": "string",
- "index": "not_analyzed"
- },
- "sip": {
- "type": "string",
- "index": "not_analyzed"
- },
- "sp": {
- "type": "string",
- "index": "not_analyzed"
- },
- "dip": {
- "type": "string",
- "index": "not_analyzed"
- },
- "dp": {
- "type": "string",
- "index": "not_analyzed"
- },
- "iflags": {
- "type": "string",
- "index": "not_analyzed"
- },
- "uflags": {
- "type": "string",
- "index": "not_analyzed"
- },
- "riflags": {
- "type": "string"
- },
- "ruflags": {
- "type": "string",
- "index": "not_analyzed"
- },
- "isn": {
- "type": "string",
- "index": "not_analyzed"
- },
- "risn": {
- "type": "string",
- "index": "not_analyzed"
- },
- "tag": {
- "type": "string",
- "index": "not_analyzed"
- },
- "rtag": {
- "type": "string",
- "index": "not_analyzed"
- },
- "pkt": {
- "type": "integer"
- },
- "oct": {
- "type": "integer"
- },
- "rpkt": {
- "type": "integer"
- },
- "roct": {
- "type": "integer"
- },
- "app": {
- "type": "string",
- "index": "not_analyzed"
- },
- "end-reason": {
- "type": "string"
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/enrichment_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/enrichment_commands.py
deleted file mode 100755
index f316089..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/enrichment_commands.py
+++ /dev/null
@@ -1,177 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-
-import os
-import time
-
-from resource_management.core.logger import Logger
-from resource_management.core.resources.system import Execute, File
-from resource_management.libraries.resources.hdfs_resource import HdfsResource
-
-import metron_service
-
-
-# Wrap major operations and functionality in this class
-class EnrichmentCommands:
- __params = None
- __enrichment_topology = None
- __enrichment_topic = None
- __configured = False
-
- def __init__(self, params):
- if params is None:
- raise ValueError("params argument is required for initialization")
- self.__params = params
- self.__enrichment_topology = params.metron_enrichment_topology
- self.__enrichment_topic = params.metron_enrichment_topic
- self.__configured = os.path.isfile(self.__params.enrichment_configured_flag_file)
-
- def is_configured(self):
- return self.__configured
-
- def set_configured(self):
- File(self.__params.enrichment_configured_flag_file,
- content="",
- owner=self.__params.metron_user,
- mode=0775)
-
- def setup_repo(self):
-
- def local_repo():
- Logger.info("Setting up local repo")
- Execute("yum -y install createrepo")
- Execute("createrepo /localrepo")
- Execute("chmod -R o-w+r /localrepo")
-
- def remote_repo():
- Logger.info('Using remote repo')
-
- yum_repo_types = {
- 'local': local_repo,
- 'remote': remote_repo
- }
-
- repo_type = self.__params.yum_repo_type
-
- if repo_type in yum_repo_types:
- yum_repo_types[repo_type]()
- Logger.info("Writing out repo file")
- repo_template = ("echo \"[METRON-0.2.1BETA]\n"
- "name=Metron 0.2.1BETA packages\n"
- "baseurl={0}\n"
- "gpgcheck=0\n"
- "enabled=1\n\""
- " > /etc/yum.repos.d/metron.repo")
- Execute(repo_template.format(self.__params.repo_url))
- else:
- raise ValueError("Unsupported repo type '{0}'".format(repo_type))
-
- def init_kafka_topics(self):
- Logger.info('Creating Kafka topics')
- command_template = """{0}/kafka-topics.sh \
- --zookeeper {1} \
- --create \
- --topic {2} \
- --partitions {3} \
- --replication-factor {4} \
- --config retention.bytes={5}"""
- num_partitions = 1
- replication_factor = 1
- retention_gigabytes = int(self.__params.metron_topic_retention)
- retention_bytes = retention_gigabytes * 1024 * 1024 * 1024
- Logger.info("Creating topics for enrichment")
-
- Logger.info("Creating topic'{0}'".format(self.__enrichment_topic))
- Execute(command_template.format(self.__params.kafka_bin_dir,
- self.__params.zookeeper_quorum,
- self.__enrichment_topic,
- num_partitions,
- replication_factor,
- retention_bytes))
- Logger.info("Done creating Kafka topics")
-
- def init_hdfs_dir(self):
- self.__params.HdfsResource(self.__params.metron_apps_enrichment_dir,
- type="directory",
- action="create_on_execute",
- owner=self.__params.metron_user,
- group=self.__params.user_group,
- mode=0775,
- )
-
- def start_enrichment_topology(self):
- Logger.info("Starting Metron enrichment topology: {0}".format(self.__enrichment_topology))
- start_cmd_template = """{0}/bin/start_enrichment_topology.sh \
- -s {1} \
- -z {2}"""
- Logger.info('Starting ' + self.__enrichment_topology)
- Execute(start_cmd_template.format(self.__params.metron_home, self.__enrichment_topology, self.__params.zookeeper_quorum))
-
- Logger.info('Finished starting enrichment topology')
-
- def stop_enrichment_topology(self):
- Logger.info('Stopping ' + self.__enrichment_topology)
- stop_cmd = 'storm kill ' + self.__enrichment_topology
- Execute(stop_cmd)
- Logger.info('Done stopping enrichment topologies')
-
- def restart_enrichment_topology(self, env):
- Logger.info('Restarting the enrichment topologies')
- self.stop_enrichment_topology()
-
- # Wait for old topology to be cleaned up by Storm, before starting again.
- retries = 0
- topology_active = self.is_topology_active(env)
- while topology_active and retries < 3:
- Logger.info('Existing topology still active. Will wait and retry')
- time.sleep(40)
- topology_active = self.is_topology_active(env)
- retries += 1
-
- if not topology_active:
- self.start_enrichment_topology()
- Logger.info('Done restarting the enrichment topology')
- else:
- Logger.warning('Retries exhausted. Existing topology not cleaned up. Aborting topology start.')
-
- def is_topology_active(self, env):
- env.set_params(self.__params)
-
- active = True
- topologies = metron_service.get_running_topologies()
- is_running = False
- if self.__enrichment_topology in topologies:
- is_running = topologies[self.__enrichment_topology] in ['ACTIVE', 'REBALANCING']
- active &= is_running
- return active
-
- def create_hbase_tables(self):
- add_enrichment_cmd = "echo \"create '{0}','{1}'\" | hbase shell -n".format(self.__params.enrichment_table, self.__params.enrichment_cf)
- Execute(add_enrichment_cmd,
- tries=3,
- try_sleep=5,
- logoutput=False,
- path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'
- )
-
- add_threatintel_cmd = "echo \"create '{0}','{1}'\" | hbase shell -n".format(self.__params.threatintel_table, self.__params.threatintel_cf)
- Execute(add_threatintel_cmd,
- tries=3,
- try_sleep=5,
- logoutput=False,
- path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'
- )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/enrichment_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/enrichment_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/enrichment_master.py
deleted file mode 100755
index 97ace20..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/enrichment_master.py
+++ /dev/null
@@ -1,87 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-
-from resource_management.core.exceptions import ComponentIsNotRunning
-from resource_management.core.logger import Logger
-from resource_management.core.resources.system import File
-from resource_management.core.source import Template
-from resource_management.libraries.functions.format import format
-from resource_management.libraries.script import Script
-
-from enrichment_commands import EnrichmentCommands
-
-
-class Enrichment(Script):
- def install(self, env):
- from params import params
- env.set_params(params)
- commands = EnrichmentCommands(params)
- commands.setup_repo()
- self.install_packages(env)
- self.configure(env)
-
- def configure(self, env, upgrade_type=None, config_dir=None):
- from params import params
- env.set_params(params)
-
- File(format("{metron_config_path}/enrichment.properties"),
- content=Template("enrichment.properties.j2"),
- owner=params.metron_user,
- group=params.metron_group
- )
-
- def start(self, env, upgrade_type=None):
- from params import params
- env.set_params(params)
- commands = EnrichmentCommands(params)
-
- if not commands.is_configured():
- commands.init_kafka_topics()
- commands.create_hbase_tables()
- commands.init_hdfs_dir()
- commands.set_configured()
-
- commands.start_enrichment_topology()
-
- def stop(self, env, upgrade_type=None):
- from params import params
- env.set_params(params)
- commands = EnrichmentCommands(params)
- commands.stop_enrichment_topology()
-
- def status(self, env):
- from params import status_params
- env.set_params(status_params)
- commands = EnrichmentCommands(status_params)
-
- if not commands.is_topology_active(env):
- raise ComponentIsNotRunning()
-
- def restart(self, env):
- from params import params
- env.set_params(params)
- commands = EnrichmentCommands(params)
- commands.restart_enrichment_topology(env)
-
- def kafkabuild(self, env, upgrade_type=None):
- from params import params
- env.set_params(params)
- commands = EnrichmentCommands(params)
- commands.init_kafka_topics()
-
-
-if __name__ == "__main__":
- Enrichment().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/indexing_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/indexing_commands.py
deleted file mode 100755
index fdb90e9..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/indexing_commands.py
+++ /dev/null
@@ -1,142 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-
-import os
-import time
-
-from resource_management.core.logger import Logger
-from resource_management.core.resources.system import Execute, File
-
-import metron_service
-
-
-# Wrap major operations and functionality in this class
-class IndexingCommands:
- __params = None
- __indexing = None
- __configured = False
-
- def __init__(self, params):
- if params is None:
- raise ValueError("params argument is required for initialization")
- self.__params = params
- self.__indexing = params.metron_indexing_topology
- self.__configured = os.path.isfile(self.__params.indexing_configured_flag_file)
-
- def is_configured(self):
- return self.__configured
-
- def set_configured(self):
- File(self.__params.indexing_configured_flag_file,
- content="",
- owner=self.__params.metron_user,
- mode=0775)
-
- def setup_repo(self):
- def local_repo():
- Logger.info("Setting up local repo")
- Execute("yum -y install createrepo")
- Execute("createrepo /localrepo")
- Execute("chmod -R o-w+r /localrepo")
- Execute("echo \"[METRON-0.2.1BETA]\n"
- "name=Metron 0.2.1BETA packages\n"
- "baseurl=file:///localrepo\n"
- "gpgcheck=0\n"
- "enabled=1\" > /etc/yum.repos.d/local.repo")
-
- def remote_repo():
- print('Using remote repo')
-
- yum_repo_types = {
- 'local': local_repo,
- 'remote': remote_repo
- }
- repo_type = self.__params.yum_repo_type
- if repo_type in yum_repo_types:
- yum_repo_types[repo_type]()
- else:
- raise ValueError("Unsupported repo type '{0}'".format(repo_type))
-
- def init_kafka_topics(self):
- Logger.info('Creating Kafka topics')
- command_template = """{0}/kafka-topics.sh \
- --zookeeper {1} \
- --create \
- --topic {2} \
- --partitions {3} \
- --replication-factor {4} \
- --config retention.bytes={5}"""
- num_partitions = 1
- replication_factor = 1
- retention_gigabytes = int(self.__params.metron_topic_retention)
- retention_bytes = retention_gigabytes * 1024 * 1024 * 1024
- Logger.info("Creating topics for indexing")
-
- Logger.info("Creating topic'{0}'".format(self.__indexing))
- Execute(command_template.format(self.__params.kafka_bin_dir,
- self.__params.zookeeper_quorum,
- self.__indexing,
- num_partitions,
- replication_factor,
- retention_bytes))
- Logger.info("Done creating Kafka topics")
-
- def start_indexing_topology(self):
- Logger.info("Starting Metron indexing topology: {0}".format(self.__indexing))
- start_cmd_template = """{0}/bin/start_elasticsearch_topology.sh \
- -s {1} \
- -z {2}"""
- Logger.info('Starting ' + self.__indexing)
- Execute(start_cmd_template.format(self.__params.metron_home, self.__indexing, self.__params.zookeeper_quorum))
-
- Logger.info('Finished starting indexing topology')
-
- def stop_indexing_topology(self):
- Logger.info('Stopping ' + self.__indexing)
- stop_cmd = 'storm kill ' + self.__indexing
- Execute(stop_cmd)
- Logger.info('Done stopping indexing topologies')
-
- def restart_indexing_topology(self, env):
- Logger.info('Restarting the indexing topologies')
- self.stop_indexing_topology()
-
- # Wait for old topology to be cleaned up by Storm, before starting again.
- retries = 0
- topology_active = self.is_topology_active(env)
- while self.is_topology_active(env) and retries < 3:
- Logger.info('Existing topology still active. Will wait and retry')
- time.sleep(10)
- retries += 1
-
- if not topology_active:
- Logger.info('Waiting for storm kill to complete')
- time.sleep(30)
- self.start_indexing_topology()
- Logger.info('Done restarting the indexing topologies')
- else:
- Logger.warning('Retries exhausted. Existing topology not cleaned up. Aborting topology start.')
-
- def is_topology_active(self, env):
- env.set_params(self.__params)
- active = True
- topologies = metron_service.get_running_topologies()
- is_running = False
- if self.__indexing in topologies:
- is_running = topologies[self.__indexing] in ['ACTIVE', 'REBALANCING']
- active &= is_running
- return active
[2/4] incubator-metron git commit: METRON-533 Up the version for
release closes apache/incubator-metron#353
Posted by le...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/metainfo.xml
new file mode 100644
index 0000000..08b8461
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/metainfo.xml
@@ -0,0 +1,288 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>METRON</name>
+ <displayName>Metron</displayName>
+ <comment>A scalable advanced security analytics framework built on Hadoop</comment>
+ <version>0.3.0</version>
+ <components>
+
+ <component>
+ <name>METRON_PARSERS</name>
+ <displayName>Metron Parsers</displayName>
+ <category>MASTER</category>
+ <cardinality>1</cardinality>
+ <versionAdvertised>false</versionAdvertised>
+ <reassignAllowed>false</reassignAllowed>
+ <clientsToUpdateConfigs></clientsToUpdateConfigs>
+ <dependencies>
+ <dependency>
+ <name>HDFS/HDFS_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
+ <scope>cluster</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>STORM/SUPERVISOR</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>KAFKA/KAFKA_BROKER</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ <commandScript>
+ <script>scripts/parser_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>METRON_ENRICHMENT_MASTER</name>
+ <displayName>Metron Enrichment</displayName>
+ <category>MASTER</category>
+ <cardinality>1</cardinality>
+ <versionAdvertised>false</versionAdvertised>
+ <dependencies>
+ <dependency>
+ <name>HDFS/HDFS_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
+ <scope>cluster</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>KAFKA/KAFKA_BROKER</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ <commandScript>
+ <script>scripts/enrichment_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>METRON_ENRICHMENT_MYSQL_SERVER</name>
+ <displayName>MySQL Server</displayName>
+ <category>MASTER</category>
+ <cardinality>1</cardinality>
+ <versionAdvertised>false</versionAdvertised>
+ <clientsToUpdateConfigs></clientsToUpdateConfigs>
+ <commandScript>
+ <script>scripts/mysql_server.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ <customCommands>
+ <customCommand>
+ <name>CLEAN</name>
+ <commandScript>
+ <script>scripts/mysql_server.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </customCommand>
+ </customCommands>
+ </component>
+
+ <component>
+ <name>METRON_INDEXING</name>
+ <displayName>Metron Indexing</displayName>
+ <category>MASTER</category>
+ <cardinality>1</cardinality>
+ <versionAdvertised>false</versionAdvertised>
+ <reassignAllowed>false</reassignAllowed>
+ <clientsToUpdateConfigs></clientsToUpdateConfigs>
+ <dependencies>
+ <dependency>
+ <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
+ <scope>cluster</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>STORM/SUPERVISOR</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>KAFKA/KAFKA_BROKER</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ <commandScript>
+ <script>scripts/indexing_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ <customCommands>
+ <customCommand>
+ <name>ELASTICSEARCH_TEMPLATE_INSTALL</name>
+ <commandScript>
+ <script>scripts/indexing_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </customCommand>
+ <customCommand>
+ <name>ELASTICSEARCH_TEMPLATE_DELETE</name>
+ <commandScript>
+ <script>scripts/indexing_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </customCommand>
+ </customCommands>
+ </component>
+ </components>
+
+ <themes>
+ <theme>
+ <fileName>metron_theme.json</fileName>
+ <default>true</default>
+ </theme>
+ </themes>
+
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>any</osFamily>
+ <packages>
+ <package>
+ <name>metron-common</name>
+ </package>
+ <package>
+ <name>metron-data-management</name>
+ </package>
+ <package>
+ <name>metron-parsers</name>
+ </package>
+ <package>
+ <name>metron-enrichment</name>
+ </package>
+ <package>
+ <name>metron-indexing</name>
+ </package>
+ <package>
+ <name>metron-elasticsearch</name>
+ </package>
+ <package>
+ <name>metron-pcap</name>
+ </package>
+ </packages>
+ </osSpecific>
+ <osSpecific>
+ <osFamily>redhat6,redhat7</osFamily>
+ <packages>
+ <package>
+ <name>expect</name>
+ <skipUpgrade>true</skipUpgrade>
+ </package>
+ </packages>
+ </osSpecific>
+ <osSpecific>
+ <osFamily>redhat7</osFamily>
+ <packages>
+ <package>
+ <name>mysql-community-release</name>
+ <skipUpgrade>true</skipUpgrade>
+ </package>
+ <package>
+ <name>mysql-community-server</name>
+ <skipUpgrade>true</skipUpgrade>
+ </package>
+ <package>MySQL-python</package>
+ </packages>
+ </osSpecific>
+ <osSpecific>
+ <osFamily>redhat6</osFamily>
+ <packages>
+ <package>
+ <name>mysql-server</name>
+ <skipUpgrade>true</skipUpgrade>
+ </package>
+ <package>
+ <name>mysql</name>
+ <skipUpgrade>true</skipUpgrade>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+ <commandScript>
+ <script>scripts/service_check.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>300</timeout>
+ </commandScript>
+
+ <requiredServices>
+ <service>HDFS</service>
+ <service>KAFKA</service>
+ <service>STORM</service>
+ <service>ZOOKEEPER</service>
+ <service>HBASE</service>
+ </requiredServices>
+
+ <configuration-dependencies>
+ <config-type>metron-env</config-type>
+ <config-type>storm-env</config-type>
+ <config-type>storm-site</config-type>
+ <config-type>kafka-broker</config-type>
+ <config-type>kafka-env</config-type>
+ </configuration-dependencies>
+ <restartRequiredAfterChange>true</restartRequiredAfterChange>
+ <quickLinksConfigurations>
+ <quickLinksConfiguration>
+ <fileName>quicklinks.json</fileName>
+ <default>true</default>
+ </quickLinksConfiguration>
+ </quickLinksConfigurations>
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/addMysqlUser.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/addMysqlUser.sh b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/addMysqlUser.sh
new file mode 100755
index 0000000..8a6a788
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/addMysqlUser.sh
@@ -0,0 +1,51 @@
+#!/bin/sh
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+mysqldbuser=$1
+mysqldbpasswd=$2
+mysqldbhost=$3
+mysqlqdminpassword=$4
+myhostname=$(hostname -f)
+
+
+echo "Adding user ${mysqldbuser}@${mysqldbhost} and ${mysqldbuser}@localhost"
+expect <<EOF
+log_user 0
+# start mysql process using password prompt
+spawn mysql -u root -p
+expect "password:"
+send "${mysqlqdminpassword}\r"
+# echo all output until the end
+expect "mysql>"
+send "CREATE USER '${mysqldbuser}'@'${mysqldbhost}' IDENTIFIED BY '${mysqldbpasswd}';\r"
+expect "mysql>"
+send "CREATE USER '${mysqldbuser}'@'localhost' IDENTIFIED BY '${mysqldbpasswd}';\r"
+expect "mysql>"
+send "GRANT ALL PRIVILEGES ON GEO.* TO '${mysqldbuser}'@'%' IDENTIFIED BY '${mysqldbpasswd}';\r"
+log_user 1
+expect "mysql>"
+send "GRANT ALL PRIVILEGES ON GEO.* TO '${mysqldbuser}'@'${mysqldbhost}';\r"
+expect "mysql>"
+send "GRANT ALL PRIVILEGES ON GEO.* TO '${mysqldbuser}'@'localhost';\r"
+expect "mysql>"
+send "flush privileges;\r"
+send "\q"
+EOF
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/bro_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/bro_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/bro_index.template
new file mode 100644
index 0000000..951d967
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/bro_index.template
@@ -0,0 +1,217 @@
+{
+ "template": "bro_index*",
+ "mappings": {
+ "bro_doc": {
+ "_timestamp": {
+ "enabled": true
+ },
+ "dynamic_templates": [
+ {
+ "geo_location_point": {
+ "match": "enrichments:geo:*:location_point",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "geo_point"
+ }
+ }
+ },
+ {
+ "geo_country": {
+ "match": "enrichments:geo:*:country",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_city": {
+ "match": "enrichments:geo:*:city",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_location_id": {
+ "match": "enrichments:geo:*:locID",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_dma_code": {
+ "match": "enrichments:geo:*:dmaCode",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_postal_code": {
+ "match": "enrichments:geo:*:postalCode",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_latitude": {
+ "match": "enrichments:geo:*:latitude",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "float"
+ }
+ }
+ },
+ {
+ "geo_longitude": {
+ "match": "enrichments:geo:*:longitude",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "float"
+ }
+ }
+ },
+ {
+ "timestamps": {
+ "match": "*:ts",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "date",
+ "format": "epoch_millis"
+ }
+ }
+ }
+ ],
+ "properties": {
+ "timestamp": {
+ "type": "date",
+ "format": "epoch_millis"
+ },
+ "source:type": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "ip_dst_addr": {
+ "type": "ip"
+ },
+ "ip_dst_port": {
+ "type": "integer"
+ },
+ "ip_src_addr": {
+ "type": "ip"
+ },
+ "ip_src_port": {
+ "type": "integer"
+ },
+ "status_code": {
+ "type": "integer"
+ },
+ "method": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "protocol": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "request_body_len": {
+ "type": "integer"
+ },
+ "uri": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "uid": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "referrer": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "trans_depth": {
+ "type": "integer"
+ },
+ "host": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "status_msg": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "response_body_len": {
+ "type": "integer"
+ },
+ "user_agent": {
+ "type": "string"
+ },
+ "query": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "answers": {
+ "type": "ip"
+ },
+ "AA": {
+ "type": "boolean"
+ },
+ "TC": {
+ "type": "boolean"
+ },
+ "RA": {
+ "type": "boolean"
+ },
+ "RD": {
+ "type": "boolean"
+ },
+ "rejected": {
+ "type": "boolean"
+ },
+ "qclass_name": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "proto": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "rcode": {
+ "type": "integer"
+ },
+ "rcode_name": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "trans_id": {
+ "type": "integer"
+ },
+ "Z": {
+ "type": "integer"
+ },
+ "qclass": {
+ "type": "integer"
+ },
+ "qtype": {
+ "type": "integer"
+ },
+ "qtype_name": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/createMysqlGeoIp.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/createMysqlGeoIp.sh b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/createMysqlGeoIp.sh
new file mode 100755
index 0000000..40f11e1
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/createMysqlGeoIp.sh
@@ -0,0 +1,53 @@
+#!/bin/sh
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+from ../scripts/params import params
+
+geoipscript=$1
+geoipurl=$2
+mysqlqdminpassword=$3
+
+# Download and extract the actual GeoIP files
+mkdir -p /tmp/geoip
+mkdir -p /var/lib/mysql-files/
+
+# Stage the GeoIP data
+pushd /tmp/geoip
+curl -O ${geoipurl}
+tar xf GeoLiteCity-latest.tar.xz
+cp /tmp/geoip/*/*.csv /var/lib/mysql-files/
+popd
+
+# Load MySQL with the GeoIP data
+expect <<EOF
+log_user 0
+# start mysql process using password prompt
+spawn mysql -u root -p
+expect "password:"
+send "${mysqlqdminpassword}\r"
+# echo all output until the end
+log_user 1
+expect "mysql>"
+send "source ${geoipscript}\r"
+expect "mysql>"
+send "show databases;\r"
+send "\q"
+EOF
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/removeMysqlUser.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/removeMysqlUser.sh b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/removeMysqlUser.sh
new file mode 100755
index 0000000..3fc7afb
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/removeMysqlUser.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+
+mysqldbuser=$1
+userhost=$2
+mysqlqdminpassword=$3
+myhostname=$(hostname -f)
+sudo_prefix="/var/lib/ambari-agent/ambari-sudo.sh -H -E"
+
+echo "Removing user $mysqldbuser@$userhost"
+expect <<EOF
+log_user 0
+# start mysql process using password prompt
+spawn mysql -u root -p
+expect "password:"
+send "${mysqlqdminpassword}\r"
+# echo all output until the end
+log_user 1
+expect "mysql>"
+send "DROP USER '${mysqldbuser}'@'${userhost}';\r"
+expect "mysql>"
+send "flush privileges;;\r"
+send "\q"
+EOF
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/snort_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/snort_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/snort_index.template
new file mode 100644
index 0000000..bf943df
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/snort_index.template
@@ -0,0 +1,183 @@
+{
+ "template": "snort_index*",
+ "mappings": {
+ "snort_doc": {
+ "_timestamp": {
+ "enabled": true
+ },
+ "dynamic_templates": [
+ {
+ "geo_location_point": {
+ "match": "enrichments:geo:*:location_point",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "geo_point"
+ }
+ }
+ },
+ {
+ "geo_country": {
+ "match": "enrichments:geo:*:country",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_city": {
+ "match": "enrichments:geo:*:city",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_location_id": {
+ "match": "enrichments:geo:*:locID",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_dma_code": {
+ "match": "enrichments:geo:*:dmaCode",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_postal_code": {
+ "match": "enrichments:geo:*:postalCode",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_latitude": {
+ "match": "enrichments:geo:*:latitude",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "float"
+ }
+ }
+ },
+ {
+ "geo_longitude": {
+ "match": "enrichments:geo:*:longitude",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "float"
+ }
+ }
+ },
+ {
+ "timestamps": {
+ "match": "*:ts",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "date",
+ "format": "epoch_millis"
+ }
+ }
+ }
+ ],
+ "properties": {
+ "timestamp": {
+ "type": "date",
+ "format": "epoch_millis"
+ },
+ "source:type": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "ip_dst_addr": {
+ "type": "ip"
+ },
+ "ip_dst_port": {
+ "type": "integer"
+ },
+ "ip_src_addr": {
+ "type": "ip"
+ },
+ "ip_src_port": {
+ "type": "integer"
+ },
+ "dgmlen": {
+ "type": "integer"
+ },
+ "ethdst": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "ethlen": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "ethsrc": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "iplen": {
+ "type": "integer"
+ },
+ "is_alert": {
+ "type": "boolean"
+ },
+ "msg": {
+ "type": "string"
+ },
+ "protocol": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "sig_generator": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "sig_id": {
+ "type": "integer"
+ },
+ "sig_rev": {
+ "type": "string"
+ },
+ "tcpack": {
+ "type": "string"
+ },
+ "tcpflags": {
+ "type": "string"
+ },
+ "tcpseq": {
+ "type": "string"
+ },
+ "tcpwindow": {
+ "type": "string"
+ },
+ "threat:triage:level": {
+ "type": "double"
+ },
+ "tos": {
+ "type": "integer"
+ },
+ "ttl": {
+ "type": "integer"
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/yaf_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/yaf_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/yaf_index.template
new file mode 100644
index 0000000..7743afc
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/files/yaf_index.template
@@ -0,0 +1,205 @@
+{
+ "template": "yaf_index*",
+ "mappings": {
+ "yaf_doc": {
+ "_timestamp": {
+ "enabled": true
+ },
+ "dynamic_templates": [
+ {
+ "geo_location_point": {
+ "match": "enrichments:geo:*:location_point",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "geo_point"
+ }
+ }
+ },
+ {
+ "geo_country": {
+ "match": "enrichments:geo:*:country",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_city": {
+ "match": "enrichments:geo:*:city",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_location_id": {
+ "match": "enrichments:geo:*:locID",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_dma_code": {
+ "match": "enrichments:geo:*:dmaCode",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_postal_code": {
+ "match": "enrichments:geo:*:postalCode",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_latitude": {
+ "match": "enrichments:geo:*:latitude",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "float"
+ }
+ }
+ },
+ {
+ "geo_longitude": {
+ "match": "enrichments:geo:*:longitude",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "float"
+ }
+ }
+ },
+ {
+ "timestamps": {
+ "match": "*:ts",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "date",
+ "format": "epoch_millis"
+ }
+ }
+ }
+ ],
+ "properties": {
+ "timestamp": {
+ "type": "date",
+ "format": "epoch_millis"
+ },
+ "source:type": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "ip_dst_addr": {
+ "type": "ip"
+ },
+ "ip_dst_port": {
+ "type": "integer"
+ },
+ "ip_src_addr": {
+ "type": "ip"
+ },
+ "ip_src_port": {
+ "type": "integer"
+ },
+ "start_time": {
+ "type": "date",
+ "format": "epoch_millis"
+ },
+ "end_time": {
+ "type": "date",
+ "format": "epoch_millis"
+ },
+ "duration": {
+ "type": "double"
+ },
+ "rtt": {
+ "type": "double"
+ },
+ "proto": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "sip": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "sp": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "dip": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "dp": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "iflags": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "uflags": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "riflags": {
+ "type": "string"
+ },
+ "ruflags": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "isn": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "risn": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "tag": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "rtag": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "pkt": {
+ "type": "integer"
+ },
+ "oct": {
+ "type": "integer"
+ },
+ "rpkt": {
+ "type": "integer"
+ },
+ "roct": {
+ "type": "integer"
+ },
+ "app": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "end-reason": {
+ "type": "string"
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/enrichment_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/enrichment_commands.py
new file mode 100755
index 0000000..f43073d
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/enrichment_commands.py
@@ -0,0 +1,177 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import os
+import time
+
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute, File
+from resource_management.libraries.resources.hdfs_resource import HdfsResource
+
+import metron_service
+
+
+# Wrap major operations and functionality in this class
+class EnrichmentCommands:
+ __params = None
+ __enrichment_topology = None
+ __enrichment_topic = None
+ __configured = False
+
+ def __init__(self, params):
+ if params is None:
+ raise ValueError("params argument is required for initialization")
+ self.__params = params
+ self.__enrichment_topology = params.metron_enrichment_topology
+ self.__enrichment_topic = params.metron_enrichment_topic
+ self.__configured = os.path.isfile(self.__params.enrichment_configured_flag_file)
+
+ def is_configured(self):
+ return self.__configured
+
+ def set_configured(self):
+ File(self.__params.enrichment_configured_flag_file,
+ content="",
+ owner=self.__params.metron_user,
+ mode=0775)
+
+ def setup_repo(self):
+
+ def local_repo():
+ Logger.info("Setting up local repo")
+ Execute("yum -y install createrepo")
+ Execute("createrepo /localrepo")
+ Execute("chmod -R o-w+r /localrepo")
+
+ def remote_repo():
+ Logger.info('Using remote repo')
+
+ yum_repo_types = {
+ 'local': local_repo,
+ 'remote': remote_repo
+ }
+
+ repo_type = self.__params.yum_repo_type
+
+ if repo_type in yum_repo_types:
+ yum_repo_types[repo_type]()
+ Logger.info("Writing out repo file")
+ repo_template = ("echo \"[METRON-0.3.0]\n"
+ "name=Metron 0.3.0 packages\n"
+ "baseurl={0}\n"
+ "gpgcheck=0\n"
+ "enabled=1\n\""
+ " > /etc/yum.repos.d/metron.repo")
+ Execute(repo_template.format(self.__params.repo_url))
+ else:
+ raise ValueError("Unsupported repo type '{0}'".format(repo_type))
+
+ def init_kafka_topics(self):
+ Logger.info('Creating Kafka topics')
+ command_template = """{0}/kafka-topics.sh \
+ --zookeeper {1} \
+ --create \
+ --topic {2} \
+ --partitions {3} \
+ --replication-factor {4} \
+ --config retention.bytes={5}"""
+ num_partitions = 1
+ replication_factor = 1
+ retention_gigabytes = int(self.__params.metron_topic_retention)
+ retention_bytes = retention_gigabytes * 1024 * 1024 * 1024
+ Logger.info("Creating topics for enrichment")
+
+ Logger.info("Creating topic'{0}'".format(self.__enrichment_topic))
+ Execute(command_template.format(self.__params.kafka_bin_dir,
+ self.__params.zookeeper_quorum,
+ self.__enrichment_topic,
+ num_partitions,
+ replication_factor,
+ retention_bytes))
+ Logger.info("Done creating Kafka topics")
+
+ def init_hdfs_dir(self):
+ self.__params.HdfsResource(self.__params.metron_apps_enrichment_dir,
+ type="directory",
+ action="create_on_execute",
+ owner=self.__params.metron_user,
+ group=self.__params.user_group,
+ mode=0775,
+ )
+
+ def start_enrichment_topology(self):
+ Logger.info("Starting Metron enrichment topology: {0}".format(self.__enrichment_topology))
+ start_cmd_template = """{0}/bin/start_enrichment_topology.sh \
+ -s {1} \
+ -z {2}"""
+ Logger.info('Starting ' + self.__enrichment_topology)
+ Execute(start_cmd_template.format(self.__params.metron_home, self.__enrichment_topology, self.__params.zookeeper_quorum))
+
+ Logger.info('Finished starting enrichment topology')
+
+ def stop_enrichment_topology(self):
+ Logger.info('Stopping ' + self.__enrichment_topology)
+ stop_cmd = 'storm kill ' + self.__enrichment_topology
+ Execute(stop_cmd)
+ Logger.info('Done stopping enrichment topologies')
+
+ def restart_enrichment_topology(self, env):
+ Logger.info('Restarting the enrichment topologies')
+ self.stop_enrichment_topology()
+
+ # Wait for old topology to be cleaned up by Storm, before starting again.
+ retries = 0
+ topology_active = self.is_topology_active(env)
+ while topology_active and retries < 3:
+ Logger.info('Existing topology still active. Will wait and retry')
+ time.sleep(40)
+ topology_active = self.is_topology_active(env)
+ retries += 1
+
+ if not topology_active:
+ self.start_enrichment_topology()
+ Logger.info('Done restarting the enrichment topology')
+ else:
+ Logger.warning('Retries exhausted. Existing topology not cleaned up. Aborting topology start.')
+
+ def is_topology_active(self, env):
+ env.set_params(self.__params)
+
+ active = True
+ topologies = metron_service.get_running_topologies()
+ is_running = False
+ if self.__enrichment_topology in topologies:
+ is_running = topologies[self.__enrichment_topology] in ['ACTIVE', 'REBALANCING']
+ active &= is_running
+ return active
+
+ def create_hbase_tables(self):
+ add_enrichment_cmd = "echo \"create '{0}','{1}'\" | hbase shell -n".format(self.__params.enrichment_table, self.__params.enrichment_cf)
+ Execute(add_enrichment_cmd,
+ tries=3,
+ try_sleep=5,
+ logoutput=False,
+ path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'
+ )
+
+ add_threatintel_cmd = "echo \"create '{0}','{1}'\" | hbase shell -n".format(self.__params.threatintel_table, self.__params.threatintel_cf)
+ Execute(add_threatintel_cmd,
+ tries=3,
+ try_sleep=5,
+ logoutput=False,
+ path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'
+ )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/enrichment_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/enrichment_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/enrichment_master.py
new file mode 100755
index 0000000..97ace20
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/enrichment_master.py
@@ -0,0 +1,87 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import File
+from resource_management.core.source import Template
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.script import Script
+
+from enrichment_commands import EnrichmentCommands
+
+
+class Enrichment(Script):
+ def install(self, env):
+ from params import params
+ env.set_params(params)
+ commands = EnrichmentCommands(params)
+ commands.setup_repo()
+ self.install_packages(env)
+ self.configure(env)
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ from params import params
+ env.set_params(params)
+
+ File(format("{metron_config_path}/enrichment.properties"),
+ content=Template("enrichment.properties.j2"),
+ owner=params.metron_user,
+ group=params.metron_group
+ )
+
+ def start(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ commands = EnrichmentCommands(params)
+
+ if not commands.is_configured():
+ commands.init_kafka_topics()
+ commands.create_hbase_tables()
+ commands.init_hdfs_dir()
+ commands.set_configured()
+
+ commands.start_enrichment_topology()
+
+ def stop(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ commands = EnrichmentCommands(params)
+ commands.stop_enrichment_topology()
+
+ def status(self, env):
+ from params import status_params
+ env.set_params(status_params)
+ commands = EnrichmentCommands(status_params)
+
+ if not commands.is_topology_active(env):
+ raise ComponentIsNotRunning()
+
+ def restart(self, env):
+ from params import params
+ env.set_params(params)
+ commands = EnrichmentCommands(params)
+ commands.restart_enrichment_topology(env)
+
+ def kafkabuild(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ commands = EnrichmentCommands(params)
+ commands.init_kafka_topics()
+
+
+if __name__ == "__main__":
+ Enrichment().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/indexing_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/indexing_commands.py
new file mode 100755
index 0000000..165a4ae
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/indexing_commands.py
@@ -0,0 +1,142 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import os
+import time
+
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute, File
+
+import metron_service
+
+
+# Wrap major operations and functionality in this class
+class IndexingCommands:
+ __params = None
+ __indexing = None
+ __configured = False
+
+ def __init__(self, params):
+ if params is None:
+ raise ValueError("params argument is required for initialization")
+ self.__params = params
+ self.__indexing = params.metron_indexing_topology
+ self.__configured = os.path.isfile(self.__params.indexing_configured_flag_file)
+
+ def is_configured(self):
+ return self.__configured
+
+ def set_configured(self):
+ File(self.__params.indexing_configured_flag_file,
+ content="",
+ owner=self.__params.metron_user,
+ mode=0775)
+
+ def setup_repo(self):
+ def local_repo():
+ Logger.info("Setting up local repo")
+ Execute("yum -y install createrepo")
+ Execute("createrepo /localrepo")
+ Execute("chmod -R o-w+r /localrepo")
+ Execute("echo \"[METRON-0.3.0]\n"
+ "name=Metron 0.3.0 packages\n"
+ "baseurl=file:///localrepo\n"
+ "gpgcheck=0\n"
+ "enabled=1\" > /etc/yum.repos.d/local.repo")
+
+ def remote_repo():
+ print('Using remote repo')
+
+ yum_repo_types = {
+ 'local': local_repo,
+ 'remote': remote_repo
+ }
+ repo_type = self.__params.yum_repo_type
+ if repo_type in yum_repo_types:
+ yum_repo_types[repo_type]()
+ else:
+ raise ValueError("Unsupported repo type '{0}'".format(repo_type))
+
+ def init_kafka_topics(self):
+ Logger.info('Creating Kafka topics')
+ command_template = """{0}/kafka-topics.sh \
+ --zookeeper {1} \
+ --create \
+ --topic {2} \
+ --partitions {3} \
+ --replication-factor {4} \
+ --config retention.bytes={5}"""
+ num_partitions = 1
+ replication_factor = 1
+ retention_gigabytes = int(self.__params.metron_topic_retention)
+ retention_bytes = retention_gigabytes * 1024 * 1024 * 1024
+ Logger.info("Creating topics for indexing")
+
+ Logger.info("Creating topic'{0}'".format(self.__indexing))
+ Execute(command_template.format(self.__params.kafka_bin_dir,
+ self.__params.zookeeper_quorum,
+ self.__indexing,
+ num_partitions,
+ replication_factor,
+ retention_bytes))
+ Logger.info("Done creating Kafka topics")
+
+ def start_indexing_topology(self):
+ Logger.info("Starting Metron indexing topology: {0}".format(self.__indexing))
+ start_cmd_template = """{0}/bin/start_elasticsearch_topology.sh \
+ -s {1} \
+ -z {2}"""
+ Logger.info('Starting ' + self.__indexing)
+ Execute(start_cmd_template.format(self.__params.metron_home, self.__indexing, self.__params.zookeeper_quorum))
+
+ Logger.info('Finished starting indexing topology')
+
+ def stop_indexing_topology(self):
+ Logger.info('Stopping ' + self.__indexing)
+ stop_cmd = 'storm kill ' + self.__indexing
+ Execute(stop_cmd)
+ Logger.info('Done stopping indexing topologies')
+
+ def restart_indexing_topology(self, env):
+ Logger.info('Restarting the indexing topologies')
+ self.stop_indexing_topology()
+
+ # Wait for old topology to be cleaned up by Storm, before starting again.
+ retries = 0
+ topology_active = self.is_topology_active(env)
+ while self.is_topology_active(env) and retries < 3:
+ Logger.info('Existing topology still active. Will wait and retry')
+ time.sleep(10)
+ retries += 1
+
+ if not topology_active:
+ Logger.info('Waiting for storm kill to complete')
+ time.sleep(30)
+ self.start_indexing_topology()
+ Logger.info('Done restarting the indexing topologies')
+ else:
+ Logger.warning('Retries exhausted. Existing topology not cleaned up. Aborting topology start.')
+
+ def is_topology_active(self, env):
+ env.set_params(self.__params)
+ active = True
+ topologies = metron_service.get_running_topologies()
+ is_running = False
+ if self.__indexing in topologies:
+ is_running = topologies[self.__indexing] in ['ACTIVE', 'REBALANCING']
+ active &= is_running
+ return active
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/indexing_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/indexing_master.py
new file mode 100755
index 0000000..e9efa03
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/indexing_master.py
@@ -0,0 +1,119 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute
+from resource_management.core.resources.system import File
+from resource_management.core.source import StaticFile
+from resource_management.libraries.functions import format as ambari_format
+from resource_management.libraries.script import Script
+
+import metron_service
+from indexing_commands import IndexingCommands
+
+
+class Indexing(Script):
+ __configured = False
+
+ def install(self, env):
+ from params import params
+ env.set_params(params)
+ commands = IndexingCommands(params)
+ commands.setup_repo()
+ self.install_packages(env)
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ from params import params
+ env.set_params(params)
+
+ commands = IndexingCommands(params)
+ metron_service.load_global_config(params)
+
+ if not commands.is_configured():
+ commands.init_kafka_topics()
+ commands.set_configured()
+
+ def start(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ self.configure(env)
+ commands = IndexingCommands(params)
+ commands.start_indexing_topology()
+
+ def stop(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ commands = IndexingCommands(params)
+ commands.stop_indexing_topology()
+
+ def status(self, env):
+ from params import status_params
+ env.set_params(status_params)
+ commands = IndexingCommands(status_params)
+ if not commands.is_topology_active(env):
+ raise ComponentIsNotRunning()
+
+ def restart(self, env):
+ from params import params
+ env.set_params(params)
+ self.configure(env)
+ commands = IndexingCommands(params)
+ commands.restart_indexing_topology(env)
+
+ def elasticsearch_template_install(self, env):
+ from params import params
+ env.set_params(params)
+
+ File(params.bro_index_path,
+ mode=0755,
+ content=StaticFile('bro_index.template')
+ )
+
+ File(params.snort_index_path,
+ mode=0755,
+ content=StaticFile('snort_index.template')
+ )
+
+ File(params.yaf_index_path,
+ mode=0755,
+ content=StaticFile('yaf_index.template')
+ )
+
+ bro_cmd = ambari_format(
+ 'curl -s -XPOST http://{es_http_url}/_template/bro_index -d @{bro_index_path}')
+ Execute(bro_cmd, logoutput=True)
+ snort_cmd = ambari_format(
+ 'curl -s -XPOST http://{es_http_url}/_template/snort_index -d @{snort_index_path}')
+ Execute(snort_cmd, logoutput=True)
+ yaf_cmd = ambari_format(
+ 'curl -s -XPOST http://{es_http_url}/_template/yaf_index -d @{yaf_index_path}')
+ Execute(yaf_cmd, logoutput=True)
+
+ def elasticsearch_template_delete(self, env):
+ from params import params
+ env.set_params(params)
+
+ bro_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/bro_index*"')
+ Execute(bro_cmd, logoutput=True)
+ snort_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/snort_index*"')
+ Execute(snort_cmd, logoutput=True)
+ yaf_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/yaf_index*"')
+ Execute(yaf_cmd, logoutput=True)
+
+
+if __name__ == "__main__":
+ Indexing().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/metron_service.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/metron_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/metron_service.py
new file mode 100644
index 0000000..57da2c7
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/metron_service.py
@@ -0,0 +1,76 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import json
+import subprocess
+
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Directory, File
+from resource_management.core.resources.system import Execute
+from resource_management.core.source import InlineTemplate
+from resource_management.libraries.functions import format as ambari_format
+
+
+def init_config():
+ Logger.info('Loading config into ZooKeeper')
+ Execute(ambari_format(
+ "{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_quorum}"),
+ path=ambari_format("{java_home}/bin")
+ )
+
+
+def get_running_topologies():
+ Logger.info('Getting Running Storm Topologies from Storm REST Server')
+
+ cmd = ambari_format('curl --max-time 3 {storm_rest_addr}/api/v1/topology/summary')
+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+ (stdout, stderr) = proc.communicate()
+
+ try:
+ stormjson = json.loads(stdout)
+ except ValueError:
+ return {}
+
+ topologiesDict = {}
+
+ for topology in stormjson['topologies']:
+ topologiesDict[topology['name']] = topology['status']
+
+ Logger.info("Topologies: " + str(topologiesDict))
+ return topologiesDict
+
+
+def load_global_config(params):
+ Logger.info('Create Metron Local Config Directory')
+ Logger.info("Configure Metron global.json")
+
+ directories = [params.metron_zookeeper_config_path]
+ Directory(directories,
+ mode=0755,
+ owner=params.metron_user,
+ group=params.metron_group
+ )
+
+ File("{0}/global.json".format(params.metron_zookeeper_config_path),
+ owner=params.metron_user,
+ content=InlineTemplate(params.global_json_template)
+ )
+
+ File("{0}/elasticsearch.properties".format(params.metron_zookeeper_config_path + '/..'),
+ owner=params.metron_user,
+ content=InlineTemplate(params.global_properties_template))
+
+ init_config()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/mysql_server.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/mysql_server.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/mysql_server.py
new file mode 100755
index 0000000..fcc83af
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/mysql_server.py
@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.libraries.script.script import Script
+
+import mysql_users
+from mysql_service import mysql_service
+from mysql_utils import mysql_configure
+from resource_management.core.resources.packaging import Package
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+
+class MysqlServer(Script):
+ def install(self, env):
+ self.install_packages(env)
+ self.configure(env)
+
+ def clean(self, env):
+ from params import params
+ env.set_params(params)
+ if params.install_mysql == 'Yes':
+ mysql_users.mysql_deluser()
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ from params import params
+ env.set_params(params)
+ mysql_configure()
+
+ def start(self, env, rolling_restart=False):
+ from params import params
+ env.set_params(params)
+ mysql_service(daemon_name=params.daemon_name, action='start')
+
+ def stop(self, env, rolling_restart=False):
+ from params import params
+ env.set_params(params)
+ mysql_service(daemon_name=params.daemon_name, action='stop')
+
+ def status(self, env):
+ from params import status_params
+ env.set_params(status_params)
+
+ mysql_service(daemon_name=status_params.daemon_name, action='status')
+
+
+if __name__ == "__main__":
+ MysqlServer().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/mysql_service.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/mysql_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/mysql_service.py
new file mode 100755
index 0000000..2e0ce8b
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/mysql_service.py
@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.exceptions import ComponentIsNotRunning, Fail
+from resource_management.core.resources.system import Execute
+from resource_management.libraries.functions.format import format
+
+
+def mysql_service(daemon_name=None, action='start'):
+ status_cmd = format("pgrep -l '^{mysql_process_name}$'")
+ cmd = ('service', daemon_name, action)
+
+ if action == 'status':
+ try:
+ Execute(status_cmd)
+ except Fail:
+ raise ComponentIsNotRunning()
+ elif action == 'stop':
+ Execute(cmd,
+ logoutput=True,
+ only_if=status_cmd,
+ sudo=True,
+ )
+ elif action == 'start':
+ Execute(cmd,
+ logoutput=True,
+ not_if=status_cmd,
+ sudo=True,
+ )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/mysql_users.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/mysql_users.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/mysql_users.py
new file mode 100755
index 0000000..b778e85
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/mysql_users.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.resources.system import Execute, File
+from resource_management.core.source import StaticFile
+from resource_management.libraries.functions.format import format
+
+
+# Used to add metron access to the needed components
+def mysql_adduser():
+ from params import params
+
+ File(params.mysql_adduser_path,
+ mode=0755,
+ content=StaticFile('addMysqlUser.sh')
+ )
+
+ add_user_cmd = format("bash -x {mysql_adduser_path} {metron_user} {enrichment_metron_user_passwd!p} {mysql_host} {mysql_admin_password!p}")
+ Execute(add_user_cmd,
+ tries=3,
+ try_sleep=5,
+ logoutput=False,
+ path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'
+ )
+
+
+# Removes hive metron from components
+def mysql_deluser():
+ from params import params
+
+ File(params.mysql_deluser_path,
+ mode=0755,
+ content=StaticFile('removeMysqlUser.sh')
+ )
+
+ del_user_cmd = format("bash -x {mysql_deluser_path} {metron_user} {mysql_host} {mysql_admin_password!p}")
+ Execute(del_user_cmd,
+ tries=3,
+ try_sleep=5,
+ path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
+ )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/mysql_utils.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/mysql_utils.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/mysql_utils.py
new file mode 100755
index 0000000..63cdb38
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/mysql_utils.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.resources.system import Execute, File
+from resource_management.core.source import StaticFile
+from resource_management.libraries.functions.format import format
+from mysql_service import mysql_service
+
+import mysql_users
+
+
+def mysql_configure():
+ from params import params
+
+ if params.install_mysql == 'Yes':
+ mysql_service(daemon_name=params.daemon_name, action='start')
+
+ replace_bind_address = ('sed', '-i', 's|^bind-address[ \t]*=.*|bind-address = 0.0.0.0|', params.mysql_configname)
+ Execute(replace_bind_address,
+ sudo=True,
+ )
+
+ mysql_users.mysql_adduser()
+
+ File(params.mysql_create_geoip_path,
+ mode=0755,
+ content=StaticFile('createMysqlGeoIp.sh')
+ )
+
+ geoip_setup_cmd = format("bash -x {mysql_create_geoip_path} {geoip_ddl} {geoip_url} {mysql_admin_password!p}")
+
+ Execute(geoip_setup_cmd,
+ tries=3,
+ try_sleep=5,
+ path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
+ )
+
+ if params.install_mysql == 'Yes':
+ mysql_service(daemon_name=params.daemon_name, action='stop')
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/__init__.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/__init__.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/__init__.py
new file mode 100755
index 0000000..242460e
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/__init__.py
@@ -0,0 +1,18 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/params.py
new file mode 100755
index 0000000..953435d
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/params.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from ambari_commons import OSCheck
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.expect import expect
+
+if OSCheck.is_windows_family():
+ from params_windows import *
+else:
+ from params_linux import *
+
+java_home = config['hostLevelParams']['java_home']
+java_version = expect("/hostLevelParams/java_version", int)
+
+host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/params_linux.py
new file mode 100755
index 0000000..c418ad9
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/params_linux.py
@@ -0,0 +1,186 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import functools
+import os
+
+from ambari_commons.os_check import OSCheck
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions import get_kinit_path
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
+from resource_management.libraries.functions.is_empty import is_empty
+from resource_management.libraries.resources.hdfs_resource import HdfsResource
+from resource_management.libraries.script import Script
+
+import status_params
+
+# server configurations
+config = Script.get_config()
+tmp_dir = Script.get_tmp_dir()
+
+hostname = config['hostname']
+user_group = config['configurations']['cluster-env']['user_group']
+metron_home = status_params.metron_home
+parsers = status_params.parsers
+metron_ddl_dir = metron_home + '/ddl'
+geoip_ddl = metron_ddl_dir + '/geoip_ddl.sql'
+geoip_url = config['configurations']['metron-env']['geoip_url']
+metron_indexing_topology = status_params.metron_indexing_topology
+metron_user = config['configurations']['metron-env']['metron_user']
+metron_group = config['configurations']['metron-env']['metron_group']
+metron_config_path = metron_home + '/config'
+metron_zookeeper_config_dir = status_params.metron_zookeeper_config_dir
+metron_zookeeper_config_path = status_params.metron_zookeeper_config_path
+parsers_configured_flag_file = status_params.parsers_configured_flag_file
+enrichment_configured_flag_file = status_params.enrichment_configured_flag_file
+indexing_configured_flag_file = status_params.indexing_configured_flag_file
+global_json_template = config['configurations']['metron-env']['global-json']
+global_properties_template = config['configurations']['metron-env']['elasticsearch-properties']
+
+# Elasticsearch hosts and port management
+es_cluster_name = config['configurations']['metron-env']['es_cluster_name']
+es_hosts = config['configurations']['metron-env']['es_hosts']
+es_host_list = es_hosts.split(",")
+es_binary_port = config['configurations']['metron-env']['es_binary_port']
+es_url = ",".join([host + ":" + es_binary_port for host in es_host_list])
+es_http_port = config['configurations']['metron-env']['es_http_port']
+es_http_url = es_host_list[0] + ":" + es_http_port
+
+# install repo
+yum_repo_type = config['configurations']['metron-env']['repo_type']
+if yum_repo_type == 'local':
+ repo_url = 'file:///localrepo'
+else:
+ repo_url = config['configurations']['metron-env']['repo_url']
+
+# hadoop params
+stack_root = Script.get_stack_root()
+hadoop_home_dir = stack_select.get_hadoop_dir("home")
+hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
+hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
+kafka_home = os.path.join(stack_root, "current", "kafka-broker")
+kafka_bin_dir = os.path.join(kafka_home, "bin")
+
+# zookeeper
+zk_hosts = default("/clusterHostInfo/zookeeper_hosts", [])
+has_zk_host = not len(zk_hosts) == 0
+zookeeper_quorum = None
+if has_zk_host:
+ if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']:
+ zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort']
+ else:
+ zookeeper_clientPort = '2181'
+ zookeeper_quorum = (':' + zookeeper_clientPort + ',').join(config['clusterHostInfo']['zookeeper_hosts'])
+ # last port config
+ zookeeper_quorum += ':' + zookeeper_clientPort
+
+# Storm
+storm_rest_addr = status_params.storm_rest_addr
+
+# Kafka
+kafka_hosts = default("/clusterHostInfo/kafka_broker_hosts", [])
+has_kafka_host = not len(kafka_hosts) == 0
+kafka_brokers = None
+if has_kafka_host:
+ if 'port' in config['configurations']['kafka-broker']:
+ kafka_broker_port = config['configurations']['kafka-broker']['port']
+ else:
+ kafka_broker_port = '6667'
+ kafka_brokers = (':' + kafka_broker_port + ',').join(config['clusterHostInfo']['kafka_broker_hosts'])
+ kafka_brokers += ':' + kafka_broker_port
+
+metron_apps_dir = config['configurations']['metron-env']['metron_apps_hdfs_dir']
+metron_apps_enrichment_dir = metron_apps_dir + '/enrichment'
+metron_topic_retention = config['configurations']['metron-env']['metron_topic_retention']
+
+local_grok_patterns_dir = format("{metron_home}/patterns")
+hdfs_grok_patterns_dir = format("{metron_apps_dir}/patterns")
+
+# for create_hdfs_directory
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
+smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
+kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
+hdfs_site = config['configurations']['hdfs-site']
+default_fs = config['configurations']['core-site']['fs.defaultFS']
+dfs_type = default("/commandParams/dfs_type", "")
+
+# MYSQL
+if OSCheck.is_ubuntu_family():
+ mysql_configname = '/etc/mysql/my.cnf'
+else:
+ mysql_configname = '/etc/my.cnf'
+
+daemon_name = status_params.daemon_name
+
+install_mysql = config['configurations']['metron-env']['install_mysql']
+mysql_admin_password = config['configurations']['metron-env']['mysql_admin_password']
+
+# There will always be exactly one mysql_host
+mysql_host = config['clusterHostInfo']['metron_enrichment_mysql_server_hosts'][0]
+
+mysql_port = config['configurations']['metron-env']['metron_enrichment_db_port']
+
+mysql_adduser_path = tmp_dir + "/addMysqlUser.sh"
+mysql_deluser_path = tmp_dir + "/removeMysqlUser.sh"
+mysql_create_geoip_path = tmp_dir + "/createMysqlGeoIp.sh"
+
+enrichment_metron_user = config['configurations']['metron-env']['metron_enrichment_db_user']
+enrichment_metron_user_passwd = config['configurations']['metron-env']['metron_enrichment_db_password']
+enrichment_metron_user_passwd = unicode(enrichment_metron_user_passwd) if not is_empty(
+ enrichment_metron_user_passwd) else enrichment_metron_user_passwd
+mysql_process_name = status_params.mysql_process_name
+
+# create partial functions with common arguments for every HdfsResource call
+# to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code
+HdfsResource = functools.partial(
+ HdfsResource,
+ user=hdfs_user,
+ hdfs_resource_ignore_file="/var/lib/ambari-agent/data/.hdfs_resource_ignore",
+ security_enabled=security_enabled,
+ keytab=hdfs_user_keytab,
+ kinit_path_local=kinit_path_local,
+ hadoop_bin_dir=hadoop_bin_dir,
+ hadoop_conf_dir=hadoop_conf_dir,
+ principal_name=hdfs_principal_name,
+ hdfs_site=hdfs_site,
+ default_fs=default_fs,
+ immutable_paths=get_not_managed_resources(),
+ dfs_type=dfs_type
+)
+
+# HBase
+enrichment_table = status_params.enrichment_table
+enrichment_cf = status_params.enrichment_cf
+threatintel_table = status_params.threatintel_table
+threatintel_cf = status_params.threatintel_cf
+
+metron_enrichment_topology = status_params.metron_enrichment_topology
+metron_enrichment_topic = status_params.metron_enrichment_topic
+
+# ES Templates
+bro_index_path = tmp_dir + "/bro_index.template"
+snort_index_path = tmp_dir + "/snort_index.template"
+yaf_index_path = tmp_dir + "/yaf_index.template"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/params_windows.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/params_windows.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/params_windows.py
new file mode 100755
index 0000000..4d11b35
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/params_windows.py
@@ -0,0 +1,20 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+raise NotImplementedError
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/status_params.py
new file mode 100644
index 0000000..196defb
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/package/scripts/params/status_params.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from ambari_commons import OSCheck
+from resource_management.libraries.functions import format
+from resource_management.libraries.script import Script
+
+config = Script.get_config()
+
+# Parsers
+parsers = config['configurations']['metron-env']['parsers']
+metron_home = config['configurations']['metron-env']['metron_home']
+metron_zookeeper_config_dir = config['configurations']['metron-env']['metron_zookeeper_config_dir']
+metron_zookeeper_config_path = format('{metron_home}/{metron_zookeeper_config_dir}')
+parsers_configured_flag_file = metron_zookeeper_config_path + '/../metron_parsers_configured'
+
+# Enrichment
+metron_enrichment_topology = 'enrichment'
+metron_enrichment_topic = 'enrichments'
+
+enrichment_table = 'enrichment'
+enrichment_cf = 't'
+threatintel_table = 'threatintel'
+threatintel_cf = 't'
+
+mysql_process_name = 'mysqld'
+if OSCheck.is_suse_family() or OSCheck.is_ubuntu_family():
+ daemon_name = 'mysql'
+else:
+ daemon_name = 'mysqld'
+
+# ing
+metron_indexing_topology = config['configurations']['metron-env']['metron_indexing_topology']
+indexing_configured_flag_file = metron_zookeeper_config_path + '/../metron_indexing_configured'
+
+# Enrichment
+enrichment_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_configured'
+
+# Storm
+storm_rest_addr = config['configurations']['metron-env']['storm_rest_addr']
[3/4] incubator-metron git commit: METRON-533 Up the version for
release closes apache/incubator-metron#353
Posted by le...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/indexing_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/indexing_master.py
deleted file mode 100755
index e9efa03..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/indexing_master.py
+++ /dev/null
@@ -1,119 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-
-from resource_management.core.exceptions import ComponentIsNotRunning
-from resource_management.core.logger import Logger
-from resource_management.core.resources.system import Execute
-from resource_management.core.resources.system import File
-from resource_management.core.source import StaticFile
-from resource_management.libraries.functions import format as ambari_format
-from resource_management.libraries.script import Script
-
-import metron_service
-from indexing_commands import IndexingCommands
-
-
-class Indexing(Script):
- __configured = False
-
- def install(self, env):
- from params import params
- env.set_params(params)
- commands = IndexingCommands(params)
- commands.setup_repo()
- self.install_packages(env)
-
- def configure(self, env, upgrade_type=None, config_dir=None):
- from params import params
- env.set_params(params)
-
- commands = IndexingCommands(params)
- metron_service.load_global_config(params)
-
- if not commands.is_configured():
- commands.init_kafka_topics()
- commands.set_configured()
-
- def start(self, env, upgrade_type=None):
- from params import params
- env.set_params(params)
- self.configure(env)
- commands = IndexingCommands(params)
- commands.start_indexing_topology()
-
- def stop(self, env, upgrade_type=None):
- from params import params
- env.set_params(params)
- commands = IndexingCommands(params)
- commands.stop_indexing_topology()
-
- def status(self, env):
- from params import status_params
- env.set_params(status_params)
- commands = IndexingCommands(status_params)
- if not commands.is_topology_active(env):
- raise ComponentIsNotRunning()
-
- def restart(self, env):
- from params import params
- env.set_params(params)
- self.configure(env)
- commands = IndexingCommands(params)
- commands.restart_indexing_topology(env)
-
- def elasticsearch_template_install(self, env):
- from params import params
- env.set_params(params)
-
- File(params.bro_index_path,
- mode=0755,
- content=StaticFile('bro_index.template')
- )
-
- File(params.snort_index_path,
- mode=0755,
- content=StaticFile('snort_index.template')
- )
-
- File(params.yaf_index_path,
- mode=0755,
- content=StaticFile('yaf_index.template')
- )
-
- bro_cmd = ambari_format(
- 'curl -s -XPOST http://{es_http_url}/_template/bro_index -d @{bro_index_path}')
- Execute(bro_cmd, logoutput=True)
- snort_cmd = ambari_format(
- 'curl -s -XPOST http://{es_http_url}/_template/snort_index -d @{snort_index_path}')
- Execute(snort_cmd, logoutput=True)
- yaf_cmd = ambari_format(
- 'curl -s -XPOST http://{es_http_url}/_template/yaf_index -d @{yaf_index_path}')
- Execute(yaf_cmd, logoutput=True)
-
- def elasticsearch_template_delete(self, env):
- from params import params
- env.set_params(params)
-
- bro_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/bro_index*"')
- Execute(bro_cmd, logoutput=True)
- snort_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/snort_index*"')
- Execute(snort_cmd, logoutput=True)
- yaf_cmd = ambari_format('curl -s -XDELETE "http://{es_http_url}/yaf_index*"')
- Execute(yaf_cmd, logoutput=True)
-
-
-if __name__ == "__main__":
- Indexing().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/metron_service.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/metron_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/metron_service.py
deleted file mode 100644
index 57da2c7..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/metron_service.py
+++ /dev/null
@@ -1,76 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-
-import json
-import subprocess
-
-from resource_management.core.logger import Logger
-from resource_management.core.resources.system import Directory, File
-from resource_management.core.resources.system import Execute
-from resource_management.core.source import InlineTemplate
-from resource_management.libraries.functions import format as ambari_format
-
-
-def init_config():
- Logger.info('Loading config into ZooKeeper')
- Execute(ambari_format(
- "{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_quorum}"),
- path=ambari_format("{java_home}/bin")
- )
-
-
-def get_running_topologies():
- Logger.info('Getting Running Storm Topologies from Storm REST Server')
-
- cmd = ambari_format('curl --max-time 3 {storm_rest_addr}/api/v1/topology/summary')
- proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
- (stdout, stderr) = proc.communicate()
-
- try:
- stormjson = json.loads(stdout)
- except ValueError:
- return {}
-
- topologiesDict = {}
-
- for topology in stormjson['topologies']:
- topologiesDict[topology['name']] = topology['status']
-
- Logger.info("Topologies: " + str(topologiesDict))
- return topologiesDict
-
-
-def load_global_config(params):
- Logger.info('Create Metron Local Config Directory')
- Logger.info("Configure Metron global.json")
-
- directories = [params.metron_zookeeper_config_path]
- Directory(directories,
- mode=0755,
- owner=params.metron_user,
- group=params.metron_group
- )
-
- File("{0}/global.json".format(params.metron_zookeeper_config_path),
- owner=params.metron_user,
- content=InlineTemplate(params.global_json_template)
- )
-
- File("{0}/elasticsearch.properties".format(params.metron_zookeeper_config_path + '/..'),
- owner=params.metron_user,
- content=InlineTemplate(params.global_properties_template))
-
- init_config()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_server.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_server.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_server.py
deleted file mode 100755
index fcc83af..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_server.py
+++ /dev/null
@@ -1,63 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.libraries.script.script import Script
-
-import mysql_users
-from mysql_service import mysql_service
-from mysql_utils import mysql_configure
-from resource_management.core.resources.packaging import Package
-from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
-
-class MysqlServer(Script):
- def install(self, env):
- self.install_packages(env)
- self.configure(env)
-
- def clean(self, env):
- from params import params
- env.set_params(params)
- if params.install_mysql == 'Yes':
- mysql_users.mysql_deluser()
-
- def configure(self, env, upgrade_type=None, config_dir=None):
- from params import params
- env.set_params(params)
- mysql_configure()
-
- def start(self, env, rolling_restart=False):
- from params import params
- env.set_params(params)
- mysql_service(daemon_name=params.daemon_name, action='start')
-
- def stop(self, env, rolling_restart=False):
- from params import params
- env.set_params(params)
- mysql_service(daemon_name=params.daemon_name, action='stop')
-
- def status(self, env):
- from params import status_params
- env.set_params(status_params)
-
- mysql_service(daemon_name=status_params.daemon_name, action='status')
-
-
-if __name__ == "__main__":
- MysqlServer().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_service.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_service.py
deleted file mode 100755
index 2e0ce8b..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_service.py
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.core.exceptions import ComponentIsNotRunning, Fail
-from resource_management.core.resources.system import Execute
-from resource_management.libraries.functions.format import format
-
-
-def mysql_service(daemon_name=None, action='start'):
- status_cmd = format("pgrep -l '^{mysql_process_name}$'")
- cmd = ('service', daemon_name, action)
-
- if action == 'status':
- try:
- Execute(status_cmd)
- except Fail:
- raise ComponentIsNotRunning()
- elif action == 'stop':
- Execute(cmd,
- logoutput=True,
- only_if=status_cmd,
- sudo=True,
- )
- elif action == 'start':
- Execute(cmd,
- logoutput=True,
- not_if=status_cmd,
- sudo=True,
- )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_users.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_users.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_users.py
deleted file mode 100755
index b778e85..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_users.py
+++ /dev/null
@@ -1,58 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.core.resources.system import Execute, File
-from resource_management.core.source import StaticFile
-from resource_management.libraries.functions.format import format
-
-
-# Used to add metron access to the needed components
-def mysql_adduser():
- from params import params
-
- File(params.mysql_adduser_path,
- mode=0755,
- content=StaticFile('addMysqlUser.sh')
- )
-
- add_user_cmd = format("bash -x {mysql_adduser_path} {metron_user} {enrichment_metron_user_passwd!p} {mysql_host} {mysql_admin_password!p}")
- Execute(add_user_cmd,
- tries=3,
- try_sleep=5,
- logoutput=False,
- path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'
- )
-
-
-# Removes hive metron from components
-def mysql_deluser():
- from params import params
-
- File(params.mysql_deluser_path,
- mode=0755,
- content=StaticFile('removeMysqlUser.sh')
- )
-
- del_user_cmd = format("bash -x {mysql_deluser_path} {metron_user} {mysql_host} {mysql_admin_password!p}")
- Execute(del_user_cmd,
- tries=3,
- try_sleep=5,
- path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
- )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_utils.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_utils.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_utils.py
deleted file mode 100755
index 63cdb38..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_utils.py
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.core.resources.system import Execute, File
-from resource_management.core.source import StaticFile
-from resource_management.libraries.functions.format import format
-from mysql_service import mysql_service
-
-import mysql_users
-
-
-def mysql_configure():
- from params import params
-
- if params.install_mysql == 'Yes':
- mysql_service(daemon_name=params.daemon_name, action='start')
-
- replace_bind_address = ('sed', '-i', 's|^bind-address[ \t]*=.*|bind-address = 0.0.0.0|', params.mysql_configname)
- Execute(replace_bind_address,
- sudo=True,
- )
-
- mysql_users.mysql_adduser()
-
- File(params.mysql_create_geoip_path,
- mode=0755,
- content=StaticFile('createMysqlGeoIp.sh')
- )
-
- geoip_setup_cmd = format("bash -x {mysql_create_geoip_path} {geoip_ddl} {geoip_url} {mysql_admin_password!p}")
-
- Execute(geoip_setup_cmd,
- tries=3,
- try_sleep=5,
- path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
- )
-
- if params.install_mysql == 'Yes':
- mysql_service(daemon_name=params.daemon_name, action='stop')
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/__init__.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/__init__.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/__init__.py
deleted file mode 100755
index 242460e..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/__init__.py
+++ /dev/null
@@ -1,18 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params.py
deleted file mode 100755
index 953435d..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params.py
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-from ambari_commons import OSCheck
-from resource_management.libraries.functions.default import default
-from resource_management.libraries.functions.expect import expect
-
-if OSCheck.is_windows_family():
- from params_windows import *
-else:
- from params_linux import *
-
-java_home = config['hostLevelParams']['java_home']
-java_version = expect("/hostLevelParams/java_version", int)
-
-host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params_linux.py
deleted file mode 100755
index c418ad9..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params_linux.py
+++ /dev/null
@@ -1,186 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-import functools
-import os
-
-from ambari_commons.os_check import OSCheck
-from resource_management.libraries.functions import conf_select
-from resource_management.libraries.functions import format
-from resource_management.libraries.functions import get_kinit_path
-from resource_management.libraries.functions import stack_select
-from resource_management.libraries.functions.default import default
-from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
-from resource_management.libraries.functions.is_empty import is_empty
-from resource_management.libraries.resources.hdfs_resource import HdfsResource
-from resource_management.libraries.script import Script
-
-import status_params
-
-# server configurations
-config = Script.get_config()
-tmp_dir = Script.get_tmp_dir()
-
-hostname = config['hostname']
-user_group = config['configurations']['cluster-env']['user_group']
-metron_home = status_params.metron_home
-parsers = status_params.parsers
-metron_ddl_dir = metron_home + '/ddl'
-geoip_ddl = metron_ddl_dir + '/geoip_ddl.sql'
-geoip_url = config['configurations']['metron-env']['geoip_url']
-metron_indexing_topology = status_params.metron_indexing_topology
-metron_user = config['configurations']['metron-env']['metron_user']
-metron_group = config['configurations']['metron-env']['metron_group']
-metron_config_path = metron_home + '/config'
-metron_zookeeper_config_dir = status_params.metron_zookeeper_config_dir
-metron_zookeeper_config_path = status_params.metron_zookeeper_config_path
-parsers_configured_flag_file = status_params.parsers_configured_flag_file
-enrichment_configured_flag_file = status_params.enrichment_configured_flag_file
-indexing_configured_flag_file = status_params.indexing_configured_flag_file
-global_json_template = config['configurations']['metron-env']['global-json']
-global_properties_template = config['configurations']['metron-env']['elasticsearch-properties']
-
-# Elasticsearch hosts and port management
-es_cluster_name = config['configurations']['metron-env']['es_cluster_name']
-es_hosts = config['configurations']['metron-env']['es_hosts']
-es_host_list = es_hosts.split(",")
-es_binary_port = config['configurations']['metron-env']['es_binary_port']
-es_url = ",".join([host + ":" + es_binary_port for host in es_host_list])
-es_http_port = config['configurations']['metron-env']['es_http_port']
-es_http_url = es_host_list[0] + ":" + es_http_port
-
-# install repo
-yum_repo_type = config['configurations']['metron-env']['repo_type']
-if yum_repo_type == 'local':
- repo_url = 'file:///localrepo'
-else:
- repo_url = config['configurations']['metron-env']['repo_url']
-
-# hadoop params
-stack_root = Script.get_stack_root()
-hadoop_home_dir = stack_select.get_hadoop_dir("home")
-hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
-hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
-kafka_home = os.path.join(stack_root, "current", "kafka-broker")
-kafka_bin_dir = os.path.join(kafka_home, "bin")
-
-# zookeeper
-zk_hosts = default("/clusterHostInfo/zookeeper_hosts", [])
-has_zk_host = not len(zk_hosts) == 0
-zookeeper_quorum = None
-if has_zk_host:
- if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']:
- zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort']
- else:
- zookeeper_clientPort = '2181'
- zookeeper_quorum = (':' + zookeeper_clientPort + ',').join(config['clusterHostInfo']['zookeeper_hosts'])
- # last port config
- zookeeper_quorum += ':' + zookeeper_clientPort
-
-# Storm
-storm_rest_addr = status_params.storm_rest_addr
-
-# Kafka
-kafka_hosts = default("/clusterHostInfo/kafka_broker_hosts", [])
-has_kafka_host = not len(kafka_hosts) == 0
-kafka_brokers = None
-if has_kafka_host:
- if 'port' in config['configurations']['kafka-broker']:
- kafka_broker_port = config['configurations']['kafka-broker']['port']
- else:
- kafka_broker_port = '6667'
- kafka_brokers = (':' + kafka_broker_port + ',').join(config['clusterHostInfo']['kafka_broker_hosts'])
- kafka_brokers += ':' + kafka_broker_port
-
-metron_apps_dir = config['configurations']['metron-env']['metron_apps_hdfs_dir']
-metron_apps_enrichment_dir = metron_apps_dir + '/enrichment'
-metron_topic_retention = config['configurations']['metron-env']['metron_topic_retention']
-
-local_grok_patterns_dir = format("{metron_home}/patterns")
-hdfs_grok_patterns_dir = format("{metron_apps_dir}/patterns")
-
-# for create_hdfs_directory
-security_enabled = config['configurations']['cluster-env']['security_enabled']
-hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
-hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
-hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
-smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
-kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
-hdfs_site = config['configurations']['hdfs-site']
-default_fs = config['configurations']['core-site']['fs.defaultFS']
-dfs_type = default("/commandParams/dfs_type", "")
-
-# MYSQL
-if OSCheck.is_ubuntu_family():
- mysql_configname = '/etc/mysql/my.cnf'
-else:
- mysql_configname = '/etc/my.cnf'
-
-daemon_name = status_params.daemon_name
-
-install_mysql = config['configurations']['metron-env']['install_mysql']
-mysql_admin_password = config['configurations']['metron-env']['mysql_admin_password']
-
-# There will always be exactly one mysql_host
-mysql_host = config['clusterHostInfo']['metron_enrichment_mysql_server_hosts'][0]
-
-mysql_port = config['configurations']['metron-env']['metron_enrichment_db_port']
-
-mysql_adduser_path = tmp_dir + "/addMysqlUser.sh"
-mysql_deluser_path = tmp_dir + "/removeMysqlUser.sh"
-mysql_create_geoip_path = tmp_dir + "/createMysqlGeoIp.sh"
-
-enrichment_metron_user = config['configurations']['metron-env']['metron_enrichment_db_user']
-enrichment_metron_user_passwd = config['configurations']['metron-env']['metron_enrichment_db_password']
-enrichment_metron_user_passwd = unicode(enrichment_metron_user_passwd) if not is_empty(
- enrichment_metron_user_passwd) else enrichment_metron_user_passwd
-mysql_process_name = status_params.mysql_process_name
-
-# create partial functions with common arguments for every HdfsResource call
-# to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code
-HdfsResource = functools.partial(
- HdfsResource,
- user=hdfs_user,
- hdfs_resource_ignore_file="/var/lib/ambari-agent/data/.hdfs_resource_ignore",
- security_enabled=security_enabled,
- keytab=hdfs_user_keytab,
- kinit_path_local=kinit_path_local,
- hadoop_bin_dir=hadoop_bin_dir,
- hadoop_conf_dir=hadoop_conf_dir,
- principal_name=hdfs_principal_name,
- hdfs_site=hdfs_site,
- default_fs=default_fs,
- immutable_paths=get_not_managed_resources(),
- dfs_type=dfs_type
-)
-
-# HBase
-enrichment_table = status_params.enrichment_table
-enrichment_cf = status_params.enrichment_cf
-threatintel_table = status_params.threatintel_table
-threatintel_cf = status_params.threatintel_cf
-
-metron_enrichment_topology = status_params.metron_enrichment_topology
-metron_enrichment_topic = status_params.metron_enrichment_topic
-
-# ES Templates
-bro_index_path = tmp_dir + "/bro_index.template"
-snort_index_path = tmp_dir + "/snort_index.template"
-yaf_index_path = tmp_dir + "/yaf_index.template"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params_windows.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params_windows.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params_windows.py
deleted file mode 100755
index 4d11b35..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params_windows.py
+++ /dev/null
@@ -1,20 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-raise NotImplementedError
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/status_params.py
deleted file mode 100644
index 196defb..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/status_params.py
+++ /dev/null
@@ -1,57 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from ambari_commons import OSCheck
-from resource_management.libraries.functions import format
-from resource_management.libraries.script import Script
-
-config = Script.get_config()
-
-# Parsers
-parsers = config['configurations']['metron-env']['parsers']
-metron_home = config['configurations']['metron-env']['metron_home']
-metron_zookeeper_config_dir = config['configurations']['metron-env']['metron_zookeeper_config_dir']
-metron_zookeeper_config_path = format('{metron_home}/{metron_zookeeper_config_dir}')
-parsers_configured_flag_file = metron_zookeeper_config_path + '/../metron_parsers_configured'
-
-# Enrichment
-metron_enrichment_topology = 'enrichment'
-metron_enrichment_topic = 'enrichments'
-
-enrichment_table = 'enrichment'
-enrichment_cf = 't'
-threatintel_table = 'threatintel'
-threatintel_cf = 't'
-
-mysql_process_name = 'mysqld'
-if OSCheck.is_suse_family() or OSCheck.is_ubuntu_family():
- daemon_name = 'mysql'
-else:
- daemon_name = 'mysqld'
-
-# ing
-metron_indexing_topology = config['configurations']['metron-env']['metron_indexing_topology']
-indexing_configured_flag_file = metron_zookeeper_config_path + '/../metron_indexing_configured'
-
-# Enrichment
-enrichment_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_configured'
-
-# Storm
-storm_rest_addr = config['configurations']['metron-env']['storm_rest_addr']
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/parser_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/parser_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/parser_commands.py
deleted file mode 100755
index 034a193..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/parser_commands.py
+++ /dev/null
@@ -1,208 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-import os
-import re
-import subprocess
-import time
-
-from resource_management.core.logger import Logger
-from resource_management.core.resources.system import Execute, File
-
-import metron_service
-
-
-# Wrap major operations and functionality in this class
-class ParserCommands:
- __params = None
- __parser_list = None
- __configured = False
-
- def __init__(self, params):
- if params is None:
- raise ValueError("params argument is required for initialization")
- self.__params = params
- self.__parser_list = self.__get_parsers(params)
- self.__configured = os.path.isfile(self.__params.parsers_configured_flag_file)
-
- # get list of parsers
- def __get_parsers(self, params):
- return params.parsers.replace(' ', '').split(',')
-
- def is_configured(self):
- return self.__configured
-
- def set_configured(self):
- File(self.__params.parsers_configured_flag_file,
- content="",
- owner=self.__params.metron_user,
- mode=0775)
-
- def init_parsers(self):
- Logger.info(
- "Copying grok patterns from local directory '{0}' to HDFS '{1}'".format(self.__params.local_grok_patterns_dir,
- self.__params.hdfs_grok_patterns_dir))
- self.__params.HdfsResource(self.__params.hdfs_grok_patterns_dir,
- type="directory",
- action="create_on_execute",
- owner=self.__params.metron_user,
- mode=0775,
- source=self.__params.local_grok_patterns_dir)
-
- Logger.info("Done initializing parser configuration")
-
- def get_parser_list(self):
- return self.__parser_list
-
- def setup_repo(self):
- def local_repo():
- Logger.info("Setting up local repo")
- Execute("yum -y install createrepo")
- Execute("createrepo /localrepo")
- Execute("chmod -R o-w+r /localrepo")
- Execute("echo \"[METRON-0.2.1BETA]\n"
- "name=Metron 0.2.1BETA packages\n"
- "baseurl=file:///localrepo\n"
- "gpgcheck=0\n"
- "enabled=1\" > /etc/yum.repos.d/local.repo")
-
- def remote_repo():
- print('Using remote repo')
-
- yum_repo_types = {
- 'local': local_repo,
- 'remote': remote_repo
- }
- repo_type = self.__params.yum_repo_type
- if repo_type in yum_repo_types:
- yum_repo_types[repo_type]()
- else:
- raise ValueError("Unsupported repo type '{0}'".format(repo_type))
-
- def init_kafka_topics(self):
- Logger.info('Creating Kafka topics')
- command_template = """{0}/kafka-topics.sh \
- --zookeeper {1} \
- --create \
- --topic {2} \
- --partitions {3} \
- --replication-factor {4} \
- --config retention.bytes={5}"""
- num_partitions = 1
- replication_factor = 1
- retention_gigabytes = int(self.__params.metron_topic_retention)
- retention_bytes = retention_gigabytes * 1024 * 1024 * 1024
- Logger.info("Creating main topics for parsers")
- for parser_name in self.get_parser_list():
- Logger.info("Creating topic'{0}'".format(parser_name))
- Execute(command_template.format(self.__params.kafka_bin_dir,
- self.__params.zookeeper_quorum,
- parser_name,
- num_partitions,
- replication_factor,
- retention_bytes))
- Logger.info("Creating topics for error handling")
- Execute(command_template.format(self.__params.kafka_bin_dir,
- self.__params.zookeeper_quorum,
- "parser_invalid",
- num_partitions,
- replication_factor,
- retention_bytes))
- Execute(command_template.format(self.__params.kafka_bin_dir,
- self.__params.zookeeper_quorum,
- "parser_error",
- num_partitions, replication_factor,
- retention_bytes))
- Logger.info("Done creating Kafka topics")
-
- def start_parser_topologies(self):
- Logger.info("Starting Metron parser topologies: {0}".format(self.get_parser_list()))
- start_cmd_template = """{0}/bin/start_parser_topology.sh \
- -k {1} \
- -z {2} \
- -s {3}"""
- for parser in self.get_parser_list():
- Logger.info('Starting ' + parser)
- Execute(start_cmd_template.format(self.__params.metron_home, self.__params.kafka_brokers,
- self.__params.zookeeper_quorum, parser))
-
- Logger.info('Finished starting parser topologies')
-
- def stop_parser_topologies(self):
- Logger.info('Stopping parsers')
- for parser in self.get_parser_list():
- Logger.info('Stopping ' + parser)
- stop_cmd = 'storm kill ' + parser
- Execute(stop_cmd)
- Logger.info('Done stopping parser topologies')
-
- def restart_parser_topologies(self, env):
- Logger.info('Restarting the parser topologies')
- self.stop_parser_topologies()
- attempt_count = 0
- while self.topologies_running(env):
- if attempt_count > 2:
- raise Exception("Unable to kill topologies")
- attempt_count += 1
- time.sleep(10)
- self.start_parser_topologies()
- Logger.info('Done restarting the parser topologies')
-
- def topologies_exist(self):
- cmd_open = subprocess.Popen(["storm", "list"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- (stdout, stderr) = cmd_open.communicate()
- stdout_lines = stdout.splitlines()
- if stdout_lines:
- status_lines = self.__get_status_lines(stdout_lines)
- for parser in self.get_parser_list():
- for line in status_lines:
- items = re.sub('[\s]+', ' ', line).split()
- if items and items[0] == parser:
- return True
- return False
-
- def topologies_running(self, env):
- env.set_params(self.__params)
- all_running = True
- topologies = metron_service.get_running_topologies()
- for parser in self.get_parser_list():
- parser_found = False
- is_running = False
- if parser in topologies:
- parser_found = True
- is_running = topologies[parser] in ['ACTIVE', 'REBALANCING']
- all_running &= parser_found and is_running
- return all_running
-
- def __get_status_lines(self, lines):
- status_lines = []
- do_stat = False
- skipped = 0
- for line in lines:
- if line.startswith("Topology_name"):
- do_stat = True
- if do_stat and skipped == 2:
- status_lines += [line]
- elif do_stat:
- skipped += 1
- return status_lines
-
- def __is_running(self, status):
- return status in ['ACTIVE', 'REBALANCING']
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/parser_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/parser_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/parser_master.py
deleted file mode 100755
index e6d2ac6..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/parser_master.py
+++ /dev/null
@@ -1,88 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.core.exceptions import ComponentIsNotRunning
-from resource_management.core.logger import Logger
-from resource_management.libraries.script import Script
-
-import metron_service
-from parser_commands import ParserCommands
-
-
-class ParserMaster(Script):
- def get_component_name(self):
- # TODO add this at some point - currently will cause problems with hdp-select
- # return "parser-master"
- pass
-
- def install(self, env):
- from params import params
- env.set_params(params)
- commands = ParserCommands(params)
- commands.setup_repo()
- self.install_packages(env)
-
- def configure(self, env, upgrade_type=None, config_dir=None):
- from params import params
- env.set_params(params)
- metron_service.load_global_config(params)
- commands = ParserCommands(params)
- if not commands.is_configured():
- commands.init_parsers()
- commands.init_kafka_topics()
- commands.set_configured()
-
- def start(self, env, upgrade_type=None):
- from params import params
- env.set_params(params)
- self.configure(env)
- commands = ParserCommands(params)
- commands.start_parser_topologies()
-
- def stop(self, env, upgrade_type=None):
- from params import params
- env.set_params(params)
- commands = ParserCommands(params)
- commands.stop_parser_topologies()
-
- def status(self, env):
- from params import status_params
- env.set_params(status_params)
- commands = ParserCommands(status_params)
- if not commands.topologies_running(env):
- raise ComponentIsNotRunning()
-
- def restart(self, env):
- from params import params
- env.set_params(params)
- self.configure(env)
- commands = ParserCommands(params)
- commands.restart_parser_topologies(env)
-
- def servicechecktest(self, env):
- from params import params
- env.set_params(params)
- from service_check import ServiceCheck
- service_check = ServiceCheck()
- Logger.info('Service Check Test')
- service_check.service_check(env)
-
-
-if __name__ == "__main__":
- ParserMaster().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/service_check.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/service_check.py
deleted file mode 100755
index 7dd9dfb..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/service_check.py
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-from __future__ import print_function
-
-from resource_management.libraries.script import Script
-
-from indexing_commands import IndexingCommands
-from parser_commands import ParserCommands
-
-
-class ServiceCheck(Script):
- def service_check(self, env):
- from params import params
- parsercommands = ParserCommands(params)
- indexingcommands = IndexingCommands(params)
- all_found = parsercommands.topologies_running(env) and indexingcommands.is_topology_active(env)
- if all_found:
- exit(0)
- else:
- exit(1)
-
-
-if __name__ == "__main__":
- ServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/templates/enrichment.properties.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/templates/enrichment.properties.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/templates/enrichment.properties.j2
deleted file mode 100755
index bab2f52..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/templates/enrichment.properties.j2
+++ /dev/null
@@ -1,88 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-##### Kafka #####
-
-kafka.zk={{zookeeper_quorum}}
-kafka.broker={{kafka_brokers}}
-enrichment.output.topic=indexing
-
-##### MySQL #####
-
-mysql.ip={{mysql_host}}
-mysql.port={{mysql_port}}
-mysql.username={{enrichment_metron_user}}
-mysql.password={{enrichment_metron_user_passwd}}
-
-##### Metrics #####
-
-#reporters
-org.apache.metron.metrics.reporter.graphite=true
-org.apache.metron.metrics.reporter.console=false
-org.apache.metron.metrics.reporter.jmx=false
-
-#Graphite Addresses
-
-org.apache.metron.metrics.graphite.address=localhost
-org.apache.metron.metrics.graphite.port=2023
-
-#TelemetryParserBolt
-org.apache.metron.metrics.TelemetryParserBolt.acks=true
-org.apache.metron.metrics.TelemetryParserBolt.emits=true
-org.apache.metron.metrics.TelemetryParserBolt.fails=true
-
-
-#GenericEnrichmentBolt
-org.apache.metron.metrics.GenericEnrichmentBolt.acks=true
-org.apache.metron.metrics.GenericEnrichmentBolt.emits=true
-org.apache.metron.metrics.GenericEnrichmentBolt.fails=true
-
-
-#TelemetryIndexingBolt
-org.apache.metron.metrics.TelemetryIndexingBolt.acks=true
-org.apache.metron.metrics.TelemetryIndexingBolt.emits=true
-org.apache.metron.metrics.TelemetryIndexingBolt.fails=true
-
-##### Host Enrichment #####
-
-hbase.provider.impl=org.apache.metron.hbase.HTableProvider
-enrichment.simple.hbase.table={{enrichment_table}}
-enrichment.simple.hbase.cf={{enrichment_cf}}
-org.apache.metron.enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\
-{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},\
-{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}]
-
-
-##### HBase #####
-bolt.hbase.table.name=pcap
-bolt.hbase.table.fields=t:value
-bolt.hbase.table.key.tuple.field.name=key
-bolt.hbase.table.timestamp.tuple.field.name=timestamp
-bolt.hbase.enable.batching=false
-bolt.hbase.write.buffer.size.in.bytes=2000000
-bolt.hbase.durability=SKIP_WAL
-bolt.hbase.partitioner.region.info.refresh.interval.mins=60
-
-##### Threat Intel #####
-
-threat.intel.tracker.table={{threatintel_table}}
-threat.intel.tracker.cf={{threatintel_cf}}
-threat.intel.simple.hbase.table={{threatintel_table}}
-threat.intel.simple.hbase.cf={{threatintel_cf}}
-threat.intel.ip.table=
-threat.intel.ip.cf=
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/quicklinks/quicklinks.json
deleted file mode 100755
index ee1b225..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/quicklinks/quicklinks.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
- "name": "default",
- "description": "default quick links configuration",
- "configuration": {
- "protocol":
- {
- "type":"HTTP_ONLY"
- },
-
- "links": [
- {
- "name": "storm_ui",
- "label": "Storm UI",
- "requires_user_name": "false",
- "component_name": "STORM_UI_SERVER",
- "url":"%@://%@:%@/",
- "port":{
- "http_property": "ui.port",
- "http_default_port": "8744",
- "https_property": "ui.port",
- "https_default_port": "8744",
- "regex": "^(\\d+)$",
- "site": "storm-site"
- }
- }
- ]
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/service_advisor.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/service_advisor.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/service_advisor.py
deleted file mode 100644
index c481ce2..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/service_advisor.py
+++ /dev/null
@@ -1,141 +0,0 @@
-#!/usr/bin/env ambari-python-wrap
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-import os
-import fnmatch
-import imp
-import socket
-import sys
-import traceback
-
-SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
-STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../../../stacks/')
-PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py')
-
-try:
- with open(PARENT_FILE, 'rb') as fp:
- service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE))
-except Exception as e:
- traceback.print_exc()
- print "Failed to load parent"
-
-class METRON021BETAServiceAdvisor(service_advisor.ServiceAdvisor):
-
- def getServiceComponentLayoutValidations(self, services, hosts):
-
- componentsListList = [service["components"] for service in services["services"]]
- componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist]
-
- metronParsersHost = self.getHosts(componentsList, "METRON_PARSERS")[0]
- metronEnrichmentMaster = self.getHosts(componentsList, "METRON_ENRICHMENT_MASTER")[0]
- metronIndexingHost = self.getHosts(componentsList, "METRON_INDEXING")[0]
- metronEnrichmentMysqlServer = self.getHosts(componentsList, "METRON_ENRICHMENT_MYSQL_SERVER")[0]
-
- kafkaBrokers = self.getHosts(componentsList, "KAFKA_BROKER")
- stormSupervisors = self.getHosts(componentsList,"SUPERVISOR")
-
- items = []
-
- #Metron Must Co-locate with KAFKA_BROKER and STORM_SUPERVISOR
- if metronParsersHost not in kafkaBrokers:
- message = "Metron must be colocated with an instance of KAFKA BROKER"
- items.append({ "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'METRON_PARSERS', "host": metronParsersHost })
-
- if metronParsersHost not in stormSupervisors:
- message = "Metron must be colocated with an instance of STORM SUPERVISOR"
- items.append({ "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'METRON_PARSERS', "host": metronParsersHost })
-
- if metronParsersHost != metronEnrichmentMaster:
- message = "Metron Enrichment Master must be co-located with Metron Parsers on {0}".format(metronParsersHost)
- items.append({ "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'METRON_ENRICHMENT_MASTER', "host": metronEnrichmentMaster })
-
- if metronParsersHost != metronIndexingHost:
- message = "Metron Indexing must be co-located with Metron Parsers on {0}".format(metronParsersHost)
- items.append({ "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'METRON_INDEXING', "host": metronIndexingHost })
-
- if metronParsersHost != metronEnrichmentMysqlServer:
- message = "Metron MySQL Server must be co-located with Metron Parsers on {0}".format(metronParsersHost)
- items.append({ "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'METRON_ENRICHMENT_MYSQL_SERVER', "host": metronEnrichmentMysqlServer })
-
- return items
-
- def getServiceConfigurationsValidationItems(self, configurations, recommendedDefaults, services, hosts):
-
- # validate recommended properties in storm-site
- siteName = "storm-site"
- method = self.validateSTORMSiteConfigurations
- items = self.validateConfigurationsForSite(configurations, recommendedDefaults, services, hosts, siteName, method)
-
- return items
-
- def getServiceConfigurationRecommendations(self, configurations, clusterData, services, hosts):
- #Suggest mysql server hostname
- mySQLServerHost = self.getComponentHostNames(services, "METRON", "METRON_ENRICHMENT_MYSQL_SERVER")[0]
- putMetronEnvProperty = self.putProperty(configurations, "metron-env", services)
- putMetronEnvProperty("mysql_host",mySQLServerHost)
-
- #Suggest Storm Rest URL
- if "storm-site" in services["configurations"]:
- stormUIServerHost = self.getComponentHostNames(services, "STORM", "STORM_UI_SERVER")[0]
- stormUIServerPort = services["configurations"]["storm-site"]["properties"]["ui.port"]
- stormUIServerURL = stormUIServerHost + ":" + stormUIServerPort
- putMetronEnvProperty = self.putProperty(configurations, "metron-env", services)
- putMetronEnvProperty("storm_rest_addr",stormUIServerURL)
-
- storm_site = services["configurations"]["storm-site"]["properties"]
- putStormSiteProperty = self.putProperty(configurations, "storm-site", services)
-
- for property, desired_value in self.getSTORMSiteDesiredValues().iteritems():
- if property not in storm_site:
- putStormSiteProperty(property, desired_value)
- elif property == "topology.classpath" and storm_site[property] != desired_value:
- topololgyClasspath = storm_site[property]
- #check that desired values exist in topology.classpath. append them if they do not
- for path in desired_value.split(':'):
- if path not in topololgyClasspath:
- topololgyClasspath += ":" + path
- putStormSiteProperty(property,topololgyClasspath)
-
- def validateSTORMSiteConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
-
- storm_site = properties
- validationItems = []
-
- for property, desired_value in self.getSTORMSiteDesiredValues().iteritems():
- if property not in storm_site :
- message = "Metron requires this property to be set to the recommended value of " + desired_value
- item = self.getErrorItem(message) if property == "topology.classpath" else self.getWarnItem(message)
- validationItems.append({"config-name": property, "item": item})
- elif storm_site[property] != desired_value:
- topologyClasspath = storm_site[property]
- for path in desired_value.split(':'):
- if path not in topologyClasspath:
- message = "Metron requires this property to contain " + desired_value
- item = self.getErrorItem(message)
- validationItems.append({"config-name": property, "item": item})
-
- return self.toConfigurationValidationProblems(validationItems, "storm-site")
-
- def getSTORMSiteDesiredValues(self):
-
- storm_site_desired_values = {
- "topology.classpath" : "/etc/hbase/conf:/etc/hadoop/conf"
- }
-
- return storm_site_desired_values
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/themes/metron_theme.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/themes/metron_theme.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/themes/metron_theme.json
deleted file mode 100644
index b939182..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/themes/metron_theme.json
+++ /dev/null
@@ -1,219 +0,0 @@
-{
- "configuration": {
- "layouts": [
- {
- "name": "default",
- "tabs": [
- {
- "name": "metron_general",
- "display-name": "Settings",
- "layout": {
- "tab-columns": "1",
- "tab-rows": "1",
- "sections": [
- {
- "name": "section-general",
- "row-index": "0",
- "column-index": "0",
- "row-span": "1",
- "column-span": "1",
- "section-columns": "2",
- "section-rows": "1",
- "subsections": [
- {
- "name": "subsection-general-database",
- "display-name" : "Database Settings",
- "row-index": "0",
- "column-index": "0",
- "row-span": "1",
- "column-span": "1"
- },
- {
- "name": "subsection-general-indexing",
- "display-name": "Index Settings",
- "row-index": "0",
- "column-index": "1",
- "row-span": "1",
- "column-span": "1"
- }
- ]
- }
- ]
- }
- },
- {
- "name": "metron_repo",
- "display-name" : "Repository",
- "layout": {
- "tab-columns" : "1",
- "tab-rows" : "1",
- "sections" : [
- {
- "name" : "section-repo",
- "row-index": "0",
- "column-index" : "0",
- "row-span": "1",
- "column-span": "1",
- "section-columns": "1",
- "section-rows": "1",
- "subsections": [
- {
- "name": "subsection-repo",
- "row-index": "0",
- "column-index": "0",
- "row-span": "1",
- "column-span": "1"
- }
- ]
- }
- ]
- }
- }
- ]
- }
- ],
- "placement": {
- "configuration-layout": "default",
- "configs": [
- {
- "config": "metron-env/install_mysql",
- "subsection-name": "subsection-general-database"
- },
- {
- "config": "metron-env/mysql_admin_password",
- "subsection-name": "subsection-general-database",
- "depends-on": [
- {
- "configs":[
- "metron-env/install_mysql"
- ],
- "if": "${metron-env/install_mysql} === No",
- "then": {
- "property_value_attributes": {
- "visible": true
- }
- },
- "else": {
- "property_value_attributes": {
- "visible": false
- }
- }
- }
- ]
- },
- {
- "config": "metron-env/metron_enrichment_db_password",
- "subsection-name": "subsection-general-database"
- },
- {
- "config": "metron-env/metron_enrichment_db_port",
- "subsection-name": "subsection-general-database"
- },
- {
- "config": "metron-env/es_hosts",
- "subsection-name": "subsection-general-indexing"
- },
- {
- "config": "metron-env/es_binary_port",
- "subsection-name": "subsection-general-indexing"
- },
- {
- "config": "metron-env/es_http_port",
- "subsection-name": "subsection-general-indexing"
- },
- {
- "config": "metron-env/es_cluster_name",
- "subsection-name": "subsection-general-indexing"
- },
- {
- "config": "metron-env/repo_type",
- "subsection-name": "subsection-repo"
- },
- {
- "config": "metron-env/repo_url",
- "subsection-name": "subsection-repo",
- "depends-on": [
- {
- "configs":[
- "metron-env/repo_type"
- ],
- "if": "${metron-env/repo_type} === remote",
- "then": {
- "property_value_attributes": {
- "visible": true
- }
- },
- "else": {
- "property_value_attributes": {
- "visible": false
- }
- }
- }
- ]
- }
- ]
- },
- "widgets": [
- {
- "config": "metron-env/install_mysql",
- "widget": {
- "type": "toggle"
- }
- },
- {
- "config": "metron-env/mysql_admin_password",
- "widget": {
- "type": "password"
- }
- },
- {
- "config": "metron-env/metron_enrichment_db_password",
- "widget": {
- "type": "password"
- }
- },
- {
- "config": "metron-env/metron_enrichment_db_port",
- "widget": {
- "type": "text-field"
- }
- },
- {
- "config": "metron-env/es_hosts",
- "widget": {
- "type": "text-field"
- }
- },
- {
- "config": "metron-env/es_binary_port",
- "widget": {
- "type": "text-field"
- }
- },
- {
- "config": "metron-env/es_http_port",
- "widget": {
- "type": "text-field"
- }
- },
- {
- "config": "metron-env/es_cluster_name",
- "widget": {
- "type": "text-field"
- }
- },
- {
- "config": "metron-env/repo_type",
- "widget": {
- "type": "toggle"
- }
- },
- {
- "config": "metron-env/repo_url",
- "widget": {
- "type": "text-field"
- }
- }
- ]
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d7784d43/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/configuration/metron-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/configuration/metron-env.xml
new file mode 100644
index 0000000..da96be8
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.3.0/configuration/metron-env.xml
@@ -0,0 +1,256 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration supports_final="true">
+ <property>
+ <name>metron_home</name>
+ <value>/usr/metron/0.3.0</value>
+ <description>Metron home directory</description>
+ <display-name>Metron home</display-name>
+ </property>
+ <property>
+ <name>metron_apps_hdfs_dir</name>
+ <value>/apps/metron</value>
+ <description>Metron apps HDFS dir</description>
+ <display-name>Metron apps HDFS dir</display-name>
+ </property>
+ <property>
+ <name>metron_zookeeper_config_dir</name>
+ <value>config/zookeeper</value>
+ <description>Metron Zookeeper config dir. Relative path to Metron home.</description>
+ <display-name>Metron Zookeeper config dir</display-name>
+ </property>
+ <property>
+ <name>metron_user</name>
+ <value>metron</value>
+ <property-type>USER</property-type>
+ <description>The user for Metron</description>
+ <display-name>Metron User</display-name>
+ </property>
+ <property>
+ <name>metron_group</name>
+ <value>metron</value>
+ <property-type>GROUP</property-type>
+ <description>The group for Metron</description>
+ <display-name>Metron Group Name</display-name>
+ </property>
+ <property>
+ <name>metron_topic_retention</name>
+ <description>Kafka Retention in GB</description>
+ <value>10</value>
+ <display-name>Topic Retention</display-name>
+ </property>
+ <property>
+ <name>parsers</name>
+ <value>bro,snort,yaf</value>
+ <description>Metron parsers to deploy</description>
+ <display-name>Metron Parsers</display-name>
+ </property>
+ <property>
+ <name>metron_enrichment_db_user</name>
+ <value>metron</value>
+ <description>Database username to use to connect to the database.</description>
+ <display-name>Enrichment Database User</display-name>
+ </property>
+ <property>
+ <name>metron_enrichment_db_port</name>
+ <display-name>Metron Enrichment Database Port</display-name>
+ <value>3306</value>
+ <description>Database port to use to connect to the database.</description>
+ </property>
+ <property>
+ <name>metron_enrichment_db_password</name>
+ <value></value>
+ <property-type>PASSWORD</property-type>
+ <display-name>Metron Enrichment Database Password</display-name>
+ <description>Password to use against database</description>
+ <value-attributes>
+ <type>password</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>mysql_admin_password</name>
+ <value></value>
+ <property-type>PASSWORD</property-type>
+ <display-name>MySQL root user password</display-name>
+ <description>Password to use to add Metron user to MySQL</description>
+ <value-attributes>
+ <type>password</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>metron_indexing_topology</name>
+ <value>indexing</value>
+ <description>The Storm topology name for Indexing</description>
+ <display-name>Indexing Topology Name</display-name>
+ </property>
+ <property>
+ <name>es_cluster_name</name>
+ <value>metron</value>
+ <description>Name of Elasticsearch Cluster</description>
+ <display-name>Elasticsearch Cluster Name</display-name>
+ </property>
+ <property>
+ <name>geoip_url</name>
+ <value>http://geolite.maxmind.com/download/geoip/database/GeoLiteCity_CSV/GeoLiteCity-latest.tar.xz</value>
+ <description>Location of the GeoIP data to load.</description>
+ <display-name>GEOIP Load Datafile URL</display-name>
+ </property>
+ <property require-input="true">
+ <name>es_hosts</name>
+ <value></value>
+ <description>Comma delimited list of Elasticsearch Hosts. (eshost1,eshost2)</description>
+ <display-name>Elasticsearch Hosts</display-name>
+ </property>
+ <property>
+ <name>es_binary_port</name>
+ <value>9300</value>
+ <description>Elasticsearch binary port. (9300)</description>
+ <display-name>Elasticsearch Binary Port</display-name>
+ </property>
+ <property>
+ <name>es_http_port</name>
+ <value>9200</value>
+ <description>Elasticsearch HTTP port. (9200)</description>
+ <display-name>Elasticsearch HTTP port</display-name>
+ </property>
+ <property require-input = "true">
+ <name>storm_rest_addr</name>
+ <display-name>Storm Rest Server Address</display-name>
+ <description>URL of Storm UI (storm.ui.hostname:8744)</description>
+ <value></value>
+ </property>
+ <property>
+ <name>repo_type</name>
+ <display-name>Repository Type</display-name>
+ <description>Type of Repository: Local or Remote</description>
+ <value>local</value>
+ <value-attributes>
+ <overridable>false</overridable>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>local</value>
+ <label>Local</label>
+ </entry>
+ <entry>
+ <value>remote</value>
+ <label>Remote</label>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ </property>
+ <property>
+ <name>repo_url</name>
+ <display-name>Repository URL</display-name>
+ <value></value>
+ </property>
+ <property>
+ <name>install_mysql</name>
+ <description>Install New MySQL Instance for Enrichments</description>
+ <display-name>Install MySQL</display-name>
+ <value>Yes</value>
+ <value-attributes>
+ <overridable>false</overridable>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>Yes</value>
+ <label>Yes</label>
+ </entry>
+ <entry>
+ <value>No</value>
+ <label>No</label>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ </property>
+ <property>
+ <name>mysql_host</name>
+ <display-name>MySQL Host Address</display-name>
+ <value></value>
+ </property>
+ <property>
+ <name>global-json</name>
+ <display-name>global.json template</display-name>
+ <description>This is the jinja template for global.json file</description>
+ <value>
+{
+"es.clustername": "{{ es_cluster_name }}",
+"es.ip": "{{ es_url }}",
+"es.date.format": "yyyy.MM.dd.HH"
+}
+ </value>
+ <value-attributes>
+ <type>content</type>
+ </value-attributes>
+ </property>
+ <property>
+ <name>elasticsearch-properties</name>
+ <description>The template for the elasticsearch.properties file.</description>
+ <display-name>elasticsearch.properties template</display-name>
+ <value>
+##### Storm #####
+indexing.workers=1
+indexing.executors=0
+##### Kafka #####
+kafka.zk={{ zookeeper_quorum }}
+kafka.broker={{ kafka_brokers }}
+kafka.start=WHERE_I_LEFT_OFF
+##### Indexing #####
+index.input.topic=indexing
+index.error.topic=indexing_error
+writer.class.name=org.apache.metron.elasticsearch.writer.ElasticsearchWriter
+##### Metrics #####
+#reporters
+org.apache.metron.metrics.reporter.graphite=true
+org.apache.metron.metrics.reporter.console=false
+org.apache.metron.metrics.reporter.jmx=false
+#Graphite Addresses
+org.apache.metron.metrics.graphite.address=localhost
+org.apache.metron.metrics.graphite.port=2023
+#TelemetryParserBolt
+org.apache.metron.metrics.TelemetryParserBolt.acks=true
+org.apache.metron.metrics.TelemetryParserBolt.emits=true
+org.apache.metron.metrics.TelemetryParserBolt.fails=true
+##### HDFS #####
+bolt.hdfs.batch.size=5000
+bolt.hdfs.field.delimiter=|
+bolt.hdfs.rotation.policy=org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy
+bolt.hdfs.rotation.policy.units=DAYS
+bolt.hdfs.rotation.policy.count=1
+bolt.hdfs.file.rotation.size.in.mb=5
+bolt.hdfs.file.system.url={{ default_fs }}
+bolt.hdfs.wip.file.path=/paloalto/wip
+bolt.hdfs.finished.file.path=/paloalto/rotated
+bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec
+index.hdfs.output={{ metron_apps_enrichment_dir }}
+ </value>
+ <value-attributes>
+ <type>content</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+</configuration>