You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by mm...@apache.org on 2016/09/29 16:04:52 UTC
[1/4] incubator-metron git commit: METRON-398: Bump release version
to 0.2.1BETA in master (mmiklavc) closes apache/incubator-metron#280
Repository: incubator-metron
Updated Branches:
refs/heads/master dcda53825 -> 823cd2a83
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-platform/metron-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/pom.xml b/metron-platform/metron-elasticsearch/pom.xml
index 3ecd84d..f080516 100644
--- a/metron-platform/metron-elasticsearch/pom.xml
+++ b/metron-platform/metron-elasticsearch/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.2.0BETA</version>
+ <version>0.2.1BETA</version>
</parent>
<artifactId>metron-elasticsearch</artifactId>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-platform/metron-enrichment/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/pom.xml b/metron-platform/metron-enrichment/pom.xml
index 84891e8..a577e7a 100644
--- a/metron-platform/metron-enrichment/pom.xml
+++ b/metron-platform/metron-enrichment/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.2.0BETA</version>
+ <version>0.2.1BETA</version>
</parent>
<artifactId>metron-enrichment</artifactId>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-platform/metron-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-hbase/pom.xml b/metron-platform/metron-hbase/pom.xml
index c346325..234f6be 100644
--- a/metron-platform/metron-hbase/pom.xml
+++ b/metron-platform/metron-hbase/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.2.0BETA</version>
+ <version>0.2.1BETA</version>
</parent>
<artifactId>metron-hbase</artifactId>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-platform/metron-indexing/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/pom.xml b/metron-platform/metron-indexing/pom.xml
index 9760a11..e5306e4 100644
--- a/metron-platform/metron-indexing/pom.xml
+++ b/metron-platform/metron-indexing/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.2.0BETA</version>
+ <version>0.2.1BETA</version>
</parent>
<artifactId>metron-indexing</artifactId>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-platform/metron-integration-test/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/pom.xml b/metron-platform/metron-integration-test/pom.xml
index 4aa10cf..14fe5c4 100644
--- a/metron-platform/metron-integration-test/pom.xml
+++ b/metron-platform/metron-integration-test/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.2.0BETA</version>
+ <version>0.2.1BETA</version>
</parent>
<artifactId>metron-integration-test</artifactId>
<description>Metron Integration Test</description>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-platform/metron-management/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-management/pom.xml b/metron-platform/metron-management/pom.xml
index d791809..de40aff 100644
--- a/metron-platform/metron-management/pom.xml
+++ b/metron-platform/metron-management/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.2.0BETA</version>
+ <version>0.2.1BETA</version>
</parent>
<artifactId>metron-management</artifactId>
<name>metron-management</name>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-platform/metron-parsers/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/pom.xml b/metron-platform/metron-parsers/pom.xml
index 47f7bdd..5c52734 100644
--- a/metron-platform/metron-parsers/pom.xml
+++ b/metron-platform/metron-parsers/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.2.0BETA</version>
+ <version>0.2.1BETA</version>
</parent>
<artifactId>metron-parsers</artifactId>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-platform/metron-pcap-backend/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap-backend/pom.xml b/metron-platform/metron-pcap-backend/pom.xml
index 792bef5..1318ee1 100644
--- a/metron-platform/metron-pcap-backend/pom.xml
+++ b/metron-platform/metron-pcap-backend/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.2.0BETA</version>
+ <version>0.2.1BETA</version>
</parent>
<artifactId>metron-pcap-backend</artifactId>
<name>Metron PCAP Backend</name>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-platform/metron-pcap/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap/pom.xml b/metron-platform/metron-pcap/pom.xml
index 792cecb..6b14712 100644
--- a/metron-platform/metron-pcap/pom.xml
+++ b/metron-platform/metron-pcap/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.2.0BETA</version>
+ <version>0.2.1BETA</version>
</parent>
<artifactId>metron-pcap</artifactId>
<description>Metron Pcap</description>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-platform/metron-solr/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/pom.xml b/metron-platform/metron-solr/pom.xml
index 3aceda0..79a7c60 100644
--- a/metron-platform/metron-solr/pom.xml
+++ b/metron-platform/metron-solr/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.2.0BETA</version>
+ <version>0.2.1BETA</version>
</parent>
<artifactId>metron-solr</artifactId>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-platform/metron-test-utilities/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-test-utilities/pom.xml b/metron-platform/metron-test-utilities/pom.xml
index 860be5b..4796667 100644
--- a/metron-platform/metron-test-utilities/pom.xml
+++ b/metron-platform/metron-test-utilities/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.2.0BETA</version>
+ <version>0.2.1BETA</version>
</parent>
<artifactId>metron-test-utilities</artifactId>
<description>Metron Test Utilities</description>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-platform/metron-writer/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/pom.xml b/metron-platform/metron-writer/pom.xml
index d088c4b..1859c7f 100644
--- a/metron-platform/metron-writer/pom.xml
+++ b/metron-platform/metron-writer/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.2.0BETA</version>
+ <version>0.2.1BETA</version>
</parent>
<artifactId>metron-writer</artifactId>
<name>metron-writer</name>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-platform/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/pom.xml b/metron-platform/pom.xml
index 15b07fa..7dc2489 100644
--- a/metron-platform/pom.xml
+++ b/metron-platform/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>Metron</artifactId>
- <version>0.2.0BETA</version>
+ <version>0.2.1BETA</version>
</parent>
<description>Stream analytics for Metron</description>
<url>https://metron.incubator.apache.org/</url>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 81d9d8f..4d6adb3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -19,7 +19,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.metron</groupId>
<artifactId>Metron</artifactId>
- <version>0.2.0BETA</version>
+ <version>0.2.1BETA</version>
<packaging>pom</packaging>
<name>Metron</name>
<description>Metron Top Level Project</description>
@@ -176,7 +176,6 @@
<exclude>**/bro-plugin-kafka/build/**</exclude>
<exclude>**/files/opensoc-ui</exclude>
<!-- pickle file - binary format -->
- <exclude>**/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/*.p</exclude>
<exclude>**/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p</exclude>
</excludes>
</configuration>
[2/4] incubator-metron git commit: METRON-398: Bump release version
to 0.2.1BETA in master (mmiklavc) closes apache/incubator-metron#280
Posted by mm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/yaf_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/yaf_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/yaf_index.template
new file mode 100644
index 0000000..7743afc
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/yaf_index.template
@@ -0,0 +1,205 @@
+{
+ "template": "yaf_index*",
+ "mappings": {
+ "yaf_doc": {
+ "_timestamp": {
+ "enabled": true
+ },
+ "dynamic_templates": [
+ {
+ "geo_location_point": {
+ "match": "enrichments:geo:*:location_point",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "geo_point"
+ }
+ }
+ },
+ {
+ "geo_country": {
+ "match": "enrichments:geo:*:country",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_city": {
+ "match": "enrichments:geo:*:city",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_location_id": {
+ "match": "enrichments:geo:*:locID",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_dma_code": {
+ "match": "enrichments:geo:*:dmaCode",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_postal_code": {
+ "match": "enrichments:geo:*:postalCode",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_latitude": {
+ "match": "enrichments:geo:*:latitude",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "float"
+ }
+ }
+ },
+ {
+ "geo_longitude": {
+ "match": "enrichments:geo:*:longitude",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "float"
+ }
+ }
+ },
+ {
+ "timestamps": {
+ "match": "*:ts",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "date",
+ "format": "epoch_millis"
+ }
+ }
+ }
+ ],
+ "properties": {
+ "timestamp": {
+ "type": "date",
+ "format": "epoch_millis"
+ },
+ "source:type": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "ip_dst_addr": {
+ "type": "ip"
+ },
+ "ip_dst_port": {
+ "type": "integer"
+ },
+ "ip_src_addr": {
+ "type": "ip"
+ },
+ "ip_src_port": {
+ "type": "integer"
+ },
+ "start_time": {
+ "type": "date",
+ "format": "epoch_millis"
+ },
+ "end_time": {
+ "type": "date",
+ "format": "epoch_millis"
+ },
+ "duration": {
+ "type": "double"
+ },
+ "rtt": {
+ "type": "double"
+ },
+ "proto": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "sip": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "sp": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "dip": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "dp": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "iflags": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "uflags": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "riflags": {
+ "type": "string"
+ },
+ "ruflags": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "isn": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "risn": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "tag": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "rtag": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "pkt": {
+ "type": "integer"
+ },
+ "oct": {
+ "type": "integer"
+ },
+ "rpkt": {
+ "type": "integer"
+ },
+ "roct": {
+ "type": "integer"
+ },
+ "app": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "end-reason": {
+ "type": "string"
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/enrichment_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/enrichment_commands.py
new file mode 100755
index 0000000..65b834b
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/enrichment_commands.py
@@ -0,0 +1,161 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import os
+import time
+
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute, File
+
+import metron_service
+
+
+# Wrap major operations and functionality in this class
+class EnrichmentCommands:
+ __params = None
+ __enrichment_topology = None
+ __enrichment_topic = None
+ __configured = False
+
+ def __init__(self, params):
+ if params is None:
+ raise ValueError("params argument is required for initialization")
+ self.__params = params
+ self.__enrichment_topology = params.metron_enrichment_topology
+ self.__enrichment_topic = params.metron_enrichment_topic
+ self.__configured = os.path.isfile(self.__params.enrichment_configured_flag_file)
+
+ def is_configured(self):
+ return self.__configured
+
+ def set_configured(self):
+ File(self.__params.enrichment_configured_flag_file,
+ content="",
+ owner=self.__params.metron_user,
+ mode=0775)
+
+ def setup_repo(self):
+ def local_repo():
+ Logger.info("Setting up local repo")
+ Execute("yum -y install createrepo")
+ Execute("createrepo /localrepo")
+ Execute("chmod -R o-w+r /localrepo")
+ Execute("echo \"[METRON-0.2.1BETA]\n"
+ "name=Metron 0.2.1BETA packages\n"
+ "baseurl=file:///localrepo\n"
+ "gpgcheck=0\n"
+ "enabled=1\" > /etc/yum.repos.d/local.repo")
+
+ def remote_repo():
+ print('Using remote repo')
+
+ yum_repo_types = {
+ 'local': local_repo,
+ 'remote': remote_repo
+ }
+ repo_type = self.__params.yum_repo_type
+ if repo_type in yum_repo_types:
+ yum_repo_types[repo_type]()
+ else:
+ raise ValueError("Unsupported repo type '{0}'".format(repo_type))
+
+ def init_kafka_topics(self):
+ Logger.info('Creating Kafka topics')
+ command_template = """{0}/kafka-topics.sh \
+ --zookeeper {1} \
+ --create \
+ --topic {2} \
+ --partitions {3} \
+ --replication-factor {4} \
+ --config retention.bytes={5}"""
+ num_partitions = 1
+ replication_factor = 1
+ retention_gigabytes = int(self.__params.metron_topic_retention)
+ retention_bytes = retention_gigabytes * 1024 * 1024 * 1024
+ Logger.info("Creating topics for enrichment")
+
+ Logger.info("Creating topic'{0}'".format(self.__enrichment_topic))
+ Execute(command_template.format(self.__params.kafka_bin_dir,
+ self.__params.zookeeper_quorum,
+ self.__enrichment_topic,
+ num_partitions,
+ replication_factor,
+ retention_bytes))
+ Logger.info("Done creating Kafka topics")
+
+ def start_enrichment_topology(self):
+ Logger.info("Starting Metron enrichment topology: {0}".format(self.__enrichment_topology))
+ start_cmd_template = """{0}/bin/start_enrichment_topology.sh \
+ -s {1} \
+ -z {2}"""
+ Logger.info('Starting ' + self.__enrichment_topology)
+ Execute(start_cmd_template.format(self.__params.metron_home, self.__enrichment_topology, self.__params.zookeeper_quorum))
+
+ Logger.info('Finished starting enrichment topology')
+
+ def stop_enrichment_topology(self):
+ Logger.info('Stopping ' + self.__enrichment_topology)
+ stop_cmd = 'storm kill ' + self.__enrichment_topology
+ Execute(stop_cmd)
+ Logger.info('Done stopping enrichment topologies')
+
+ def restart_enrichment_topology(self, env):
+ Logger.info('Restarting the enrichment topologies')
+ self.stop_enrichment_topology()
+
+ # Wait for old topology to be cleaned up by Storm, before starting again.
+ retries = 0
+ topology_active = self.is_topology_active(env)
+ while topology_active and retries < 3:
+ Logger.info('Existing topology still active. Will wait and retry')
+ time.sleep(40)
+ topology_active = self.is_topology_active(env)
+ retries += 1
+
+ if not topology_active:
+ self.start_enrichment_topology()
+ Logger.info('Done restarting the enrichment topology')
+ else:
+ Logger.warning('Retries exhausted. Existing topology not cleaned up. Aborting topology start.')
+
+ def is_topology_active(self, env):
+ env.set_params(self.__params)
+
+ active = True
+ topologies = metron_service.get_running_topologies()
+ is_running = False
+ if self.__enrichment_topology in topologies:
+ is_running = topologies[self.__enrichment_topology] in ['ACTIVE', 'REBALANCING']
+ active &= is_running
+ return active
+
+ def create_hbase_tables(self):
+ add_enrichment_cmd = "echo \"create '{0}','{1}'\" | hbase shell -n".format(self.__params.enrichment_table, self.__params.enrichment_cf)
+ Execute(add_enrichment_cmd,
+ tries=3,
+ try_sleep=5,
+ logoutput=False,
+ path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'
+ )
+
+ add_threatintel_cmd = "echo \"create '{0}','{1}'\" | hbase shell -n".format(self.__params.threatintel_table, self.__params.threatintel_cf)
+ Execute(add_threatintel_cmd,
+ tries=3,
+ try_sleep=5,
+ logoutput=False,
+ path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'
+ )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/enrichment_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/enrichment_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/enrichment_master.py
new file mode 100755
index 0000000..390576b
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/enrichment_master.py
@@ -0,0 +1,87 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import File
+from resource_management.core.source import Template
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.script import Script
+
+from enrichment_commands import EnrichmentCommands
+
+
+class Enrichment(Script):
+ def install(self, env):
+ from params import params
+ env.set_params(params)
+ commands = EnrichmentCommands(params)
+ commands.setup_repo()
+ Logger.info('Install RPM packages')
+ self.install_packages(env)
+ self.configure(env)
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ from params import params
+ env.set_params(params)
+
+ File(format("{metron_config_path}/enrichment.properties"),
+ content=Template("enrichment.properties.j2"),
+ owner=params.metron_user,
+ group=params.metron_group
+ )
+
+ def start(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ commands = EnrichmentCommands(params)
+
+ if not commands.is_configured():
+ commands.init_kafka_topics()
+ commands.create_hbase_tables()
+ commands.set_configured()
+
+ commands.start_enrichment_topology()
+
+ def stop(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ commands = EnrichmentCommands(params)
+ commands.stop_enrichment_topology()
+
+ def status(self, env):
+ from params import status_params
+ env.set_params(status_params)
+ commands = EnrichmentCommands(status_params)
+
+ if not commands.is_topology_active(env):
+ raise ComponentIsNotRunning()
+
+ def restart(self, env):
+ from params import params
+ env.set_params(params)
+ commands = EnrichmentCommands(params)
+ commands.restart_enrichment_topology(env)
+
+ def kafkabuild(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ commands = EnrichmentCommands(params)
+ commands.init_kafka_topics()
+
+
+if __name__ == "__main__":
+ Enrichment().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/indexing_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/indexing_commands.py
new file mode 100755
index 0000000..fdb90e9
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/indexing_commands.py
@@ -0,0 +1,142 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import os
+import time
+
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute, File
+
+import metron_service
+
+
+# Wrap major operations and functionality in this class
+class IndexingCommands:
+ __params = None
+ __indexing = None
+ __configured = False
+
+ def __init__(self, params):
+ if params is None:
+ raise ValueError("params argument is required for initialization")
+ self.__params = params
+ self.__indexing = params.metron_indexing_topology
+ self.__configured = os.path.isfile(self.__params.indexing_configured_flag_file)
+
+ def is_configured(self):
+ return self.__configured
+
+ def set_configured(self):
+ File(self.__params.indexing_configured_flag_file,
+ content="",
+ owner=self.__params.metron_user,
+ mode=0775)
+
+ def setup_repo(self):
+ def local_repo():
+ Logger.info("Setting up local repo")
+ Execute("yum -y install createrepo")
+ Execute("createrepo /localrepo")
+ Execute("chmod -R o-w+r /localrepo")
+ Execute("echo \"[METRON-0.2.1BETA]\n"
+ "name=Metron 0.2.1BETA packages\n"
+ "baseurl=file:///localrepo\n"
+ "gpgcheck=0\n"
+ "enabled=1\" > /etc/yum.repos.d/local.repo")
+
+ def remote_repo():
+ print('Using remote repo')
+
+ yum_repo_types = {
+ 'local': local_repo,
+ 'remote': remote_repo
+ }
+ repo_type = self.__params.yum_repo_type
+ if repo_type in yum_repo_types:
+ yum_repo_types[repo_type]()
+ else:
+ raise ValueError("Unsupported repo type '{0}'".format(repo_type))
+
+ def init_kafka_topics(self):
+ Logger.info('Creating Kafka topics')
+ command_template = """{0}/kafka-topics.sh \
+ --zookeeper {1} \
+ --create \
+ --topic {2} \
+ --partitions {3} \
+ --replication-factor {4} \
+ --config retention.bytes={5}"""
+ num_partitions = 1
+ replication_factor = 1
+ retention_gigabytes = int(self.__params.metron_topic_retention)
+ retention_bytes = retention_gigabytes * 1024 * 1024 * 1024
+ Logger.info("Creating topics for indexing")
+
+ Logger.info("Creating topic'{0}'".format(self.__indexing))
+ Execute(command_template.format(self.__params.kafka_bin_dir,
+ self.__params.zookeeper_quorum,
+ self.__indexing,
+ num_partitions,
+ replication_factor,
+ retention_bytes))
+ Logger.info("Done creating Kafka topics")
+
+ def start_indexing_topology(self):
+ Logger.info("Starting Metron indexing topology: {0}".format(self.__indexing))
+ start_cmd_template = """{0}/bin/start_elasticsearch_topology.sh \
+ -s {1} \
+ -z {2}"""
+ Logger.info('Starting ' + self.__indexing)
+ Execute(start_cmd_template.format(self.__params.metron_home, self.__indexing, self.__params.zookeeper_quorum))
+
+ Logger.info('Finished starting indexing topology')
+
+ def stop_indexing_topology(self):
+ Logger.info('Stopping ' + self.__indexing)
+ stop_cmd = 'storm kill ' + self.__indexing
+ Execute(stop_cmd)
+ Logger.info('Done stopping indexing topologies')
+
+ def restart_indexing_topology(self, env):
+ Logger.info('Restarting the indexing topologies')
+ self.stop_indexing_topology()
+
+ # Wait for old topology to be cleaned up by Storm, before starting again.
+ retries = 0
+ topology_active = self.is_topology_active(env)
+ while self.is_topology_active(env) and retries < 3:
+ Logger.info('Existing topology still active. Will wait and retry')
+ time.sleep(10)
+ retries += 1
+
+ if not topology_active:
+ Logger.info('Waiting for storm kill to complete')
+ time.sleep(30)
+ self.start_indexing_topology()
+ Logger.info('Done restarting the indexing topologies')
+ else:
+ Logger.warning('Retries exhausted. Existing topology not cleaned up. Aborting topology start.')
+
+ def is_topology_active(self, env):
+ env.set_params(self.__params)
+ active = True
+ topologies = metron_service.get_running_topologies()
+ is_running = False
+ if self.__indexing in topologies:
+ is_running = topologies[self.__indexing] in ['ACTIVE', 'REBALANCING']
+ active &= is_running
+ return active
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/indexing_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/indexing_master.py
new file mode 100755
index 0000000..eabf5a1
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/indexing_master.py
@@ -0,0 +1,120 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute
+from resource_management.core.resources.system import File
+from resource_management.core.source import StaticFile
+from resource_management.libraries.functions import format as ambari_format
+from resource_management.libraries.script import Script
+
+import metron_service
+from indexing_commands import IndexingCommands
+
+
+class Indexing(Script):
+ __configured = False
+
+ def install(self, env):
+ from params import params
+ env.set_params(params)
+ commands = IndexingCommands(params)
+ commands.setup_repo()
+ Logger.info('Install RPM packages')
+ self.install_packages(env)
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ from params import params
+ env.set_params(params)
+
+ commands = IndexingCommands(params)
+ metron_service.load_global_config(params)
+
+ if not commands.is_configured():
+ commands.init_kafka_topics()
+ commands.set_configured()
+
+ def start(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ self.configure(env)
+ commands = IndexingCommands(params)
+ commands.start_indexing_topology()
+
+ def stop(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ commands = IndexingCommands(params)
+ commands.stop_indexing_topology()
+
+ def status(self, env):
+ from params import status_params
+ env.set_params(status_params)
+ commands = IndexingCommands(status_params)
+ if not commands.is_topology_active(env):
+ raise ComponentIsNotRunning()
+
+ def restart(self, env):
+ from params import params
+ env.set_params(params)
+ self.configure(env)
+ commands = IndexingCommands(params)
+ commands.restart_indexing_topology(env)
+
+ def elasticsearch_template_install(self, env):
+ from params import params
+ env.set_params(params)
+
+ File(params.bro_index_path,
+ mode=0755,
+ content=StaticFile('bro_index.template')
+ )
+
+ File(params.snort_index_path,
+ mode=0755,
+ content=StaticFile('snort_index.template')
+ )
+
+ File(params.yaf_index_path,
+ mode=0755,
+ content=StaticFile('yaf_index.template')
+ )
+
+ bro_cmd = ambari_format(
+ 'curl -s -XPOST http://{es_url}/_template/bro_index -d @{bro_index_path}')
+ Execute(bro_cmd, logoutput=True)
+ snort_cmd = ambari_format(
+ 'curl -s -XPOST http://{es_url}/_template/snort_index -d @{snort_index_path}')
+ Execute(snort_cmd, logoutput=True)
+ yaf_cmd = ambari_format(
+ 'curl -s -XPOST http://{es_url}/_template/yaf_index -d @{yaf_index_path}')
+ Execute(yaf_cmd, logoutput=True)
+
+ def elasticsearch_template_delete(self, env):
+ from params import params
+ env.set_params(params)
+
+ bro_cmd = ambari_format('curl -s -XDELETE "http://{es_url}/bro_index*"')
+ Execute(bro_cmd, logoutput=True)
+ snort_cmd = ambari_format('curl -s -XDELETE "http://{es_url}/snort_index*"')
+ Execute(snort_cmd, logoutput=True)
+ yaf_cmd = ambari_format('curl -s -XDELETE "http://{es_url}/yaf_index*"')
+ Execute(yaf_cmd, logoutput=True)
+
+
+if __name__ == "__main__":
+ Indexing().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/metron_service.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/metron_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/metron_service.py
new file mode 100644
index 0000000..57da2c7
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/metron_service.py
@@ -0,0 +1,76 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import json
+import subprocess
+
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Directory, File
+from resource_management.core.resources.system import Execute
+from resource_management.core.source import InlineTemplate
+from resource_management.libraries.functions import format as ambari_format
+
+
+def init_config():
+ Logger.info('Loading config into ZooKeeper')
+ Execute(ambari_format(
+ "{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_quorum}"),
+ path=ambari_format("{java_home}/bin")
+ )
+
+
+def get_running_topologies():
+ Logger.info('Getting Running Storm Topologies from Storm REST Server')
+
+ cmd = ambari_format('curl --max-time 3 {storm_rest_addr}/api/v1/topology/summary')
+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+ (stdout, stderr) = proc.communicate()
+
+ try:
+ stormjson = json.loads(stdout)
+ except ValueError:
+ return {}
+
+ topologiesDict = {}
+
+ for topology in stormjson['topologies']:
+ topologiesDict[topology['name']] = topology['status']
+
+ Logger.info("Topologies: " + str(topologiesDict))
+ return topologiesDict
+
+
+def load_global_config(params):
+ Logger.info('Create Metron Local Config Directory')
+ Logger.info("Configure Metron global.json")
+
+ directories = [params.metron_zookeeper_config_path]
+ Directory(directories,
+ mode=0755,
+ owner=params.metron_user,
+ group=params.metron_group
+ )
+
+ File("{0}/global.json".format(params.metron_zookeeper_config_path),
+ owner=params.metron_user,
+ content=InlineTemplate(params.global_json_template)
+ )
+
+ File("{0}/elasticsearch.properties".format(params.metron_zookeeper_config_path + '/..'),
+ owner=params.metron_user,
+ content=InlineTemplate(params.global_properties_template))
+
+ init_config()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_server.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_server.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_server.py
new file mode 100755
index 0000000..2fc363f
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_server.py
@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.libraries.script.script import Script
+
+import mysql_users
+from mysql_service import mysql_service
+from mysql_utils import mysql_configure
+
+
+class MysqlServer(Script):
+ def install(self, env):
+ self.install_packages(env)
+ self.configure(env)
+
+ def clean(self, env):
+ from params import params
+ env.set_params(params)
+ mysql_users.mysql_deluser()
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ from params import params
+ env.set_params(params)
+ mysql_configure()
+
+ def start(self, env, rolling_restart=False):
+ from params import params
+ env.set_params(params)
+ mysql_service(daemon_name=params.daemon_name, action='start')
+
+ def stop(self, env, rolling_restart=False):
+ from params import params
+ env.set_params(params)
+ mysql_service(daemon_name=params.daemon_name, action='stop')
+
+ def status(self, env):
+ from params import status_params
+ env.set_params(status_params)
+
+ mysql_service(daemon_name=status_params.daemon_name, action='status')
+
+
+if __name__ == "__main__":
+ MysqlServer().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_service.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_service.py
new file mode 100755
index 0000000..2e0ce8b
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_service.py
@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.exceptions import ComponentIsNotRunning, Fail
+from resource_management.core.resources.system import Execute
+from resource_management.libraries.functions.format import format
+
+
+def mysql_service(daemon_name=None, action='start'):
+ status_cmd = format("pgrep -l '^{mysql_process_name}$'")
+ cmd = ('service', daemon_name, action)
+
+ if action == 'status':
+ try:
+ Execute(status_cmd)
+ except Fail:
+ raise ComponentIsNotRunning()
+ elif action == 'stop':
+ Execute(cmd,
+ logoutput=True,
+ only_if=status_cmd,
+ sudo=True,
+ )
+ elif action == 'start':
+ Execute(cmd,
+ logoutput=True,
+ not_if=status_cmd,
+ sudo=True,
+ )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_users.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_users.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_users.py
new file mode 100755
index 0000000..1721eba
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_users.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.resources.system import Execute, File
+from resource_management.core.source import StaticFile
+from resource_management.libraries.functions.format import format
+
+
+# Used to add metron access to the needed components
+def mysql_adduser():
+ from params import params
+
+ File(params.mysql_adduser_path,
+ mode=0755,
+ content=StaticFile('addMysqlUser.sh')
+ )
+
+ add_user_cmd = format("bash -x {mysql_adduser_path} {daemon_name} {metron_user} {enrichment_metron_user_passwd!p} {enrichment_host}")
+ Execute(add_user_cmd,
+ tries=3,
+ try_sleep=5,
+ logoutput=False,
+ path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'
+ )
+
+
+# Removes hive metron from components
+def mysql_deluser():
+ from params import params
+
+ File(params.mysql_deluser_path,
+ mode=0755,
+ content=StaticFile('removeMysqlUser.sh')
+ )
+
+ del_user_cmd = format("bash -x {mysql_deluser_path} {daemon_name} {metron_user} {enrichment_host}")
+ Execute(del_user_cmd,
+ tries=3,
+ try_sleep=5,
+ path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
+ )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_utils.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_utils.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_utils.py
new file mode 100755
index 0000000..40b925d
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_utils.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.resources.system import Execute, File
+from resource_management.core.source import StaticFile
+from resource_management.libraries.functions.format import format
+
+import mysql_users
+
+
+def mysql_configure():
+ from params import params
+
+ # required for running hive
+ replace_bind_address = ('sed', '-i', 's|^bind-address[ \t]*=.*|bind-address = 0.0.0.0|', params.mysql_configname)
+ Execute(replace_bind_address,
+ sudo=True,
+ )
+
+ # this also will start mysql-server
+ mysql_users.mysql_adduser()
+
+ File(params.mysql_create_geoip_path,
+ mode=0755,
+ content=StaticFile('createMysqlGeoIp.sh')
+ )
+
+ geoip_setup_cmd = format("bash -x {mysql_create_geoip_path} {daemon_name} {geoip_ddl} {geoip_url}")
+ Execute(geoip_setup_cmd,
+ tries=3,
+ try_sleep=5,
+ path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
+ )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/__init__.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/__init__.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/__init__.py
new file mode 100755
index 0000000..242460e
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/__init__.py
@@ -0,0 +1,18 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params.py
new file mode 100755
index 0000000..953435d
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from ambari_commons import OSCheck
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.expect import expect
+
+if OSCheck.is_windows_family():
+ from params_windows import *
+else:
+ from params_linux import *
+
+java_home = config['hostLevelParams']['java_home']
+java_version = expect("/hostLevelParams/java_version", int)
+
+host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params_linux.py
new file mode 100755
index 0000000..5f914f3
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params_linux.py
@@ -0,0 +1,169 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import functools
+import os
+
+from ambari_commons.os_check import OSCheck
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions import get_kinit_path
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
+from resource_management.libraries.functions.is_empty import is_empty
+from resource_management.libraries.resources.hdfs_resource import HdfsResource
+from resource_management.libraries.script import Script
+
+import status_params
+
+# server configurations
+config = Script.get_config()
+tmp_dir = Script.get_tmp_dir()
+
+hostname = config['hostname']
+metron_home = status_params.metron_home
+parsers = status_params.parsers
+metron_ddl_dir = metron_home + '/ddl'
+geoip_ddl = metron_ddl_dir + '/geoip_ddl.sql'
+geoip_url = config['configurations']['metron-env']['geoip_url']
+metron_indexing_topology = status_params.metron_indexing_topology
+metron_user = config['configurations']['metron-env']['metron_user']
+metron_group = config['configurations']['metron-env']['metron_group']
+metron_config_path = metron_home + '/config'
+metron_zookeeper_config_dir = status_params.metron_zookeeper_config_dir
+metron_zookeeper_config_path = status_params.metron_zookeeper_config_path
+parsers_configured_flag_file = status_params.parsers_configured_flag_file
+enrichment_configured_flag_file = status_params.enrichment_configured_flag_file
+indexing_configured_flag_file = status_params.indexing_configured_flag_file
+global_json_template = config['configurations']['metron-env']['global-json']
+global_properties_template = config['configurations']['metron-env']['elasticsearch-properties']
+es_cluster_name = config['configurations']['metron-env']['es_cluster_name']
+es_url = config['configurations']['metron-env']['es_url']
+yum_repo_type = 'local'
+
+# hadoop params
+stack_root = Script.get_stack_root()
+hadoop_home_dir = stack_select.get_hadoop_dir("home")
+hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
+hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
+kafka_home = os.path.join(stack_root, "current", "kafka-broker")
+kafka_bin_dir = os.path.join(kafka_home, "bin")
+
+# zookeeper
+zk_hosts = default("/clusterHostInfo/zookeeper_hosts", [])
+has_zk_host = not len(zk_hosts) == 0
+zookeeper_quorum = None
+if has_zk_host:
+ if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']:
+ zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort']
+ else:
+ zookeeper_clientPort = '2181'
+ zookeeper_quorum = (':' + zookeeper_clientPort + ',').join(config['clusterHostInfo']['zookeeper_hosts'])
+ # last port config
+ zookeeper_quorum += ':' + zookeeper_clientPort
+
+# Storm
+storm_rest_addr = status_params.storm_rest_addr
+
+# Kafka
+kafka_hosts = default("/clusterHostInfo/kafka_broker_hosts", [])
+has_kafka_host = not len(kafka_hosts) == 0
+kafka_brokers = None
+if has_kafka_host:
+ if 'port' in config['configurations']['kafka-broker']:
+ kafka_broker_port = config['configurations']['kafka-broker']['port']
+ else:
+ kafka_broker_port = '6667'
+ kafka_brokers = (':' + kafka_broker_port + ',').join(config['clusterHostInfo']['kafka_broker_hosts'])
+ kafka_brokers += ':' + kafka_broker_port
+
+metron_apps_dir = config['configurations']['metron-env']['metron_apps_hdfs_dir']
+metron_topic_retention = config['configurations']['metron-env']['metron_topic_retention']
+
+local_grok_patterns_dir = format("{metron_home}/patterns")
+hdfs_grok_patterns_dir = format("{metron_apps_dir}/patterns")
+
+# for create_hdfs_directory
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
+smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
+kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
+hdfs_site = config['configurations']['hdfs-site']
+default_fs = config['configurations']['core-site']['fs.defaultFS']
+dfs_type = default("/commandParams/dfs_type", "")
+
+# MYSQL
+if OSCheck.is_ubuntu_family():
+ mysql_configname = '/etc/mysql/my.cnf'
+else:
+ mysql_configname = '/etc/my.cnf'
+
+daemon_name = status_params.daemon_name
+# There will always be exactly one mysql_host
+mysql_host = config['clusterHostInfo']['metron_enrichment_mysql_server_hosts'][0]
+mysql_port = config['configurations']['metron-env']['metron_enrichment_db_port']
+
+mysql_adduser_path = tmp_dir + "/addMysqlUser.sh"
+mysql_deluser_path = tmp_dir + "/removeMysqlUser.sh"
+mysql_create_geoip_path = tmp_dir + "/createMysqlGeoIp.sh"
+
+enrichment_hosts = default("/clusterHostInfo/enrichment_host", [])
+enrichment_host = enrichment_hosts[0] if len(enrichment_hosts) > 0 else None
+
+enrichment_metron_user = config['configurations']['metron-env']['metron_enrichment_db_user']
+enrichment_metron_user_passwd = config['configurations']['metron-env']['metron_enrichment_db_password']
+enrichment_metron_user_passwd = unicode(enrichment_metron_user_passwd) if not is_empty(
+ enrichment_metron_user_passwd) else enrichment_metron_user_passwd
+mysql_process_name = status_params.mysql_process_name
+
+# create partial functions with common arguments for every HdfsResource call
+# to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code
+HdfsResource = functools.partial(
+ HdfsResource,
+ user=hdfs_user,
+ hdfs_resource_ignore_file="/var/lib/ambari-agent/data/.hdfs_resource_ignore",
+ security_enabled=security_enabled,
+ keytab=hdfs_user_keytab,
+ kinit_path_local=kinit_path_local,
+ hadoop_bin_dir=hadoop_bin_dir,
+ hadoop_conf_dir=hadoop_conf_dir,
+ principal_name=hdfs_principal_name,
+ hdfs_site=hdfs_site,
+ default_fs=default_fs,
+ immutable_paths=get_not_managed_resources(),
+ dfs_type=dfs_type
+)
+
+# HBase
+enrichment_table = status_params.enrichment_table
+enrichment_cf = status_params.enrichment_cf
+threatintel_table = status_params.threatintel_table
+threatintel_cf = status_params.threatintel_cf
+
+metron_enrichment_topology = status_params.metron_enrichment_topology
+metron_enrichment_topic = status_params.metron_enrichment_topic
+
+# ES Templates
+bro_index_path = tmp_dir + "/bro_index.template"
+snort_index_path = tmp_dir + "/snort_index.template"
+yaf_index_path = tmp_dir + "/yaf_index.template"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params_windows.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params_windows.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params_windows.py
new file mode 100755
index 0000000..4d11b35
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params_windows.py
@@ -0,0 +1,20 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+raise NotImplementedError
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/status_params.py
new file mode 100644
index 0000000..a0310d2
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/status_params.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from ambari_commons import OSCheck
+from resource_management.libraries.functions import format
+from resource_management.libraries.script import Script
+
+config = Script.get_config()
+
+# Parsers
+parsers = config['configurations']['metron-env']['parsers']
+metron_home = config['configurations']['metron-env']['metron_home']
+metron_zookeeper_config_dir = config['configurations']['metron-env']['metron_zookeeper_config_dir']
+metron_zookeeper_config_path = format('{metron_home}/{metron_zookeeper_config_dir}')
+parsers_configured_flag_file = metron_zookeeper_config_path + '/../metron_parsers_configured'
+
+# Enrichment
+metron_enrichment_topology = 'enrichment'
+metron_enrichment_topic = 'enrichments'
+
+enrichment_table = 'enrichment'
+enrichment_cf = 't'
+threatintel_table = 'threatintel'
+threatintel_cf = 't'
+
+mysql_process_name = 'mysqld'
+if OSCheck.is_suse_family() or OSCheck.is_ubuntu_family():
+ daemon_name = 'mysql'
+else:
+ daemon_name = 'mysqld'
+
+# ing
+metron_indexing_topology = config['configurations']['metron-env']['metron_indexing_topology']
+indexing_configured_flag_file = metron_zookeeper_config_path + '/../metron_indexing_configured'
+
+# Enrichment
+enrichment_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_is_configured'
+
+# Storm
+storm_rest_addr = config['configurations']['metron-env']['storm_rest_addr']
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/parser_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/parser_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/parser_commands.py
new file mode 100755
index 0000000..07e36db
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/parser_commands.py
@@ -0,0 +1,208 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+import re
+import subprocess
+import time
+
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute, File
+
+import metron_service
+
+
+# Wrap major operations and functionality in this class
+class ParserCommands:
+ __params = None
+ __parser_list = None
+ __configured = False
+
+ def __init__(self, params):
+ if params is None:
+ raise ValueError("params argument is required for initialization")
+ self.__params = params
+ self.__parser_list = self.__get_parsers(params)
+ self.__configured = os.path.isfile(self.__params.parsers_configured_flag_file)
+
+ # get list of parsers
+ def __get_parsers(self, params):
+ return params.parsers.replace(' ', '').split(',')
+
+ def is_configured(self):
+ return self.__configured
+
+ def set_configured(self):
+ File(self.__params.parsers_configured_flag_file,
+ content="",
+ owner=self.__params.metron_user,
+ mode=0775)
+
+ def init_parsers(self):
+ Logger.info(
+ "Copying grok patterns from local directory '{0}' to HDFS '{1}'".format(self.__params.local_grok_patterns_dir,
+ self.__params.metron_apps_dir))
+ self.__params.HdfsResource(self.__params.metron_apps_dir,
+ type="directory",
+ action="create_on_execute",
+ owner=self.__params.metron_user,
+ mode=0775,
+ source=self.__params.local_grok_patterns_dir)
+
+ Logger.info("Done initializing parser configuration")
+
+ def get_parser_list(self):
+ return self.__parser_list
+
+ def setup_repo(self):
+ def local_repo():
+ Logger.info("Setting up local repo")
+ Execute("yum -y install createrepo")
+ Execute("createrepo /localrepo")
+ Execute("chmod -R o-w+r /localrepo")
+ Execute("echo \"[METRON-0.2.1BETA]\n"
+ "name=Metron 0.2.1BETA packages\n"
+ "baseurl=file:///localrepo\n"
+ "gpgcheck=0\n"
+ "enabled=1\" > /etc/yum.repos.d/local.repo")
+
+ def remote_repo():
+ print('Using remote repo')
+
+ yum_repo_types = {
+ 'local': local_repo,
+ 'remote': remote_repo
+ }
+ repo_type = self.__params.yum_repo_type
+ if repo_type in yum_repo_types:
+ yum_repo_types[repo_type]()
+ else:
+ raise ValueError("Unsupported repo type '{0}'".format(repo_type))
+
+ def init_kafka_topics(self):
+ Logger.info('Creating Kafka topics')
+ command_template = """{0}/kafka-topics.sh \
+ --zookeeper {1} \
+ --create \
+ --topic {2} \
+ --partitions {3} \
+ --replication-factor {4} \
+ --config retention.bytes={5}"""
+ num_partitions = 1
+ replication_factor = 1
+ retention_gigabytes = int(self.__params.metron_topic_retention)
+ retention_bytes = retention_gigabytes * 1024 * 1024 * 1024
+ Logger.info("Creating main topics for parsers")
+ for parser_name in self.get_parser_list():
+ Logger.info("Creating topic'{0}'".format(parser_name))
+ Execute(command_template.format(self.__params.kafka_bin_dir,
+ self.__params.zookeeper_quorum,
+ parser_name,
+ num_partitions,
+ replication_factor,
+ retention_bytes))
+ Logger.info("Creating topics for error handling")
+ Execute(command_template.format(self.__params.kafka_bin_dir,
+ self.__params.zookeeper_quorum,
+ "parser_invalid",
+ num_partitions,
+ replication_factor,
+ retention_bytes))
+ Execute(command_template.format(self.__params.kafka_bin_dir,
+ self.__params.zookeeper_quorum,
+ "parser_error",
+ num_partitions, replication_factor,
+ retention_bytes))
+ Logger.info("Done creating Kafka topics")
+
+ def start_parser_topologies(self):
+ Logger.info("Starting Metron parser topologies: {0}".format(self.get_parser_list()))
+ start_cmd_template = """{0}/bin/start_parser_topology.sh \
+ -k {1} \
+ -z {2} \
+ -s {3}"""
+ for parser in self.get_parser_list():
+ Logger.info('Starting ' + parser)
+ Execute(start_cmd_template.format(self.__params.metron_home, self.__params.kafka_brokers,
+ self.__params.zookeeper_quorum, parser))
+
+ Logger.info('Finished starting parser topologies')
+
+ def stop_parser_topologies(self):
+ Logger.info('Stopping parsers')
+ for parser in self.get_parser_list():
+ Logger.info('Stopping ' + parser)
+ stop_cmd = 'storm kill ' + parser
+ Execute(stop_cmd)
+ Logger.info('Done stopping parser topologies')
+
+ def restart_parser_topologies(self, env):
+ Logger.info('Restarting the parser topologies')
+ self.stop_parser_topologies()
+ attempt_count = 0
+ while self.topologies_running(env):
+ if attempt_count > 2:
+ raise Exception("Unable to kill topologies")
+ attempt_count += 1
+ time.sleep(10)
+ self.start_parser_topologies()
+ Logger.info('Done restarting the parser topologies')
+
+ def topologies_exist(self):
+ cmd_open = subprocess.Popen(["storm", "list"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ (stdout, stderr) = cmd_open.communicate()
+ stdout_lines = stdout.splitlines()
+ if stdout_lines:
+ status_lines = self.__get_status_lines(stdout_lines)
+ for parser in self.get_parser_list():
+ for line in status_lines:
+ items = re.sub('[\s]+', ' ', line).split()
+ if items and items[0] == parser:
+ return True
+ return False
+
+ def topologies_running(self, env):
+ env.set_params(self.__params)
+ all_running = True
+ topologies = metron_service.get_running_topologies()
+ for parser in self.get_parser_list():
+ parser_found = False
+ is_running = False
+ if parser in topologies:
+ parser_found = True
+ is_running = topologies[parser] in ['ACTIVE', 'REBALANCING']
+ all_running &= parser_found and is_running
+ return all_running
+
+ def __get_status_lines(self, lines):
+ status_lines = []
+ do_stat = False
+ skipped = 0
+ for line in lines:
+ if line.startswith("Topology_name"):
+ do_stat = True
+ if do_stat and skipped == 2:
+ status_lines += [line]
+ elif do_stat:
+ skipped += 1
+ return status_lines
+
+ def __is_running(self, status):
+ return status in ['ACTIVE', 'REBALANCING']
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/parser_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/parser_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/parser_master.py
new file mode 100755
index 0000000..3758873
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/parser_master.py
@@ -0,0 +1,89 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.core.logger import Logger
+from resource_management.libraries.script import Script
+
+import metron_service
+from parser_commands import ParserCommands
+
+
+class ParserMaster(Script):
+ def get_component_name(self):
+ # TODO add this at some point - currently will cause problems with hdp-select
+ # return "parser-master"
+ pass
+
+ def install(self, env):
+ from params import params
+ env.set_params(params)
+ commands = ParserCommands(params)
+ commands.setup_repo()
+ Logger.info('Install RPM packages')
+ self.install_packages(env)
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ from params import params
+ env.set_params(params)
+ metron_service.load_global_config(params)
+ commands = ParserCommands(params)
+ if not commands.is_configured():
+ commands.init_parsers()
+ commands.init_kafka_topics()
+ commands.set_configured()
+
+ def start(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ self.configure(env)
+ commands = ParserCommands(params)
+ commands.start_parser_topologies()
+
+ def stop(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ commands = ParserCommands(params)
+ commands.stop_parser_topologies()
+
+ def status(self, env):
+ from params import status_params
+ env.set_params(status_params)
+ commands = ParserCommands(status_params)
+ if not commands.topologies_running(env):
+ raise ComponentIsNotRunning()
+
+ def restart(self, env):
+ from params import params
+ env.set_params(params)
+ self.configure(env)
+ commands = ParserCommands(params)
+ commands.restart_parser_topologies(env)
+
+ def servicechecktest(self, env):
+ from params import params
+ env.set_params(params)
+ from service_check import ServiceCheck
+ service_check = ServiceCheck()
+ Logger.info('Service Check Test')
+ service_check.service_check(env)
+
+
+if __name__ == "__main__":
+ ParserMaster().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/service_check.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/service_check.py
new file mode 100755
index 0000000..7dd9dfb
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/service_check.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from __future__ import print_function
+
+from resource_management.libraries.script import Script
+
+from indexing_commands import IndexingCommands
+from parser_commands import ParserCommands
+
+
+class ServiceCheck(Script):
+ def service_check(self, env):
+ from params import params
+ parsercommands = ParserCommands(params)
+ indexingcommands = IndexingCommands(params)
+ all_found = parsercommands.topologies_running(env) and indexingcommands.is_topology_active(env)
+ if all_found:
+ exit(0)
+ else:
+ exit(1)
+
+
+if __name__ == "__main__":
+ ServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/templates/enrichment.properties.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/templates/enrichment.properties.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/templates/enrichment.properties.j2
new file mode 100755
index 0000000..bab2f52
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/templates/enrichment.properties.j2
@@ -0,0 +1,88 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+##### Kafka #####
+
+kafka.zk={{zookeeper_quorum}}
+kafka.broker={{kafka_brokers}}
+enrichment.output.topic=indexing
+
+##### MySQL #####
+
+mysql.ip={{mysql_host}}
+mysql.port={{mysql_port}}
+mysql.username={{enrichment_metron_user}}
+mysql.password={{enrichment_metron_user_passwd}}
+
+##### Metrics #####
+
+#reporters
+org.apache.metron.metrics.reporter.graphite=true
+org.apache.metron.metrics.reporter.console=false
+org.apache.metron.metrics.reporter.jmx=false
+
+#Graphite Addresses
+
+org.apache.metron.metrics.graphite.address=localhost
+org.apache.metron.metrics.graphite.port=2023
+
+#TelemetryParserBolt
+org.apache.metron.metrics.TelemetryParserBolt.acks=true
+org.apache.metron.metrics.TelemetryParserBolt.emits=true
+org.apache.metron.metrics.TelemetryParserBolt.fails=true
+
+
+#GenericEnrichmentBolt
+org.apache.metron.metrics.GenericEnrichmentBolt.acks=true
+org.apache.metron.metrics.GenericEnrichmentBolt.emits=true
+org.apache.metron.metrics.GenericEnrichmentBolt.fails=true
+
+
+#TelemetryIndexingBolt
+org.apache.metron.metrics.TelemetryIndexingBolt.acks=true
+org.apache.metron.metrics.TelemetryIndexingBolt.emits=true
+org.apache.metron.metrics.TelemetryIndexingBolt.fails=true
+
+##### Host Enrichment #####
+
+hbase.provider.impl=org.apache.metron.hbase.HTableProvider
+enrichment.simple.hbase.table={{enrichment_table}}
+enrichment.simple.hbase.cf={{enrichment_cf}}
+org.apache.metron.enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\
+{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},\
+{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}]
+
+
+##### HBase #####
+bolt.hbase.table.name=pcap
+bolt.hbase.table.fields=t:value
+bolt.hbase.table.key.tuple.field.name=key
+bolt.hbase.table.timestamp.tuple.field.name=timestamp
+bolt.hbase.enable.batching=false
+bolt.hbase.write.buffer.size.in.bytes=2000000
+bolt.hbase.durability=SKIP_WAL
+bolt.hbase.partitioner.region.info.refresh.interval.mins=60
+
+##### Threat Intel #####
+
+threat.intel.tracker.table={{threatintel_table}}
+threat.intel.tracker.cf={{threatintel_cf}}
+threat.intel.simple.hbase.table={{threatintel_table}}
+threat.intel.simple.hbase.cf={{threatintel_cf}}
+threat.intel.ip.table=
+threat.intel.ip.cf=
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/quicklinks/quicklinks.json
new file mode 100755
index 0000000..ee1b225
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/quicklinks/quicklinks.json
@@ -0,0 +1,28 @@
+{
+ "name": "default",
+ "description": "default quick links configuration",
+ "configuration": {
+ "protocol":
+ {
+ "type":"HTTP_ONLY"
+ },
+
+ "links": [
+ {
+ "name": "storm_ui",
+ "label": "Storm UI",
+ "requires_user_name": "false",
+ "component_name": "STORM_UI_SERVER",
+ "url":"%@://%@:%@/",
+ "port":{
+ "http_property": "ui.port",
+ "http_default_port": "8744",
+ "https_property": "ui.port",
+ "https_default_port": "8744",
+ "regex": "^(\\d+)$",
+ "site": "storm-site"
+ }
+ }
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json
index 99dec9b..303e84f 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json
@@ -62,7 +62,7 @@
},
{
"service_name" : "METRON",
- "service_version" : "0.2.0BETA",
+ "service_version" : "0.2.1BETA",
"applicable_stacks" : [
{
"stack_name" : "HDP",
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index 6b8f195..b0ef974 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -122,6 +122,7 @@ This package installs the Metron Parser files
%dir %{metron_home}/lib
%{metron_home}/bin/start_parser_topology.sh
%{metron_home}/config/zookeeper/parsers/bro.json
+%{metron_home}/config/zookeeper/parsers/jsonMap.json
%{metron_home}/config/zookeeper/parsers/snort.json
%{metron_home}/config/zookeeper/parsers/squid.json
%{metron_home}/config/zookeeper/parsers/websphere.json
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/docker/rpm-docker/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/rpm-docker/pom.xml b/metron-deployment/packaging/docker/rpm-docker/pom.xml
index 92414c2..30e2670 100644
--- a/metron-deployment/packaging/docker/rpm-docker/pom.xml
+++ b/metron-deployment/packaging/docker/rpm-docker/pom.xml
@@ -21,11 +21,11 @@
<artifactId>metron-rpm</artifactId>
<packaging>pom</packaging>
<name>metron-rpm</name>
- <version>0.2.0BETA</version>
+ <version>0.2.1BETA</version>
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-deployment</artifactId>
- <version>0.2.0BETA</version>
+ <version>0.2.1BETA</version>
<relativePath>../../..</relativePath>
</parent>
<description>RPM Builder for Apache Metron</description>
@@ -33,7 +33,7 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<rpm.docker.tag>rpm-docker</rpm.docker.tag>
- <metron_dir>${parent.relativePath}/..</metron_dir>
+ <metron_dir>${project.parent.relativePath}/..</metron_dir>
</properties>
<build>
<plugins>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/playbooks/docker_probe_install.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/playbooks/docker_probe_install.yml b/metron-deployment/playbooks/docker_probe_install.yml
index b23e72a..2ec087f 100644
--- a/metron-deployment/playbooks/docker_probe_install.yml
+++ b/metron-deployment/playbooks/docker_probe_install.yml
@@ -30,7 +30,7 @@
- hosts: sensors
vars:
- metron_version: 0.2.0BETA
+ metron_version: 0.2.1BETA
metron_directory: /usr/metron/{{ metron_version }}
bro_version: "2.4.1"
fixbuf_version: "1.7.1"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/pom.xml b/metron-deployment/pom.xml
index d4b05f1..717b1ce 100644
--- a/metron-deployment/pom.xml
+++ b/metron-deployment/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>Metron</artifactId>
- <version>0.2.0BETA</version>
+ <version>0.2.1BETA</version>
</parent>
<description>Building and deploying Metron</description>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/roles/metron_pcapservice/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_pcapservice/defaults/main.yml b/metron-deployment/roles/metron_pcapservice/defaults/main.yml
index 23770bd..5ab28b1 100644
--- a/metron-deployment/roles/metron_pcapservice/defaults/main.yml
+++ b/metron-deployment/roles/metron_pcapservice/defaults/main.yml
@@ -15,7 +15,7 @@
# limitations under the License.
#
---
-metron_version: 0.2.0BETA
+metron_version: 0.2.1BETA
metron_directory: /usr/metron/{{ metron_version }}
pcapservice_jar_name: metron-api-{{ metron_version }}.jar
pcapservice_jar_src: "{{ playbook_dir }}/../../metron-platform/metron-api/target/{{ pcapservice_jar_name }}"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-platform/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/README.md b/metron-platform/README.md
index cdb1c0b..360997c 100644
--- a/metron-platform/README.md
+++ b/metron-platform/README.md
@@ -17,7 +17,7 @@ limitations under the License.
# Current Build
-The latest build of metron-platform is 0.2.0BETA.
+The latest build of metron-platform is 0.2.1BETA.
We are still in the process of merging/porting additional features from our production code base into this open source release. This release will be followed by a number of additional beta releases until the port is complete. We will also work on getting additional documentation and user/developer guides to the community as soon as we can. At this time we offer no support for the beta software, but will try to respond to requests as promptly as we can.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-platform/elasticsearch-shaded/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/elasticsearch-shaded/pom.xml b/metron-platform/elasticsearch-shaded/pom.xml
index 7d6ce46..1e18375 100644
--- a/metron-platform/elasticsearch-shaded/pom.xml
+++ b/metron-platform/elasticsearch-shaded/pom.xml
@@ -17,7 +17,7 @@
<parent>
<artifactId>metron-platform</artifactId>
<groupId>org.apache.metron</groupId>
- <version>0.2.0BETA</version>
+ <version>0.2.1BETA</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>elasticsearch-shaded</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-platform/metron-api/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-api/pom.xml b/metron-platform/metron-api/pom.xml
index 5e5e8e1..d600bfd 100644
--- a/metron-platform/metron-api/pom.xml
+++ b/metron-platform/metron-api/pom.xml
@@ -17,7 +17,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.2.0BETA</version>
+ <version>0.2.1BETA</version>
</parent>
<artifactId>metron-api</artifactId>
<description>Metron API</description>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-platform/metron-common/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/pom.xml b/metron-platform/metron-common/pom.xml
index 7a31e84..c19a507 100644
--- a/metron-platform/metron-common/pom.xml
+++ b/metron-platform/metron-common/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.2.0BETA</version>
+ <version>0.2.1BETA</version>
</parent>
<artifactId>metron-common</artifactId>
<name>metron-common</name>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-platform/metron-data-management/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/pom.xml b/metron-platform/metron-data-management/pom.xml
index dd7a690..0263396 100644
--- a/metron-platform/metron-data-management/pom.xml
+++ b/metron-platform/metron-data-management/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.2.0BETA</version>
+ <version>0.2.1BETA</version>
</parent>
<artifactId>metron-data-management</artifactId>
<properties>
[3/4] incubator-metron git commit: METRON-398: Bump release version
to 0.2.1BETA in master (mmiklavc) closes apache/incubator-metron#280
Posted by mm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_service.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_service.py
deleted file mode 100644
index 57da2c7..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_service.py
+++ /dev/null
@@ -1,76 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-
-import json
-import subprocess
-
-from resource_management.core.logger import Logger
-from resource_management.core.resources.system import Directory, File
-from resource_management.core.resources.system import Execute
-from resource_management.core.source import InlineTemplate
-from resource_management.libraries.functions import format as ambari_format
-
-
-def init_config():
- Logger.info('Loading config into ZooKeeper')
- Execute(ambari_format(
- "{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_quorum}"),
- path=ambari_format("{java_home}/bin")
- )
-
-
-def get_running_topologies():
- Logger.info('Getting Running Storm Topologies from Storm REST Server')
-
- cmd = ambari_format('curl --max-time 3 {storm_rest_addr}/api/v1/topology/summary')
- proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
- (stdout, stderr) = proc.communicate()
-
- try:
- stormjson = json.loads(stdout)
- except ValueError:
- return {}
-
- topologiesDict = {}
-
- for topology in stormjson['topologies']:
- topologiesDict[topology['name']] = topology['status']
-
- Logger.info("Topologies: " + str(topologiesDict))
- return topologiesDict
-
-
-def load_global_config(params):
- Logger.info('Create Metron Local Config Directory')
- Logger.info("Configure Metron global.json")
-
- directories = [params.metron_zookeeper_config_path]
- Directory(directories,
- mode=0755,
- owner=params.metron_user,
- group=params.metron_group
- )
-
- File("{0}/global.json".format(params.metron_zookeeper_config_path),
- owner=params.metron_user,
- content=InlineTemplate(params.global_json_template)
- )
-
- File("{0}/elasticsearch.properties".format(params.metron_zookeeper_config_path + '/..'),
- owner=params.metron_user,
- content=InlineTemplate(params.global_properties_template))
-
- init_config()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_server.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_server.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_server.py
deleted file mode 100755
index 2fc363f..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_server.py
+++ /dev/null
@@ -1,61 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.libraries.script.script import Script
-
-import mysql_users
-from mysql_service import mysql_service
-from mysql_utils import mysql_configure
-
-
-class MysqlServer(Script):
- def install(self, env):
- self.install_packages(env)
- self.configure(env)
-
- def clean(self, env):
- from params import params
- env.set_params(params)
- mysql_users.mysql_deluser()
-
- def configure(self, env, upgrade_type=None, config_dir=None):
- from params import params
- env.set_params(params)
- mysql_configure()
-
- def start(self, env, rolling_restart=False):
- from params import params
- env.set_params(params)
- mysql_service(daemon_name=params.daemon_name, action='start')
-
- def stop(self, env, rolling_restart=False):
- from params import params
- env.set_params(params)
- mysql_service(daemon_name=params.daemon_name, action='stop')
-
- def status(self, env):
- from params import status_params
- env.set_params(status_params)
-
- mysql_service(daemon_name=status_params.daemon_name, action='status')
-
-
-if __name__ == "__main__":
- MysqlServer().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_service.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_service.py
deleted file mode 100755
index 2e0ce8b..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_service.py
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.core.exceptions import ComponentIsNotRunning, Fail
-from resource_management.core.resources.system import Execute
-from resource_management.libraries.functions.format import format
-
-
-def mysql_service(daemon_name=None, action='start'):
- status_cmd = format("pgrep -l '^{mysql_process_name}$'")
- cmd = ('service', daemon_name, action)
-
- if action == 'status':
- try:
- Execute(status_cmd)
- except Fail:
- raise ComponentIsNotRunning()
- elif action == 'stop':
- Execute(cmd,
- logoutput=True,
- only_if=status_cmd,
- sudo=True,
- )
- elif action == 'start':
- Execute(cmd,
- logoutput=True,
- not_if=status_cmd,
- sudo=True,
- )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_users.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_users.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_users.py
deleted file mode 100755
index 1721eba..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_users.py
+++ /dev/null
@@ -1,58 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.core.resources.system import Execute, File
-from resource_management.core.source import StaticFile
-from resource_management.libraries.functions.format import format
-
-
-# Used to add metron access to the needed components
-def mysql_adduser():
- from params import params
-
- File(params.mysql_adduser_path,
- mode=0755,
- content=StaticFile('addMysqlUser.sh')
- )
-
- add_user_cmd = format("bash -x {mysql_adduser_path} {daemon_name} {metron_user} {enrichment_metron_user_passwd!p} {enrichment_host}")
- Execute(add_user_cmd,
- tries=3,
- try_sleep=5,
- logoutput=False,
- path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'
- )
-
-
-# Removes hive metron from components
-def mysql_deluser():
- from params import params
-
- File(params.mysql_deluser_path,
- mode=0755,
- content=StaticFile('removeMysqlUser.sh')
- )
-
- del_user_cmd = format("bash -x {mysql_deluser_path} {daemon_name} {metron_user} {enrichment_host}")
- Execute(del_user_cmd,
- tries=3,
- try_sleep=5,
- path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
- )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_utils.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_utils.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_utils.py
deleted file mode 100755
index 40b925d..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_utils.py
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.core.resources.system import Execute, File
-from resource_management.core.source import StaticFile
-from resource_management.libraries.functions.format import format
-
-import mysql_users
-
-
-def mysql_configure():
- from params import params
-
- # required for running hive
- replace_bind_address = ('sed', '-i', 's|^bind-address[ \t]*=.*|bind-address = 0.0.0.0|', params.mysql_configname)
- Execute(replace_bind_address,
- sudo=True,
- )
-
- # this also will start mysql-server
- mysql_users.mysql_adduser()
-
- File(params.mysql_create_geoip_path,
- mode=0755,
- content=StaticFile('createMysqlGeoIp.sh')
- )
-
- geoip_setup_cmd = format("bash -x {mysql_create_geoip_path} {daemon_name} {geoip_ddl} {geoip_url}")
- Execute(geoip_setup_cmd,
- tries=3,
- try_sleep=5,
- path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
- )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/__init__.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/__init__.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/__init__.py
deleted file mode 100755
index 242460e..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/__init__.py
+++ /dev/null
@@ -1,18 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params.py
deleted file mode 100755
index 953435d..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params.py
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-from ambari_commons import OSCheck
-from resource_management.libraries.functions.default import default
-from resource_management.libraries.functions.expect import expect
-
-if OSCheck.is_windows_family():
- from params_windows import *
-else:
- from params_linux import *
-
-java_home = config['hostLevelParams']['java_home']
-java_version = expect("/hostLevelParams/java_version", int)
-
-host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py
deleted file mode 100755
index 5f914f3..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py
+++ /dev/null
@@ -1,169 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-import functools
-import os
-
-from ambari_commons.os_check import OSCheck
-from resource_management.libraries.functions import conf_select
-from resource_management.libraries.functions import format
-from resource_management.libraries.functions import get_kinit_path
-from resource_management.libraries.functions import stack_select
-from resource_management.libraries.functions.default import default
-from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
-from resource_management.libraries.functions.is_empty import is_empty
-from resource_management.libraries.resources.hdfs_resource import HdfsResource
-from resource_management.libraries.script import Script
-
-import status_params
-
-# server configurations
-config = Script.get_config()
-tmp_dir = Script.get_tmp_dir()
-
-hostname = config['hostname']
-metron_home = status_params.metron_home
-parsers = status_params.parsers
-metron_ddl_dir = metron_home + '/ddl'
-geoip_ddl = metron_ddl_dir + '/geoip_ddl.sql'
-geoip_url = config['configurations']['metron-env']['geoip_url']
-metron_indexing_topology = status_params.metron_indexing_topology
-metron_user = config['configurations']['metron-env']['metron_user']
-metron_group = config['configurations']['metron-env']['metron_group']
-metron_config_path = metron_home + '/config'
-metron_zookeeper_config_dir = status_params.metron_zookeeper_config_dir
-metron_zookeeper_config_path = status_params.metron_zookeeper_config_path
-parsers_configured_flag_file = status_params.parsers_configured_flag_file
-enrichment_configured_flag_file = status_params.enrichment_configured_flag_file
-indexing_configured_flag_file = status_params.indexing_configured_flag_file
-global_json_template = config['configurations']['metron-env']['global-json']
-global_properties_template = config['configurations']['metron-env']['elasticsearch-properties']
-es_cluster_name = config['configurations']['metron-env']['es_cluster_name']
-es_url = config['configurations']['metron-env']['es_url']
-yum_repo_type = 'local'
-
-# hadoop params
-stack_root = Script.get_stack_root()
-hadoop_home_dir = stack_select.get_hadoop_dir("home")
-hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
-hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
-kafka_home = os.path.join(stack_root, "current", "kafka-broker")
-kafka_bin_dir = os.path.join(kafka_home, "bin")
-
-# zookeeper
-zk_hosts = default("/clusterHostInfo/zookeeper_hosts", [])
-has_zk_host = not len(zk_hosts) == 0
-zookeeper_quorum = None
-if has_zk_host:
- if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']:
- zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort']
- else:
- zookeeper_clientPort = '2181'
- zookeeper_quorum = (':' + zookeeper_clientPort + ',').join(config['clusterHostInfo']['zookeeper_hosts'])
- # last port config
- zookeeper_quorum += ':' + zookeeper_clientPort
-
-# Storm
-storm_rest_addr = status_params.storm_rest_addr
-
-# Kafka
-kafka_hosts = default("/clusterHostInfo/kafka_broker_hosts", [])
-has_kafka_host = not len(kafka_hosts) == 0
-kafka_brokers = None
-if has_kafka_host:
- if 'port' in config['configurations']['kafka-broker']:
- kafka_broker_port = config['configurations']['kafka-broker']['port']
- else:
- kafka_broker_port = '6667'
- kafka_brokers = (':' + kafka_broker_port + ',').join(config['clusterHostInfo']['kafka_broker_hosts'])
- kafka_brokers += ':' + kafka_broker_port
-
-metron_apps_dir = config['configurations']['metron-env']['metron_apps_hdfs_dir']
-metron_topic_retention = config['configurations']['metron-env']['metron_topic_retention']
-
-local_grok_patterns_dir = format("{metron_home}/patterns")
-hdfs_grok_patterns_dir = format("{metron_apps_dir}/patterns")
-
-# for create_hdfs_directory
-security_enabled = config['configurations']['cluster-env']['security_enabled']
-hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
-hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
-hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
-smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
-kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
-hdfs_site = config['configurations']['hdfs-site']
-default_fs = config['configurations']['core-site']['fs.defaultFS']
-dfs_type = default("/commandParams/dfs_type", "")
-
-# MYSQL
-if OSCheck.is_ubuntu_family():
- mysql_configname = '/etc/mysql/my.cnf'
-else:
- mysql_configname = '/etc/my.cnf'
-
-daemon_name = status_params.daemon_name
-# There will always be exactly one mysql_host
-mysql_host = config['clusterHostInfo']['metron_enrichment_mysql_server_hosts'][0]
-mysql_port = config['configurations']['metron-env']['metron_enrichment_db_port']
-
-mysql_adduser_path = tmp_dir + "/addMysqlUser.sh"
-mysql_deluser_path = tmp_dir + "/removeMysqlUser.sh"
-mysql_create_geoip_path = tmp_dir + "/createMysqlGeoIp.sh"
-
-enrichment_hosts = default("/clusterHostInfo/enrichment_host", [])
-enrichment_host = enrichment_hosts[0] if len(enrichment_hosts) > 0 else None
-
-enrichment_metron_user = config['configurations']['metron-env']['metron_enrichment_db_user']
-enrichment_metron_user_passwd = config['configurations']['metron-env']['metron_enrichment_db_password']
-enrichment_metron_user_passwd = unicode(enrichment_metron_user_passwd) if not is_empty(
- enrichment_metron_user_passwd) else enrichment_metron_user_passwd
-mysql_process_name = status_params.mysql_process_name
-
-# create partial functions with common arguments for every HdfsResource call
-# to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code
-HdfsResource = functools.partial(
- HdfsResource,
- user=hdfs_user,
- hdfs_resource_ignore_file="/var/lib/ambari-agent/data/.hdfs_resource_ignore",
- security_enabled=security_enabled,
- keytab=hdfs_user_keytab,
- kinit_path_local=kinit_path_local,
- hadoop_bin_dir=hadoop_bin_dir,
- hadoop_conf_dir=hadoop_conf_dir,
- principal_name=hdfs_principal_name,
- hdfs_site=hdfs_site,
- default_fs=default_fs,
- immutable_paths=get_not_managed_resources(),
- dfs_type=dfs_type
-)
-
-# HBase
-enrichment_table = status_params.enrichment_table
-enrichment_cf = status_params.enrichment_cf
-threatintel_table = status_params.threatintel_table
-threatintel_cf = status_params.threatintel_cf
-
-metron_enrichment_topology = status_params.metron_enrichment_topology
-metron_enrichment_topic = status_params.metron_enrichment_topic
-
-# ES Templates
-bro_index_path = tmp_dir + "/bro_index.template"
-snort_index_path = tmp_dir + "/snort_index.template"
-yaf_index_path = tmp_dir + "/yaf_index.template"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_windows.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_windows.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_windows.py
deleted file mode 100755
index 4d11b35..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_windows.py
+++ /dev/null
@@ -1,20 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-raise NotImplementedError
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/status_params.py
deleted file mode 100644
index a0310d2..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/status_params.py
+++ /dev/null
@@ -1,57 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from ambari_commons import OSCheck
-from resource_management.libraries.functions import format
-from resource_management.libraries.script import Script
-
-config = Script.get_config()
-
-# Parsers
-parsers = config['configurations']['metron-env']['parsers']
-metron_home = config['configurations']['metron-env']['metron_home']
-metron_zookeeper_config_dir = config['configurations']['metron-env']['metron_zookeeper_config_dir']
-metron_zookeeper_config_path = format('{metron_home}/{metron_zookeeper_config_dir}')
-parsers_configured_flag_file = metron_zookeeper_config_path + '/../metron_parsers_configured'
-
-# Enrichment
-metron_enrichment_topology = 'enrichment'
-metron_enrichment_topic = 'enrichments'
-
-enrichment_table = 'enrichment'
-enrichment_cf = 't'
-threatintel_table = 'threatintel'
-threatintel_cf = 't'
-
-mysql_process_name = 'mysqld'
-if OSCheck.is_suse_family() or OSCheck.is_ubuntu_family():
- daemon_name = 'mysql'
-else:
- daemon_name = 'mysqld'
-
-# ing
-metron_indexing_topology = config['configurations']['metron-env']['metron_indexing_topology']
-indexing_configured_flag_file = metron_zookeeper_config_path + '/../metron_indexing_configured'
-
-# Enrichment
-enrichment_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_is_configured'
-
-# Storm
-storm_rest_addr = config['configurations']['metron-env']['storm_rest_addr']
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py
deleted file mode 100755
index b3fb809..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py
+++ /dev/null
@@ -1,208 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-import os
-import re
-import subprocess
-import time
-
-from resource_management.core.logger import Logger
-from resource_management.core.resources.system import Execute, File
-
-import metron_service
-
-
-# Wrap major operations and functionality in this class
-class ParserCommands:
- __params = None
- __parser_list = None
- __configured = False
-
- def __init__(self, params):
- if params is None:
- raise ValueError("params argument is required for initialization")
- self.__params = params
- self.__parser_list = self.__get_parsers(params)
- self.__configured = os.path.isfile(self.__params.parsers_configured_flag_file)
-
- # get list of parsers
- def __get_parsers(self, params):
- return params.parsers.replace(' ', '').split(',')
-
- def is_configured(self):
- return self.__configured
-
- def set_configured(self):
- File(self.__params.parsers_configured_flag_file,
- content="",
- owner=self.__params.metron_user,
- mode=0775)
-
- def init_parsers(self):
- Logger.info(
- "Copying grok patterns from local directory '{0}' to HDFS '{1}'".format(self.__params.local_grok_patterns_dir,
- self.__params.metron_apps_dir))
- self.__params.HdfsResource(self.__params.metron_apps_dir,
- type="directory",
- action="create_on_execute",
- owner=self.__params.metron_user,
- mode=0775,
- source=self.__params.local_grok_patterns_dir)
-
- Logger.info("Done initializing parser configuration")
-
- def get_parser_list(self):
- return self.__parser_list
-
- def setup_repo(self):
- def local_repo():
- Logger.info("Setting up local repo")
- Execute("yum -y install createrepo")
- Execute("createrepo /localrepo")
- Execute("chmod -R o-w+r /localrepo")
- Execute("echo \"[METRON-0.2.0BETA]\n"
- "name=Metron 0.2.0BETA packages\n"
- "baseurl=file:///localrepo\n"
- "gpgcheck=0\n"
- "enabled=1\" > /etc/yum.repos.d/local.repo")
-
- def remote_repo():
- print('Using remote repo')
-
- yum_repo_types = {
- 'local': local_repo,
- 'remote': remote_repo
- }
- repo_type = self.__params.yum_repo_type
- if repo_type in yum_repo_types:
- yum_repo_types[repo_type]()
- else:
- raise ValueError("Unsupported repo type '{0}'".format(repo_type))
-
- def init_kafka_topics(self):
- Logger.info('Creating Kafka topics')
- command_template = """{0}/kafka-topics.sh \
- --zookeeper {1} \
- --create \
- --topic {2} \
- --partitions {3} \
- --replication-factor {4} \
- --config retention.bytes={5}"""
- num_partitions = 1
- replication_factor = 1
- retention_gigabytes = int(self.__params.metron_topic_retention)
- retention_bytes = retention_gigabytes * 1024 * 1024 * 1024
- Logger.info("Creating main topics for parsers")
- for parser_name in self.get_parser_list():
- Logger.info("Creating topic'{0}'".format(parser_name))
- Execute(command_template.format(self.__params.kafka_bin_dir,
- self.__params.zookeeper_quorum,
- parser_name,
- num_partitions,
- replication_factor,
- retention_bytes))
- Logger.info("Creating topics for error handling")
- Execute(command_template.format(self.__params.kafka_bin_dir,
- self.__params.zookeeper_quorum,
- "parser_invalid",
- num_partitions,
- replication_factor,
- retention_bytes))
- Execute(command_template.format(self.__params.kafka_bin_dir,
- self.__params.zookeeper_quorum,
- "parser_error",
- num_partitions, replication_factor,
- retention_bytes))
- Logger.info("Done creating Kafka topics")
-
- def start_parser_topologies(self):
- Logger.info("Starting Metron parser topologies: {0}".format(self.get_parser_list()))
- start_cmd_template = """{0}/bin/start_parser_topology.sh \
- -k {1} \
- -z {2} \
- -s {3}"""
- for parser in self.get_parser_list():
- Logger.info('Starting ' + parser)
- Execute(start_cmd_template.format(self.__params.metron_home, self.__params.kafka_brokers,
- self.__params.zookeeper_quorum, parser))
-
- Logger.info('Finished starting parser topologies')
-
- def stop_parser_topologies(self):
- Logger.info('Stopping parsers')
- for parser in self.get_parser_list():
- Logger.info('Stopping ' + parser)
- stop_cmd = 'storm kill ' + parser
- Execute(stop_cmd)
- Logger.info('Done stopping parser topologies')
-
- def restart_parser_topologies(self, env):
- Logger.info('Restarting the parser topologies')
- self.stop_parser_topologies()
- attempt_count = 0
- while self.topologies_running(env):
- if attempt_count > 2:
- raise Exception("Unable to kill topologies")
- attempt_count += 1
- time.sleep(10)
- self.start_parser_topologies()
- Logger.info('Done restarting the parser topologies')
-
- def topologies_exist(self):
- cmd_open = subprocess.Popen(["storm", "list"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- (stdout, stderr) = cmd_open.communicate()
- stdout_lines = stdout.splitlines()
- if stdout_lines:
- status_lines = self.__get_status_lines(stdout_lines)
- for parser in self.get_parser_list():
- for line in status_lines:
- items = re.sub('[\s]+', ' ', line).split()
- if items and items[0] == parser:
- return True
- return False
-
- def topologies_running(self, env):
- env.set_params(self.__params)
- all_running = True
- topologies = metron_service.get_running_topologies()
- for parser in self.get_parser_list():
- parser_found = False
- is_running = False
- if parser in topologies:
- parser_found = True
- is_running = topologies[parser] in ['ACTIVE', 'REBALANCING']
- all_running &= parser_found and is_running
- return all_running
-
- def __get_status_lines(self, lines):
- status_lines = []
- do_stat = False
- skipped = 0
- for line in lines:
- if line.startswith("Topology_name"):
- do_stat = True
- if do_stat and skipped == 2:
- status_lines += [line]
- elif do_stat:
- skipped += 1
- return status_lines
-
- def __is_running(self, status):
- return status in ['ACTIVE', 'REBALANCING']
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py
deleted file mode 100755
index 3758873..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py
+++ /dev/null
@@ -1,89 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.core.exceptions import ComponentIsNotRunning
-from resource_management.core.logger import Logger
-from resource_management.libraries.script import Script
-
-import metron_service
-from parser_commands import ParserCommands
-
-
-class ParserMaster(Script):
- def get_component_name(self):
- # TODO add this at some point - currently will cause problems with hdp-select
- # return "parser-master"
- pass
-
- def install(self, env):
- from params import params
- env.set_params(params)
- commands = ParserCommands(params)
- commands.setup_repo()
- Logger.info('Install RPM packages')
- self.install_packages(env)
-
- def configure(self, env, upgrade_type=None, config_dir=None):
- from params import params
- env.set_params(params)
- metron_service.load_global_config(params)
- commands = ParserCommands(params)
- if not commands.is_configured():
- commands.init_parsers()
- commands.init_kafka_topics()
- commands.set_configured()
-
- def start(self, env, upgrade_type=None):
- from params import params
- env.set_params(params)
- self.configure(env)
- commands = ParserCommands(params)
- commands.start_parser_topologies()
-
- def stop(self, env, upgrade_type=None):
- from params import params
- env.set_params(params)
- commands = ParserCommands(params)
- commands.stop_parser_topologies()
-
- def status(self, env):
- from params import status_params
- env.set_params(status_params)
- commands = ParserCommands(status_params)
- if not commands.topologies_running(env):
- raise ComponentIsNotRunning()
-
- def restart(self, env):
- from params import params
- env.set_params(params)
- self.configure(env)
- commands = ParserCommands(params)
- commands.restart_parser_topologies(env)
-
- def servicechecktest(self, env):
- from params import params
- env.set_params(params)
- from service_check import ServiceCheck
- service_check = ServiceCheck()
- Logger.info('Service Check Test')
- service_check.service_check(env)
-
-
-if __name__ == "__main__":
- ParserMaster().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/service_check.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/service_check.py
deleted file mode 100755
index 7dd9dfb..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/service_check.py
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-from __future__ import print_function
-
-from resource_management.libraries.script import Script
-
-from indexing_commands import IndexingCommands
-from parser_commands import ParserCommands
-
-
-class ServiceCheck(Script):
- def service_check(self, env):
- from params import params
- parsercommands = ParserCommands(params)
- indexingcommands = IndexingCommands(params)
- all_found = parsercommands.topologies_running(env) and indexingcommands.is_topology_active(env)
- if all_found:
- exit(0)
- else:
- exit(1)
-
-
-if __name__ == "__main__":
- ServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/templates/enrichment.properties.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/templates/enrichment.properties.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/templates/enrichment.properties.j2
deleted file mode 100755
index bab2f52..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/templates/enrichment.properties.j2
+++ /dev/null
@@ -1,88 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-##### Kafka #####
-
-kafka.zk={{zookeeper_quorum}}
-kafka.broker={{kafka_brokers}}
-enrichment.output.topic=indexing
-
-##### MySQL #####
-
-mysql.ip={{mysql_host}}
-mysql.port={{mysql_port}}
-mysql.username={{enrichment_metron_user}}
-mysql.password={{enrichment_metron_user_passwd}}
-
-##### Metrics #####
-
-#reporters
-org.apache.metron.metrics.reporter.graphite=true
-org.apache.metron.metrics.reporter.console=false
-org.apache.metron.metrics.reporter.jmx=false
-
-#Graphite Addresses
-
-org.apache.metron.metrics.graphite.address=localhost
-org.apache.metron.metrics.graphite.port=2023
-
-#TelemetryParserBolt
-org.apache.metron.metrics.TelemetryParserBolt.acks=true
-org.apache.metron.metrics.TelemetryParserBolt.emits=true
-org.apache.metron.metrics.TelemetryParserBolt.fails=true
-
-
-#GenericEnrichmentBolt
-org.apache.metron.metrics.GenericEnrichmentBolt.acks=true
-org.apache.metron.metrics.GenericEnrichmentBolt.emits=true
-org.apache.metron.metrics.GenericEnrichmentBolt.fails=true
-
-
-#TelemetryIndexingBolt
-org.apache.metron.metrics.TelemetryIndexingBolt.acks=true
-org.apache.metron.metrics.TelemetryIndexingBolt.emits=true
-org.apache.metron.metrics.TelemetryIndexingBolt.fails=true
-
-##### Host Enrichment #####
-
-hbase.provider.impl=org.apache.metron.hbase.HTableProvider
-enrichment.simple.hbase.table={{enrichment_table}}
-enrichment.simple.hbase.cf={{enrichment_cf}}
-org.apache.metron.enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\
-{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},\
-{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}]
-
-
-##### HBase #####
-bolt.hbase.table.name=pcap
-bolt.hbase.table.fields=t:value
-bolt.hbase.table.key.tuple.field.name=key
-bolt.hbase.table.timestamp.tuple.field.name=timestamp
-bolt.hbase.enable.batching=false
-bolt.hbase.write.buffer.size.in.bytes=2000000
-bolt.hbase.durability=SKIP_WAL
-bolt.hbase.partitioner.region.info.refresh.interval.mins=60
-
-##### Threat Intel #####
-
-threat.intel.tracker.table={{threatintel_table}}
-threat.intel.tracker.cf={{threatintel_cf}}
-threat.intel.simple.hbase.table={{threatintel_table}}
-threat.intel.simple.hbase.cf={{threatintel_cf}}
-threat.intel.ip.table=
-threat.intel.ip.cf=
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/quicklinks/quicklinks.json
deleted file mode 100755
index ee1b225..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/quicklinks/quicklinks.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
- "name": "default",
- "description": "default quick links configuration",
- "configuration": {
- "protocol":
- {
- "type":"HTTP_ONLY"
- },
-
- "links": [
- {
- "name": "storm_ui",
- "label": "Storm UI",
- "requires_user_name": "false",
- "component_name": "STORM_UI_SERVER",
- "url":"%@://%@:%@/",
- "port":{
- "http_property": "ui.port",
- "http_default_port": "8744",
- "https_property": "ui.port",
- "https_default_port": "8744",
- "regex": "^(\\d+)$",
- "site": "storm-site"
- }
- }
- ]
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/configuration/metron-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/configuration/metron-env.xml
new file mode 100644
index 0000000..3ab9fe5
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/configuration/metron-env.xml
@@ -0,0 +1,174 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration supports_final="true">
+ <property>
+ <name>metron_home</name>
+ <value>/usr/metron/0.2.1BETA</value>
+ <description>Metron home directory</description>
+ <display-name>Metron home</display-name>
+ </property>
+ <property>
+ <name>metron_apps_hdfs_dir</name>
+ <value>/apps/metron</value>
+ <description>Metron apps HDFS dir</description>
+ <display-name>Metron apps HDFS dir</display-name>
+ </property>
+ <property>
+ <name>metron_zookeeper_config_dir</name>
+ <value>config/zookeeper</value>
+ <description>Metron Zookeeper config dir. Relative path to Metron home.</description>
+ <display-name>Metron Zookeeper config dir</display-name>
+ </property>
+ <property>
+ <name>metron_user</name>
+ <value>metron</value>
+ <property-type>USER</property-type>
+ <description>The user for Metron</description>
+ <display-name>Metron User</display-name>
+ </property>
+ <property>
+ <name>metron_group</name>
+ <value>metron</value>
+ <property-type>GROUP</property-type>
+ <description>The group for Metron</description>
+ </property>
+ <property>
+ <name>metron_topic_retention</name>
+ <description>Kafka Retention in GB</description>
+ <value>10</value>
+ </property>
+ <property>
+ <name>parsers</name>
+ <value>bro,snort,yaf</value>
+ <description>Metron parsers to deploy</description>
+ <display-name>Metron parsers</display-name>
+ </property>
+ <property>
+ <name>metron_enrichment_db_user</name>
+ <value>metron</value>
+ <description>Database username to use to connect to the database.</description>
+ </property>
+ <property>
+ <name>metron_enrichment_db_port</name>
+ <value>3306</value>
+ <description>Database port to use to connect to the database.</description>
+ </property>
+ <property>
+ <name>metron_enrichment_db_password</name>
+ <value></value>
+ <property-type>PASSWORD</property-type>
+ <display-name>Metron Enrichment Database Password</display-name>
+ <description>Password to use against database</description>
+ <value-attributes>
+ <type>password</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>metron_indexing_topology</name>
+ <value>indexing</value>
+ <description>The Storm topology name for Indexing</description>
+ </property>
+ <property>
+ <name>es_cluster_name</name>
+ <value>metron</value>
+ <description>Name of Elasticsearch Cluster</description>
+ </property>
+ <property>
+ <name>geoip_url</name>
+ <value>http://geolite.maxmind.com/download/geoip/database/GeoLiteCity_CSV/GeoLiteCity-latest.tar.xz</value>
+ <description>Location of the GeoIP data to load.</description>
+ </property>
+ <property require-input="true">
+ <name>es_url</name>
+ <value></value>
+ <description>Comma delimited list of Elasticsearch URLs. (eshost1:9300,eshost2:9300)</description>
+ </property>
+ <property>
+ <name>storm_rest_addr</name>
+ <!--<value-attributes>-->
+ <!--<editable-only-at-install>true</editable-only-at-install>-->
+ <!--<overridable>false</overridable>-->
+ <!--</value-attributes>-->
+ <value></value>
+ </property>
+ <property>
+ <name>global-json</name>
+ <display-name>global.json template</display-name>
+ <description>This is the jinja template for global.json file</description>
+ <value>
+{
+"es.clustername": "{{ es_cluster_name }}",
+"es.ip": "{{ es_url }}",
+"es.date.format": "yyyy.MM.dd.HH"
+}
+ </value>
+ <value-attributes>
+ <type>content</type>
+ </value-attributes>
+ </property>
+ <property>
+ <name>elasticsearch-properties</name>
+ <description>The template for the elasticsearch.properties file.</description>
+ <display-name>elasticsearch.properties template</display-name>
+ <value>
+##### Storm #####
+indexing.workers=1
+indexing.executors=0
+##### Kafka #####
+kafka.zk={{ zookeeper_quorum }}
+kafka.broker={{ kafka_brokers }}
+kafka.start=WHERE_I_LEFT_OFF
+##### Indexing #####
+index.input.topic=indexing
+index.error.topic=indexing_error
+writer.class.name=org.apache.metron.elasticsearch.writer.ElasticsearchWriter
+##### Metrics #####
+#reporters
+org.apache.metron.metrics.reporter.graphite=true
+org.apache.metron.metrics.reporter.console=false
+org.apache.metron.metrics.reporter.jmx=false
+#Graphite Addresses
+org.apache.metron.metrics.graphite.address=localhost
+org.apache.metron.metrics.graphite.port=2023
+#TelemetryParserBolt
+org.apache.metron.metrics.TelemetryParserBolt.acks=true
+org.apache.metron.metrics.TelemetryParserBolt.emits=true
+org.apache.metron.metrics.TelemetryParserBolt.fails=true
+##### HDFS #####
+bolt.hdfs.batch.size=5000
+bolt.hdfs.field.delimiter=|
+bolt.hdfs.rotation.policy=org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy
+bolt.hdfs.rotation.policy.units=DAYS
+bolt.hdfs.rotation.policy.count=1
+bolt.hdfs.file.rotation.size.in.mb=5
+bolt.hdfs.file.system.url={{ default_fs }}
+bolt.hdfs.wip.file.path=/paloalto/wip
+bolt.hdfs.finished.file.path=/paloalto/rotated
+bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec
+index.hdfs.output=/tmp/metron/enriched
+ </value>
+ <value-attributes>
+ <type>content</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/metainfo.xml
new file mode 100644
index 0000000..d5cbd26
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/metainfo.xml
@@ -0,0 +1,273 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>METRON</name>
+ <displayName>Metron</displayName>
+ <comment>A scalable advanced security analytics framework built on Hadoop</comment>
+ <version>0.2.1BETA</version>
+ <components>
+
+ <component>
+ <name>METRON_PARSERS</name>
+ <displayName>Metron Parsers</displayName>
+ <category>MASTER</category>
+ <cardinality>1</cardinality>
+ <versionAdvertised>true</versionAdvertised>
+ <reassignAllowed>false</reassignAllowed>
+ <clientsToUpdateConfigs></clientsToUpdateConfigs>
+ <dependencies>
+ <dependency>
+ <name>HDFS/HDFS_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
+ <scope>cluster</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>STORM/SUPERVISOR</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>KAFKA/KAFKA_BROKER</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ <commandScript>
+ <script>scripts/parser_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ <configuration-dependencies>
+ <config-type>metron-parsers</config-type>
+ </configuration-dependencies>
+ </component>
+
+ <component>
+ <name>METRON_ENRICHMENT_MASTER</name>
+ <displayName>Metron Enrichment</displayName>
+ <category>MASTER</category>
+ <cardinality>1</cardinality>
+ <versionAdvertised>true</versionAdvertised>
+ <dependencies>
+ <dependency>
+ <name>HDFS/HDFS_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
+ <scope>cluster</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>KAFKA/KAFKA_BROKER</name>
+ <scope>cluster</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ <commandScript>
+ <script>scripts/enrichment_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>METRON_ENRICHMENT_MYSQL_SERVER</name>
+ <displayName>MySQL Server</displayName>
+ <category>MASTER</category>
+ <cardinality>1</cardinality>
+ <versionAdvertised>false</versionAdvertised>
+ <clientsToUpdateConfigs></clientsToUpdateConfigs>
+ <commandScript>
+ <script>scripts/mysql_server.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ <customCommands>
+ <customCommand>
+ <name>CLEAN</name>
+ <commandScript>
+ <script>scripts/mysql_server.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </customCommand>
+ </customCommands>
+ </component>
+
+ <component>
+ <name>METRON_INDEXING</name>
+ <displayName>Metron Indexing</displayName>
+ <category>MASTER</category>
+ <cardinality>1</cardinality>
+ <versionAdvertised>true</versionAdvertised>
+ <reassignAllowed>false</reassignAllowed>
+ <clientsToUpdateConfigs></clientsToUpdateConfigs>
+ <dependencies>
+ <dependency>
+ <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
+ <scope>cluster</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>STORM/SUPERVISOR</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>KAFKA/KAFKA_BROKER</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ <commandScript>
+ <script>scripts/indexing_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ <customCommands>
+ <customCommand>
+ <name>ELASTICSEARCH_TEMPLATE_INSTALL</name>
+ <commandScript>
+ <script>scripts/indexing_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </customCommand>
+ <customCommand>
+ <name>ELASTICSEARCH_TEMPLATE_DELETE</name>
+ <commandScript>
+ <script>scripts/indexing_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </customCommand>
+ </customCommands>
+ <configuration-dependencies>
+ <config-type>metron-indexing</config-type>
+ </configuration-dependencies>
+ </component>
+ </components>
+
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>any</osFamily>
+ <packages>
+ <package>
+ <name>metron-common</name>
+ </package>
+ <package>
+ <name>metron-parsers</name>
+ </package>
+ <package>
+ <name>metron-enrichment</name>
+ </package>
+ <package>
+ <name>metron-indexing</name>
+ </package>
+ <package>
+ <name>metron-elasticsearch</name>
+ </package>
+ </packages>
+ </osSpecific>
+ <osSpecific>
+ <osFamily>redhat7</osFamily>
+ <packages>
+ <package>
+ <name>mysql-community-release</name>
+ <skipUpgrade>true</skipUpgrade>
+ </package>
+ <package>
+ <name>mysql-community-server</name>
+ <skipUpgrade>true</skipUpgrade>
+ </package>
+ <package>MySQL-python</package>
+ </packages>
+ </osSpecific>
+ <osSpecific>
+ <osFamily>redhat6</osFamily>
+ <packages>
+ <package>
+ <name>mysql-server</name>
+ <skipUpgrade>true</skipUpgrade>
+ </package>
+ <package>
+ <name>mysql</name>
+ <skipUpgrade>true</skipUpgrade>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ <commandScript>
+ <script>scripts/service_check.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>300</timeout>
+ </commandScript>
+
+ <requiredServices>
+ <service>HDFS</service>
+ <service>KAFKA</service>
+ <service>STORM</service>
+ <service>ZOOKEEPER</service>
+ <service>HBASE</service>
+ </requiredServices>
+
+ <configuration-dependencies>
+ <config-type>metron-env</config-type>
+ <config-type>storm-env</config-type>
+ <config-type>storm-site</config-type>
+ <config-type>kafka-broker</config-type>
+ <config-type>kafka-env</config-type>
+ </configuration-dependencies>
+ <restartRequiredAfterChange>true</restartRequiredAfterChange>
+ <quickLinksConfigurations>
+ <quickLinksConfiguration>
+ <fileName>quicklinks.json</fileName>
+ <default>true</default>
+ </quickLinksConfiguration>
+ </quickLinksConfigurations>
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/addMysqlUser.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/addMysqlUser.sh b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/addMysqlUser.sh
new file mode 100755
index 0000000..5dafcb8
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/addMysqlUser.sh
@@ -0,0 +1,44 @@
+#!/bin/sh
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+
+mysqldservice=$1
+mysqldbuser=$2
+mysqldbpasswd=$3
+mysqldbhost=$4
+myhostname=$(hostname -f)
+
+service $mysqldservice start
+echo "Adding user $mysqldbuser@$mysqldbhost and $mysqldbuser@localhost"
+mysql -u root -e "CREATE USER '$mysqldbuser'@'$mysqldbhost' IDENTIFIED BY '$mysqldbpasswd';"
+mysql -u root -e "CREATE USER '$mysqldbuser'@'localhost' IDENTIFIED BY '$mysqldbpasswd';"
+
+mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO '$mysqldbuser'@'$mysqldbhost';"
+mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO '$mysqldbuser'@'localhost';"
+mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO '$mysqldbuser'@'%' IDENTIFIED BY '$mysqldbpasswd';"
+
+if [ '$(mysql -u root -e "select user from mysql.user where user='$mysqldbuser' and host='$myhostname'" | grep "$mysqldbuser")' != '0' ]; then
+ echo "Adding user $mysqldbuser@$myhostname";
+ mysql -u root -e "CREATE USER '$mysqldbuser'@'$myhostname' IDENTIFIED BY '$mysqldbpasswd';";
+ mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO '$mysqldbuser'@'$myhostname';";
+fi
+mysql -u root -e "flush privileges;"
+service ${mysqldservice} stop
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/bro_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/bro_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/bro_index.template
new file mode 100644
index 0000000..951d967
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/bro_index.template
@@ -0,0 +1,217 @@
+{
+ "template": "bro_index*",
+ "mappings": {
+ "bro_doc": {
+ "_timestamp": {
+ "enabled": true
+ },
+ "dynamic_templates": [
+ {
+ "geo_location_point": {
+ "match": "enrichments:geo:*:location_point",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "geo_point"
+ }
+ }
+ },
+ {
+ "geo_country": {
+ "match": "enrichments:geo:*:country",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_city": {
+ "match": "enrichments:geo:*:city",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_location_id": {
+ "match": "enrichments:geo:*:locID",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_dma_code": {
+ "match": "enrichments:geo:*:dmaCode",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_postal_code": {
+ "match": "enrichments:geo:*:postalCode",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_latitude": {
+ "match": "enrichments:geo:*:latitude",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "float"
+ }
+ }
+ },
+ {
+ "geo_longitude": {
+ "match": "enrichments:geo:*:longitude",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "float"
+ }
+ }
+ },
+ {
+ "timestamps": {
+ "match": "*:ts",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "date",
+ "format": "epoch_millis"
+ }
+ }
+ }
+ ],
+ "properties": {
+ "timestamp": {
+ "type": "date",
+ "format": "epoch_millis"
+ },
+ "source:type": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "ip_dst_addr": {
+ "type": "ip"
+ },
+ "ip_dst_port": {
+ "type": "integer"
+ },
+ "ip_src_addr": {
+ "type": "ip"
+ },
+ "ip_src_port": {
+ "type": "integer"
+ },
+ "status_code": {
+ "type": "integer"
+ },
+ "method": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "protocol": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "request_body_len": {
+ "type": "integer"
+ },
+ "uri": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "uid": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "referrer": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "trans_depth": {
+ "type": "integer"
+ },
+ "host": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "status_msg": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "response_body_len": {
+ "type": "integer"
+ },
+ "user_agent": {
+ "type": "string"
+ },
+ "query": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "answers": {
+ "type": "ip"
+ },
+ "AA": {
+ "type": "boolean"
+ },
+ "TC": {
+ "type": "boolean"
+ },
+ "RA": {
+ "type": "boolean"
+ },
+ "RD": {
+ "type": "boolean"
+ },
+ "rejected": {
+ "type": "boolean"
+ },
+ "qclass_name": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "proto": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "rcode": {
+ "type": "integer"
+ },
+ "rcode_name": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "trans_id": {
+ "type": "integer"
+ },
+ "Z": {
+ "type": "integer"
+ },
+ "qclass": {
+ "type": "integer"
+ },
+ "qtype": {
+ "type": "integer"
+ },
+ "qtype_name": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/createMysqlGeoIp.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/createMysqlGeoIp.sh b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/createMysqlGeoIp.sh
new file mode 100755
index 0000000..21ced42
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/createMysqlGeoIp.sh
@@ -0,0 +1,42 @@
+#!/bin/sh
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+
+mysqldservice=$1
+geoipscript=$2
+geoipurl=$3
+
+# Download and extract the actual GeoIP files
+mkdir -p /tmp/geoip
+mkdir -p /var/lib/mysql-files/
+
+# Stage the GeoIP data
+pushd /tmp/geoip
+curl -O ${geoipurl}
+tar xf GeoLiteCity-latest.tar.xz
+cp /tmp/geoip/*/*.csv /var/lib/mysql-files/
+popd
+
+# Load MySQL with the GeoIP data and start service
+service ${mysqldservice} start
+mysql -u root < ${geoipscript}
+mysql -u root -e "show databases;"
+service ${mysqldservice} stop
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/removeMysqlUser.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/removeMysqlUser.sh b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/removeMysqlUser.sh
new file mode 100755
index 0000000..33a5ee5
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/removeMysqlUser.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+
+mysqldservice=$1
+mysqldbuser=$2
+userhost=$3
+myhostname=$(hostname -f)
+sudo_prefix="/var/lib/ambari-agent/ambari-sudo.sh -H -E"
+
+${sudo_prefix} service ${mysqldservice} start
+echo "Removing user $mysqldbuser@$userhost"
+/var/lib/ambari-agent/ambari-sudo.sh su mysql -s /bin/bash - -c "mysql -u root -e \"DROP USER '$mysqldbuser'@'$userhost';\""
+/var/lib/ambari-agent/ambari-sudo.sh su mysql -s /bin/bash - -c "mysql -u root -e \"flush privileges;\""
+${sudo_prefix} service ${mysqldservice} stop
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/snort_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/snort_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/snort_index.template
new file mode 100644
index 0000000..bf943df
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/snort_index.template
@@ -0,0 +1,183 @@
+{
+ "template": "snort_index*",
+ "mappings": {
+ "snort_doc": {
+ "_timestamp": {
+ "enabled": true
+ },
+ "dynamic_templates": [
+ {
+ "geo_location_point": {
+ "match": "enrichments:geo:*:location_point",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "geo_point"
+ }
+ }
+ },
+ {
+ "geo_country": {
+ "match": "enrichments:geo:*:country",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_city": {
+ "match": "enrichments:geo:*:city",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_location_id": {
+ "match": "enrichments:geo:*:locID",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_dma_code": {
+ "match": "enrichments:geo:*:dmaCode",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_postal_code": {
+ "match": "enrichments:geo:*:postalCode",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_latitude": {
+ "match": "enrichments:geo:*:latitude",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "float"
+ }
+ }
+ },
+ {
+ "geo_longitude": {
+ "match": "enrichments:geo:*:longitude",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "float"
+ }
+ }
+ },
+ {
+ "timestamps": {
+ "match": "*:ts",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "date",
+ "format": "epoch_millis"
+ }
+ }
+ }
+ ],
+ "properties": {
+ "timestamp": {
+ "type": "date",
+ "format": "epoch_millis"
+ },
+ "source:type": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "ip_dst_addr": {
+ "type": "ip"
+ },
+ "ip_dst_port": {
+ "type": "integer"
+ },
+ "ip_src_addr": {
+ "type": "ip"
+ },
+ "ip_src_port": {
+ "type": "integer"
+ },
+ "dgmlen": {
+ "type": "integer"
+ },
+ "ethdst": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "ethlen": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "ethsrc": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "iplen": {
+ "type": "integer"
+ },
+ "is_alert": {
+ "type": "boolean"
+ },
+ "msg": {
+ "type": "string"
+ },
+ "protocol": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "sig_generator": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "sig_id": {
+ "type": "integer"
+ },
+ "sig_rev": {
+ "type": "string"
+ },
+ "tcpack": {
+ "type": "string"
+ },
+ "tcpflags": {
+ "type": "string"
+ },
+ "tcpseq": {
+ "type": "string"
+ },
+ "tcpwindow": {
+ "type": "string"
+ },
+ "threat:triage:level": {
+ "type": "double"
+ },
+ "tos": {
+ "type": "integer"
+ },
+ "ttl": {
+ "type": "integer"
+ }
+ }
+ }
+ }
+}
[4/4] incubator-metron git commit: METRON-398: Bump release version
to 0.2.1BETA in master (mmiklavc) closes apache/incubator-metron#280
Posted by mm...@apache.org.
METRON-398: Bump release version to 0.2.1BETA in master (mmiklavc) closes apache/incubator-metron#280
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/823cd2a8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/823cd2a8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/823cd2a8
Branch: refs/heads/master
Commit: 823cd2a83063bf232d25d3f58c08ab1fa8e06319
Parents: dcda538
Author: mmiklavc <mi...@gmail.com>
Authored: Thu Sep 29 12:03:35 2016 -0400
Committer: Michael Miklavcic <mi...@gmail.com>
Committed: Thu Sep 29 12:03:35 2016 -0400
----------------------------------------------------------------------
metron-analytics/metron-maas-common/pom.xml | 2 +-
metron-analytics/metron-maas-service/pom.xml | 2 +-
.../src/main/scripts/maas_deploy.sh | 2 +-
.../src/main/scripts/maas_service.sh | 2 +-
metron-analytics/metron-profiler-client/pom.xml | 4 +-
metron-analytics/metron-profiler/README.md | 8 +-
metron-analytics/metron-profiler/pom.xml | 2 +-
metron-analytics/pom.xml | 2 +-
metron-deployment/amazon-ec2/conf/defaults.yml | 2 +-
.../inventory/full-dev-platform/group_vars/all | 2 +-
.../inventory/metron_example/group_vars/all | 2 +-
.../inventory/multinode-vagrant/group_vars/all | 2 +-
.../packaging/ambari/metron-mpack/pom.xml | 2 +-
.../METRON/0.2.0BETA/metainfo.xml | 26 --
.../METRON/0.2.0BETA/role_command_order.json | 14 -
.../METRON/0.2.1BETA/metainfo.xml | 26 ++
.../METRON/0.2.1BETA/role_command_order.json | 14 +
.../0.2.0BETA/configuration/metron-env.xml | 174 ------------
.../METRON/0.2.0BETA/metainfo.xml | 273 -------------------
.../0.2.0BETA/package/files/addMysqlUser.sh | 44 ---
.../0.2.0BETA/package/files/bro_index.template | 217 ---------------
.../0.2.0BETA/package/files/createMysqlGeoIp.sh | 42 ---
.../0.2.0BETA/package/files/removeMysqlUser.sh | 33 ---
.../package/files/snort_index.template | 183 -------------
.../0.2.0BETA/package/files/yaf_index.template | 205 --------------
.../package/scripts/enrichment_commands.py | 161 -----------
.../package/scripts/enrichment_master.py | 87 ------
.../package/scripts/indexing_commands.py | 142 ----------
.../package/scripts/indexing_master.py | 120 --------
.../0.2.0BETA/package/scripts/metron_service.py | 76 ------
.../0.2.0BETA/package/scripts/mysql_server.py | 61 -----
.../0.2.0BETA/package/scripts/mysql_service.py | 46 ----
.../0.2.0BETA/package/scripts/mysql_users.py | 58 ----
.../0.2.0BETA/package/scripts/mysql_utils.py | 50 ----
.../package/scripts/params/__init__.py | 18 --
.../0.2.0BETA/package/scripts/params/params.py | 32 ---
.../package/scripts/params/params_linux.py | 169 ------------
.../package/scripts/params/params_windows.py | 20 --
.../package/scripts/params/status_params.py | 57 ----
.../package/scripts/parser_commands.py | 208 --------------
.../0.2.0BETA/package/scripts/parser_master.py | 89 ------
.../0.2.0BETA/package/scripts/service_check.py | 41 ---
.../package/templates/enrichment.properties.j2 | 88 ------
.../METRON/0.2.0BETA/quicklinks/quicklinks.json | 28 --
.../0.2.1BETA/configuration/metron-env.xml | 174 ++++++++++++
.../METRON/0.2.1BETA/metainfo.xml | 273 +++++++++++++++++++
.../0.2.1BETA/package/files/addMysqlUser.sh | 44 +++
.../0.2.1BETA/package/files/bro_index.template | 217 +++++++++++++++
.../0.2.1BETA/package/files/createMysqlGeoIp.sh | 42 +++
.../0.2.1BETA/package/files/removeMysqlUser.sh | 33 +++
.../package/files/snort_index.template | 183 +++++++++++++
.../0.2.1BETA/package/files/yaf_index.template | 205 ++++++++++++++
.../package/scripts/enrichment_commands.py | 161 +++++++++++
.../package/scripts/enrichment_master.py | 87 ++++++
.../package/scripts/indexing_commands.py | 142 ++++++++++
.../package/scripts/indexing_master.py | 120 ++++++++
.../0.2.1BETA/package/scripts/metron_service.py | 76 ++++++
.../0.2.1BETA/package/scripts/mysql_server.py | 61 +++++
.../0.2.1BETA/package/scripts/mysql_service.py | 46 ++++
.../0.2.1BETA/package/scripts/mysql_users.py | 58 ++++
.../0.2.1BETA/package/scripts/mysql_utils.py | 50 ++++
.../package/scripts/params/__init__.py | 18 ++
.../0.2.1BETA/package/scripts/params/params.py | 32 +++
.../package/scripts/params/params_linux.py | 169 ++++++++++++
.../package/scripts/params/params_windows.py | 20 ++
.../package/scripts/params/status_params.py | 57 ++++
.../package/scripts/parser_commands.py | 208 ++++++++++++++
.../0.2.1BETA/package/scripts/parser_master.py | 89 ++++++
.../0.2.1BETA/package/scripts/service_check.py | 41 +++
.../package/templates/enrichment.properties.j2 | 88 ++++++
.../METRON/0.2.1BETA/quicklinks/quicklinks.json | 28 ++
.../metron-mpack/src/main/resources/mpack.json | 2 +-
.../docker/rpm-docker/SPECS/metron.spec | 1 +
.../packaging/docker/rpm-docker/pom.xml | 6 +-
.../playbooks/docker_probe_install.yml | 2 +-
metron-deployment/pom.xml | 2 +-
.../roles/metron_pcapservice/defaults/main.yml | 2 +-
metron-platform/README.md | 2 +-
metron-platform/elasticsearch-shaded/pom.xml | 2 +-
metron-platform/metron-api/pom.xml | 2 +-
metron-platform/metron-common/pom.xml | 2 +-
metron-platform/metron-data-management/pom.xml | 2 +-
metron-platform/metron-elasticsearch/pom.xml | 2 +-
metron-platform/metron-enrichment/pom.xml | 2 +-
metron-platform/metron-hbase/pom.xml | 2 +-
metron-platform/metron-indexing/pom.xml | 2 +-
metron-platform/metron-integration-test/pom.xml | 2 +-
metron-platform/metron-management/pom.xml | 2 +-
metron-platform/metron-parsers/pom.xml | 2 +-
metron-platform/metron-pcap-backend/pom.xml | 2 +-
metron-platform/metron-pcap/pom.xml | 2 +-
metron-platform/metron-solr/pom.xml | 2 +-
metron-platform/metron-test-utilities/pom.xml | 2 +-
metron-platform/metron-writer/pom.xml | 2 +-
metron-platform/pom.xml | 2 +-
pom.xml | 3 +-
96 files changed, 2806 insertions(+), 2806 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-analytics/metron-maas-common/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-maas-common/pom.xml b/metron-analytics/metron-maas-common/pom.xml
index b7f5efc..cd3e0cc 100644
--- a/metron-analytics/metron-maas-common/pom.xml
+++ b/metron-analytics/metron-maas-common/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-analytics</artifactId>
- <version>0.2.0BETA</version>
+ <version>0.2.1BETA</version>
</parent>
<artifactId>metron-maas-common</artifactId>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-analytics/metron-maas-service/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-maas-service/pom.xml b/metron-analytics/metron-maas-service/pom.xml
index d22c966..0bf2b1c 100644
--- a/metron-analytics/metron-maas-service/pom.xml
+++ b/metron-analytics/metron-maas-service/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-analytics</artifactId>
- <version>0.2.0BETA</version>
+ <version>0.2.1BETA</version>
</parent>
<artifactId>metron-maas-service</artifactId>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-analytics/metron-maas-service/src/main/scripts/maas_deploy.sh
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-maas-service/src/main/scripts/maas_deploy.sh b/metron-analytics/metron-maas-service/src/main/scripts/maas_deploy.sh
index 4413af5..1d0a9f0 100755
--- a/metron-analytics/metron-maas-service/src/main/scripts/maas_deploy.sh
+++ b/metron-analytics/metron-maas-service/src/main/scripts/maas_deploy.sh
@@ -28,7 +28,7 @@ elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
fi
export HBASE_HOME=${HBASE_HOME:-/usr/hdp/current/hbase-client}
-export METRON_VERSION=0.2.0BETA
+export METRON_VERSION=${project.version}
export METRON_HOME=/usr/metron/$METRON_VERSION
export DM_JAR=metron-maas-service-$METRON_VERSION-uber.jar
CP=$METRON_HOME/lib/$DM_JAR
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-analytics/metron-maas-service/src/main/scripts/maas_service.sh
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-maas-service/src/main/scripts/maas_service.sh b/metron-analytics/metron-maas-service/src/main/scripts/maas_service.sh
index fc3ccce..75736c1 100755
--- a/metron-analytics/metron-maas-service/src/main/scripts/maas_service.sh
+++ b/metron-analytics/metron-maas-service/src/main/scripts/maas_service.sh
@@ -28,7 +28,7 @@ elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
fi
export HBASE_HOME=${HBASE_HOME:-/usr/hdp/current/hbase-client}
-export METRON_VERSION=0.2.0BETA
+export METRON_VERSION=${project.version}
export METRON_HOME=/usr/metron/$METRON_VERSION
export DM_JAR=metron-maas-service-$METRON_VERSION-uber.jar
CP=$METRON_HOME/lib/$DM_JAR
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-analytics/metron-profiler-client/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-client/pom.xml b/metron-analytics/metron-profiler-client/pom.xml
index 6b05e3a..eec190d 100644
--- a/metron-analytics/metron-profiler-client/pom.xml
+++ b/metron-analytics/metron-profiler-client/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-analytics</artifactId>
- <version>0.2.0BETA</version>
+ <version>0.2.1BETA</version>
</parent>
<artifactId>metron-profiler-client</artifactId>
<properties>
@@ -278,4 +278,4 @@
</plugin>
</plugins>
</build>
-</project>
\ No newline at end of file
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-analytics/metron-profiler/README.md
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/README.md b/metron-analytics/metron-profiler/README.md
index 6822b61..2fd83d5 100644
--- a/metron-analytics/metron-profiler/README.md
+++ b/metron-analytics/metron-profiler/README.md
@@ -239,23 +239,23 @@ This section will describe the steps required to get your first profile running.
```
$ vagrant ssh
$ sudo su -
- $ cd /usr/metron/0.2.0BETA/
+ $ cd /usr/metron/0.2.1BETA/
```
-3. Create a table within HBase that will store the profile data. The table name and column family must match the Profiler topology configuration stored at `/usr/metron/0.2.0BETA/config/profiler.properties`.
+3. Create a table within HBase that will store the profile data. The table name and column family must match the Profiler topology configuration stored at `/usr/metron/0.2.1BETA/config/profiler.properties`.
```
$ /usr/hdp/current/hbase-client/bin/hbase shell
hbase(main):001:0> create 'profiler', 'P'
```
-4. Shorten the flush intervals to more immediately see results. Edit the Profiler topology properties located at `/usr/metron/0.2.0BETA/config/profiler.properties`. Alter the following two properties.
+4. Shorten the flush intervals to more immediately see results. Edit the Profiler topology properties located at `/usr/metron/0.2.1BETA/config/profiler.properties`. Alter the following two properties.
```
profiler.period.duration=30
profiler.period.duration.units=SECONDS
profiler.hbase.flush.interval.seconds=5
```
-5. Create the Profiler definition in a file located at `/usr/metron/0.2.0BETA/config/zookeeper/profiler.json`. The following JSON will create a profile that simply counts the number of messages.
+5. Create the Profiler definition in a file located at `/usr/metron/0.2.1BETA/config/zookeeper/profiler.json`. The following JSON will create a profile that simply counts the number of messages.
```
{
"profiles": [
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-analytics/metron-profiler/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/pom.xml b/metron-analytics/metron-profiler/pom.xml
index 125a42d..49c0820 100644
--- a/metron-analytics/metron-profiler/pom.xml
+++ b/metron-analytics/metron-profiler/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-analytics</artifactId>
- <version>0.2.0BETA</version>
+ <version>0.2.1BETA</version>
</parent>
<artifactId>metron-profiler</artifactId>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-analytics/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/pom.xml b/metron-analytics/pom.xml
index 2e62be7..299efd6 100644
--- a/metron-analytics/pom.xml
+++ b/metron-analytics/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>Metron</artifactId>
- <version>0.2.0BETA</version>
+ <version>0.2.1BETA</version>
</parent>
<description>Stream analytics for Metron</description>
<url>https://metron.incubator.apache.org/</url>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/amazon-ec2/conf/defaults.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/amazon-ec2/conf/defaults.yml b/metron-deployment/amazon-ec2/conf/defaults.yml
index 5d3c1a9..0e7f8dc 100644
--- a/metron-deployment/amazon-ec2/conf/defaults.yml
+++ b/metron-deployment/amazon-ec2/conf/defaults.yml
@@ -75,7 +75,7 @@ num_partitions: 3
retention_in_gb: 25
# metron variables
-metron_version: 0.2.0BETA
+metron_version: 0.2.1BETA
metron_directory: /usr/metron/{{ metron_version }}
pcapservice_port: 8081
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/inventory/full-dev-platform/group_vars/all
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/full-dev-platform/group_vars/all b/metron-deployment/inventory/full-dev-platform/group_vars/all
index 609f388..33456dc 100644
--- a/metron-deployment/inventory/full-dev-platform/group_vars/all
+++ b/metron-deployment/inventory/full-dev-platform/group_vars/all
@@ -47,7 +47,7 @@ threatintel_hbase_table: threatintel
enrichment_hbase_table: enrichment
# metron
-metron_version: 0.2.0BETA
+metron_version: 0.2.1BETA
metron_directory: /usr/metron/{{ metron_version }}
bro_version: "2.4.1"
fixbuf_version: "1.7.1"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/inventory/metron_example/group_vars/all
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/metron_example/group_vars/all b/metron-deployment/inventory/metron_example/group_vars/all
index 6c30aec..e6052af 100644
--- a/metron-deployment/inventory/metron_example/group_vars/all
+++ b/metron-deployment/inventory/metron_example/group_vars/all
@@ -54,7 +54,7 @@ num_partitions: 3
retention_in_gb: 25
# metron variables
-metron_version: 0.2.0BETA
+metron_version: 0.2.1BETA
metron_directory: /usr/metron/{{ metron_version }}
pcapservice_port: 8081
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/inventory/multinode-vagrant/group_vars/all
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/multinode-vagrant/group_vars/all b/metron-deployment/inventory/multinode-vagrant/group_vars/all
index 4084b3d..cb300d3 100644
--- a/metron-deployment/inventory/multinode-vagrant/group_vars/all
+++ b/metron-deployment/inventory/multinode-vagrant/group_vars/all
@@ -35,7 +35,7 @@ elasticsearch_network_interface: eth1
elasticsearch_web_port: 9200
# metron variables
-metron_version: 0.2.0BETA
+metron_version: 0.2.1BETA
pcapservice_port: 8081
# sensors
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/pom.xml b/metron-deployment/packaging/ambari/metron-mpack/pom.xml
index 0a40f75..66226ce 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/pom.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/pom.xml
@@ -26,7 +26,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-deployment</artifactId>
- <version>0.2.0BETA</version>
+ <version>0.2.1BETA</version>
<relativePath>../../..</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/metainfo.xml
deleted file mode 100755
index a3c984c..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/metainfo.xml
+++ /dev/null
@@ -1,26 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<metainfo>
- <schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>METRON</name>
- <version>0.2.0BETA</version>
- <extends>common-services/METRON/0.2.0BETA</extends>
- </service>
- </services>
-</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/role_command_order.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/role_command_order.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/role_command_order.json
deleted file mode 100755
index 0f4cd39..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/role_command_order.json
+++ /dev/null
@@ -1,14 +0,0 @@
-{
- "_comment" : "Record format:",
- "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]",
- "general_deps" : {
- "_comment" : "dependencies for all cases",
- "METRON_INDEXING-INSTALL" : ["METRON_PARSERS-INSTALL"],
- "METRON_PARSERS-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START"],
- "METRON_ENRICHMENT_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START", "ENRICHMENT_MYSQL_SERVER-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START"],
- "METRON_ENRICHMENT_MYSQL_SERVER-STOP" : ["METRON_ENRICHMENT_MASTER-STOP"],
- "METRON_ENRICHMENT_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_ENRICHMENT_MASTER-START"],
- "METRON_INDEXING-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START","METRON_PARSERS-START"],
- "METRON_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_PARSERS-START","METRON_INDEXING-START"]
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.1BETA/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.1BETA/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.1BETA/metainfo.xml
new file mode 100755
index 0000000..7fac89a
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.1BETA/metainfo.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>METRON</name>
+ <version>0.2.1BETA</version>
+ <extends>common-services/METRON/0.2.1BETA</extends>
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.1BETA/role_command_order.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.1BETA/role_command_order.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.1BETA/role_command_order.json
new file mode 100755
index 0000000..0f4cd39
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.1BETA/role_command_order.json
@@ -0,0 +1,14 @@
+{
+ "_comment" : "Record format:",
+ "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]",
+ "general_deps" : {
+ "_comment" : "dependencies for all cases",
+ "METRON_INDEXING-INSTALL" : ["METRON_PARSERS-INSTALL"],
+ "METRON_PARSERS-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START"],
+ "METRON_ENRICHMENT_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START", "ENRICHMENT_MYSQL_SERVER-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START"],
+ "METRON_ENRICHMENT_MYSQL_SERVER-STOP" : ["METRON_ENRICHMENT_MASTER-STOP"],
+ "METRON_ENRICHMENT_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_ENRICHMENT_MASTER-START"],
+ "METRON_INDEXING-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START","METRON_PARSERS-START"],
+ "METRON_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_PARSERS-START","METRON_INDEXING-START"]
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml
deleted file mode 100644
index 30f9f76..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml
+++ /dev/null
@@ -1,174 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<configuration supports_final="true">
- <property>
- <name>metron_home</name>
- <value>/usr/metron/0.2.0BETA</value>
- <description>Metron home directory</description>
- <display-name>Metron home</display-name>
- </property>
- <property>
- <name>metron_apps_hdfs_dir</name>
- <value>/apps/metron</value>
- <description>Metron apps HDFS dir</description>
- <display-name>Metron apps HDFS dir</display-name>
- </property>
- <property>
- <name>metron_zookeeper_config_dir</name>
- <value>config/zookeeper</value>
- <description>Metron Zookeeper config dir. Relative path to Metron home.</description>
- <display-name>Metron Zookeeper config dir</display-name>
- </property>
- <property>
- <name>metron_user</name>
- <value>metron</value>
- <property-type>USER</property-type>
- <description>The user for Metron</description>
- <display-name>Metron User</display-name>
- </property>
- <property>
- <name>metron_group</name>
- <value>metron</value>
- <property-type>GROUP</property-type>
- <description>The group for Metron</description>
- </property>
- <property>
- <name>metron_topic_retention</name>
- <description>Kafka Retention in GB</description>
- <value>10</value>
- </property>
- <property>
- <name>parsers</name>
- <value>bro,snort,yaf</value>
- <description>Metron parsers to deploy</description>
- <display-name>Metron parsers</display-name>
- </property>
- <property>
- <name>metron_enrichment_db_user</name>
- <value>metron</value>
- <description>Database username to use to connect to the database.</description>
- </property>
- <property>
- <name>metron_enrichment_db_port</name>
- <value>3306</value>
- <description>Database port to use to connect to the database.</description>
- </property>
- <property>
- <name>metron_enrichment_db_password</name>
- <value></value>
- <property-type>PASSWORD</property-type>
- <display-name>Metron Enrichment Database Password</display-name>
- <description>Password to use against database</description>
- <value-attributes>
- <type>password</type>
- <overridable>false</overridable>
- </value-attributes>
- <on-ambari-upgrade add="true"/>
- </property>
- <property>
- <name>metron_indexing_topology</name>
- <value>indexing</value>
- <description>The Storm topology name for Indexing</description>
- </property>
- <property>
- <name>es_cluster_name</name>
- <value>metron</value>
- <description>Name of Elasticsearch Cluster</description>
- </property>
- <property>
- <name>geoip_url</name>
- <value>http://geolite.maxmind.com/download/geoip/database/GeoLiteCity_CSV/GeoLiteCity-latest.tar.xz</value>
- <description>Location of the GeoIP data to load.</description>
- </property>
- <property require-input="true">
- <name>es_url</name>
- <value></value>
- <description>Comma delimited list of Elasticsearch URLs. (eshost1:9300,eshost2:9300)</description>
- </property>
- <property>
- <name>storm_rest_addr</name>
- <!--<value-attributes>-->
- <!--<editable-only-at-install>true</editable-only-at-install>-->
- <!--<overridable>false</overridable>-->
- <!--</value-attributes>-->
- <value></value>
- </property>
- <property>
- <name>global-json</name>
- <display-name>global.json template</display-name>
- <description>This is the jinja template for global.json file</description>
- <value>
-{
-"es.clustername": "{{ es_cluster_name }}",
-"es.ip": "{{ es_url }}",
-"es.date.format": "yyyy.MM.dd.HH"
-}
- </value>
- <value-attributes>
- <type>content</type>
- </value-attributes>
- </property>
- <property>
- <name>elasticsearch-properties</name>
- <description>The template for the elasticsearch.properties file.</description>
- <display-name>elasticsearch.properties template</display-name>
- <value>
-##### Storm #####
-indexing.workers=1
-indexing.executors=0
-##### Kafka #####
-kafka.zk={{ zookeeper_quorum }}
-kafka.broker={{ kafka_brokers }}
-kafka.start=WHERE_I_LEFT_OFF
-##### Indexing #####
-index.input.topic=indexing
-index.error.topic=indexing_error
-writer.class.name=org.apache.metron.elasticsearch.writer.ElasticsearchWriter
-##### Metrics #####
-#reporters
-org.apache.metron.metrics.reporter.graphite=true
-org.apache.metron.metrics.reporter.console=false
-org.apache.metron.metrics.reporter.jmx=false
-#Graphite Addresses
-org.apache.metron.metrics.graphite.address=localhost
-org.apache.metron.metrics.graphite.port=2023
-#TelemetryParserBolt
-org.apache.metron.metrics.TelemetryParserBolt.acks=true
-org.apache.metron.metrics.TelemetryParserBolt.emits=true
-org.apache.metron.metrics.TelemetryParserBolt.fails=true
-##### HDFS #####
-bolt.hdfs.batch.size=5000
-bolt.hdfs.field.delimiter=|
-bolt.hdfs.rotation.policy=org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy
-bolt.hdfs.rotation.policy.units=DAYS
-bolt.hdfs.rotation.policy.count=1
-bolt.hdfs.file.rotation.size.in.mb=5
-bolt.hdfs.file.system.url={{ default_fs }}
-bolt.hdfs.wip.file.path=/paloalto/wip
-bolt.hdfs.finished.file.path=/paloalto/rotated
-bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec
-index.hdfs.output=/tmp/metron/enriched
- </value>
- <value-attributes>
- <type>content</type>
- </value-attributes>
- <on-ambari-upgrade add="true"/>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/metainfo.xml
deleted file mode 100644
index d9886fe..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/metainfo.xml
+++ /dev/null
@@ -1,273 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<metainfo>
- <schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>METRON</name>
- <displayName>Metron</displayName>
- <comment>A scalable advanced security analytics framework built on Hadoop</comment>
- <version>0.2.0BETA</version>
- <components>
-
- <component>
- <name>METRON_PARSERS</name>
- <displayName>Metron Parsers</displayName>
- <category>MASTER</category>
- <cardinality>1</cardinality>
- <versionAdvertised>true</versionAdvertised>
- <reassignAllowed>false</reassignAllowed>
- <clientsToUpdateConfigs></clientsToUpdateConfigs>
- <dependencies>
- <dependency>
- <name>HDFS/HDFS_CLIENT</name>
- <scope>host</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- <dependency>
- <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
- <scope>cluster</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- <dependency>
- <name>STORM/SUPERVISOR</name>
- <scope>host</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- <dependency>
- <name>KAFKA/KAFKA_BROKER</name>
- <scope>host</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- </dependencies>
- <commandScript>
- <script>scripts/parser_master.py</script>
- <scriptType>PYTHON</scriptType>
- </commandScript>
- <configuration-dependencies>
- <config-type>metron-parsers</config-type>
- </configuration-dependencies>
- </component>
-
- <component>
- <name>METRON_ENRICHMENT_MASTER</name>
- <displayName>Metron Enrichment</displayName>
- <category>MASTER</category>
- <cardinality>1</cardinality>
- <versionAdvertised>true</versionAdvertised>
- <dependencies>
- <dependency>
- <name>HDFS/HDFS_CLIENT</name>
- <scope>host</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- <dependency>
- <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
- <scope>cluster</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- <dependency>
- <name>KAFKA/KAFKA_BROKER</name>
- <scope>cluster</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- </dependencies>
- <commandScript>
- <script>scripts/enrichment_master.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
-
- <component>
- <name>METRON_ENRICHMENT_MYSQL_SERVER</name>
- <displayName>MySQL Server</displayName>
- <category>MASTER</category>
- <cardinality>1</cardinality>
- <versionAdvertised>false</versionAdvertised>
- <clientsToUpdateConfigs></clientsToUpdateConfigs>
- <commandScript>
- <script>scripts/mysql_server.py</script>
- <scriptType>PYTHON</scriptType>
- </commandScript>
- <customCommands>
- <customCommand>
- <name>CLEAN</name>
- <commandScript>
- <script>scripts/mysql_server.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </customCommand>
- </customCommands>
- </component>
-
- <component>
- <name>METRON_INDEXING</name>
- <displayName>Metron Indexing</displayName>
- <category>MASTER</category>
- <cardinality>1</cardinality>
- <versionAdvertised>true</versionAdvertised>
- <reassignAllowed>false</reassignAllowed>
- <clientsToUpdateConfigs></clientsToUpdateConfigs>
- <dependencies>
- <dependency>
- <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
- <scope>cluster</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- <dependency>
- <name>STORM/SUPERVISOR</name>
- <scope>host</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- <dependency>
- <name>KAFKA/KAFKA_BROKER</name>
- <scope>host</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- </dependencies>
- <commandScript>
- <script>scripts/indexing_master.py</script>
- <scriptType>PYTHON</scriptType>
- </commandScript>
- <customCommands>
- <customCommand>
- <name>ELASTICSEARCH_TEMPLATE_INSTALL</name>
- <commandScript>
- <script>scripts/indexing_master.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </customCommand>
- <customCommand>
- <name>ELASTICSEARCH_TEMPLATE_DELETE</name>
- <commandScript>
- <script>scripts/indexing_master.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </customCommand>
- </customCommands>
- <configuration-dependencies>
- <config-type>metron-indexing</config-type>
- </configuration-dependencies>
- </component>
- </components>
-
- <osSpecifics>
- <osSpecific>
- <osFamily>any</osFamily>
- <packages>
- <package>
- <name>metron-common</name>
- </package>
- <package>
- <name>metron-parsers</name>
- </package>
- <package>
- <name>metron-enrichment</name>
- </package>
- <package>
- <name>metron-indexing</name>
- </package>
- <package>
- <name>metron-elasticsearch</name>
- </package>
- </packages>
- </osSpecific>
- <osSpecific>
- <osFamily>redhat7</osFamily>
- <packages>
- <package>
- <name>mysql-community-release</name>
- <skipUpgrade>true</skipUpgrade>
- </package>
- <package>
- <name>mysql-community-server</name>
- <skipUpgrade>true</skipUpgrade>
- </package>
- <package>MySQL-python</package>
- </packages>
- </osSpecific>
- <osSpecific>
- <osFamily>redhat6</osFamily>
- <packages>
- <package>
- <name>mysql-server</name>
- <skipUpgrade>true</skipUpgrade>
- </package>
- <package>
- <name>mysql</name>
- <skipUpgrade>true</skipUpgrade>
- </package>
- </packages>
- </osSpecific>
- </osSpecifics>
-
- <commandScript>
- <script>scripts/service_check.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>300</timeout>
- </commandScript>
-
- <requiredServices>
- <service>HDFS</service>
- <service>KAFKA</service>
- <service>STORM</service>
- <service>ZOOKEEPER</service>
- <service>HBASE</service>
- </requiredServices>
-
- <configuration-dependencies>
- <config-type>metron-env</config-type>
- <config-type>storm-env</config-type>
- <config-type>storm-site</config-type>
- <config-type>kafka-broker</config-type>
- <config-type>kafka-env</config-type>
- </configuration-dependencies>
- <restartRequiredAfterChange>true</restartRequiredAfterChange>
- <quickLinksConfigurations>
- <quickLinksConfiguration>
- <fileName>quicklinks.json</fileName>
- <default>true</default>
- </quickLinksConfiguration>
- </quickLinksConfigurations>
- </service>
- </services>
-</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/addMysqlUser.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/addMysqlUser.sh b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/addMysqlUser.sh
deleted file mode 100755
index 5dafcb8..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/addMysqlUser.sh
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/bin/sh
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#
-
-mysqldservice=$1
-mysqldbuser=$2
-mysqldbpasswd=$3
-mysqldbhost=$4
-myhostname=$(hostname -f)
-
-service $mysqldservice start
-echo "Adding user $mysqldbuser@$mysqldbhost and $mysqldbuser@localhost"
-mysql -u root -e "CREATE USER '$mysqldbuser'@'$mysqldbhost' IDENTIFIED BY '$mysqldbpasswd';"
-mysql -u root -e "CREATE USER '$mysqldbuser'@'localhost' IDENTIFIED BY '$mysqldbpasswd';"
-
-mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO '$mysqldbuser'@'$mysqldbhost';"
-mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO '$mysqldbuser'@'localhost';"
-mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO '$mysqldbuser'@'%' IDENTIFIED BY '$mysqldbpasswd';"
-
-if [ '$(mysql -u root -e "select user from mysql.user where user='$mysqldbuser' and host='$myhostname'" | grep "$mysqldbuser")' != '0' ]; then
- echo "Adding user $mysqldbuser@$myhostname";
- mysql -u root -e "CREATE USER '$mysqldbuser'@'$myhostname' IDENTIFIED BY '$mysqldbpasswd';";
- mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO '$mysqldbuser'@'$myhostname';";
-fi
-mysql -u root -e "flush privileges;"
-service ${mysqldservice} stop
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/bro_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/bro_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/bro_index.template
deleted file mode 100644
index 951d967..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/bro_index.template
+++ /dev/null
@@ -1,217 +0,0 @@
-{
- "template": "bro_index*",
- "mappings": {
- "bro_doc": {
- "_timestamp": {
- "enabled": true
- },
- "dynamic_templates": [
- {
- "geo_location_point": {
- "match": "enrichments:geo:*:location_point",
- "match_mapping_type": "*",
- "mapping": {
- "type": "geo_point"
- }
- }
- },
- {
- "geo_country": {
- "match": "enrichments:geo:*:country",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_city": {
- "match": "enrichments:geo:*:city",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_location_id": {
- "match": "enrichments:geo:*:locID",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_dma_code": {
- "match": "enrichments:geo:*:dmaCode",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_postal_code": {
- "match": "enrichments:geo:*:postalCode",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_latitude": {
- "match": "enrichments:geo:*:latitude",
- "match_mapping_type": "*",
- "mapping": {
- "type": "float"
- }
- }
- },
- {
- "geo_longitude": {
- "match": "enrichments:geo:*:longitude",
- "match_mapping_type": "*",
- "mapping": {
- "type": "float"
- }
- }
- },
- {
- "timestamps": {
- "match": "*:ts",
- "match_mapping_type": "*",
- "mapping": {
- "type": "date",
- "format": "epoch_millis"
- }
- }
- }
- ],
- "properties": {
- "timestamp": {
- "type": "date",
- "format": "epoch_millis"
- },
- "source:type": {
- "type": "string",
- "index": "not_analyzed"
- },
- "ip_dst_addr": {
- "type": "ip"
- },
- "ip_dst_port": {
- "type": "integer"
- },
- "ip_src_addr": {
- "type": "ip"
- },
- "ip_src_port": {
- "type": "integer"
- },
- "status_code": {
- "type": "integer"
- },
- "method": {
- "type": "string",
- "index": "not_analyzed"
- },
- "protocol": {
- "type": "string",
- "index": "not_analyzed"
- },
- "request_body_len": {
- "type": "integer"
- },
- "uri": {
- "type": "string",
- "index": "not_analyzed"
- },
- "uid": {
- "type": "string",
- "index": "not_analyzed"
- },
- "referrer": {
- "type": "string",
- "index": "not_analyzed"
- },
- "trans_depth": {
- "type": "integer"
- },
- "host": {
- "type": "string",
- "index": "not_analyzed"
- },
- "status_msg": {
- "type": "string",
- "index": "not_analyzed"
- },
- "response_body_len": {
- "type": "integer"
- },
- "user_agent": {
- "type": "string"
- },
- "query": {
- "type": "string",
- "index": "not_analyzed"
- },
- "answers": {
- "type": "ip"
- },
- "AA": {
- "type": "boolean"
- },
- "TC": {
- "type": "boolean"
- },
- "RA": {
- "type": "boolean"
- },
- "RD": {
- "type": "boolean"
- },
- "rejected": {
- "type": "boolean"
- },
- "qclass_name": {
- "type": "string",
- "index": "not_analyzed"
- },
- "proto": {
- "type": "string",
- "index": "not_analyzed"
- },
- "rcode": {
- "type": "integer"
- },
- "rcode_name": {
- "type": "string",
- "index": "not_analyzed"
- },
- "trans_id": {
- "type": "integer"
- },
- "Z": {
- "type": "integer"
- },
- "qclass": {
- "type": "integer"
- },
- "qtype": {
- "type": "integer"
- },
- "qtype_name": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/createMysqlGeoIp.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/createMysqlGeoIp.sh b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/createMysqlGeoIp.sh
deleted file mode 100755
index 21ced42..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/createMysqlGeoIp.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/bin/sh
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#
-
-mysqldservice=$1
-geoipscript=$2
-geoipurl=$3
-
-# Download and extract the actual GeoIP files
-mkdir -p /tmp/geoip
-mkdir -p /var/lib/mysql-files/
-
-# Stage the GeoIP data
-pushd /tmp/geoip
-curl -O ${geoipurl}
-tar xf GeoLiteCity-latest.tar.xz
-cp /tmp/geoip/*/*.csv /var/lib/mysql-files/
-popd
-
-# Load MySQL with the GeoIP data and start service
-service ${mysqldservice} start
-mysql -u root < ${geoipscript}
-mysql -u root -e "show databases;"
-service ${mysqldservice} stop
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/removeMysqlUser.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/removeMysqlUser.sh b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/removeMysqlUser.sh
deleted file mode 100755
index 33a5ee5..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/removeMysqlUser.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/usr/bin/env bash
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#
-
-mysqldservice=$1
-mysqldbuser=$2
-userhost=$3
-myhostname=$(hostname -f)
-sudo_prefix="/var/lib/ambari-agent/ambari-sudo.sh -H -E"
-
-${sudo_prefix} service ${mysqldservice} start
-echo "Removing user $mysqldbuser@$userhost"
-/var/lib/ambari-agent/ambari-sudo.sh su mysql -s /bin/bash - -c "mysql -u root -e \"DROP USER '$mysqldbuser'@'$userhost';\""
-/var/lib/ambari-agent/ambari-sudo.sh su mysql -s /bin/bash - -c "mysql -u root -e \"flush privileges;\""
-${sudo_prefix} service ${mysqldservice} stop
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/snort_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/snort_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/snort_index.template
deleted file mode 100644
index bf943df..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/snort_index.template
+++ /dev/null
@@ -1,183 +0,0 @@
-{
- "template": "snort_index*",
- "mappings": {
- "snort_doc": {
- "_timestamp": {
- "enabled": true
- },
- "dynamic_templates": [
- {
- "geo_location_point": {
- "match": "enrichments:geo:*:location_point",
- "match_mapping_type": "*",
- "mapping": {
- "type": "geo_point"
- }
- }
- },
- {
- "geo_country": {
- "match": "enrichments:geo:*:country",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_city": {
- "match": "enrichments:geo:*:city",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_location_id": {
- "match": "enrichments:geo:*:locID",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_dma_code": {
- "match": "enrichments:geo:*:dmaCode",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_postal_code": {
- "match": "enrichments:geo:*:postalCode",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_latitude": {
- "match": "enrichments:geo:*:latitude",
- "match_mapping_type": "*",
- "mapping": {
- "type": "float"
- }
- }
- },
- {
- "geo_longitude": {
- "match": "enrichments:geo:*:longitude",
- "match_mapping_type": "*",
- "mapping": {
- "type": "float"
- }
- }
- },
- {
- "timestamps": {
- "match": "*:ts",
- "match_mapping_type": "*",
- "mapping": {
- "type": "date",
- "format": "epoch_millis"
- }
- }
- }
- ],
- "properties": {
- "timestamp": {
- "type": "date",
- "format": "epoch_millis"
- },
- "source:type": {
- "type": "string",
- "index": "not_analyzed"
- },
- "ip_dst_addr": {
- "type": "ip"
- },
- "ip_dst_port": {
- "type": "integer"
- },
- "ip_src_addr": {
- "type": "ip"
- },
- "ip_src_port": {
- "type": "integer"
- },
- "dgmlen": {
- "type": "integer"
- },
- "ethdst": {
- "type": "string",
- "index": "not_analyzed"
- },
- "ethlen": {
- "type": "string",
- "index": "not_analyzed"
- },
- "ethsrc": {
- "type": "string",
- "index": "not_analyzed"
- },
- "id": {
- "type": "integer"
- },
- "iplen": {
- "type": "integer"
- },
- "is_alert": {
- "type": "boolean"
- },
- "msg": {
- "type": "string"
- },
- "protocol": {
- "type": "string",
- "index": "not_analyzed"
- },
- "sig_generator": {
- "type": "string",
- "index": "not_analyzed"
- },
- "sig_id": {
- "type": "integer"
- },
- "sig_rev": {
- "type": "string"
- },
- "tcpack": {
- "type": "string"
- },
- "tcpflags": {
- "type": "string"
- },
- "tcpseq": {
- "type": "string"
- },
- "tcpwindow": {
- "type": "string"
- },
- "threat:triage:level": {
- "type": "double"
- },
- "tos": {
- "type": "integer"
- },
- "ttl": {
- "type": "integer"
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/yaf_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/yaf_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/yaf_index.template
deleted file mode 100644
index 7743afc..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/yaf_index.template
+++ /dev/null
@@ -1,205 +0,0 @@
-{
- "template": "yaf_index*",
- "mappings": {
- "yaf_doc": {
- "_timestamp": {
- "enabled": true
- },
- "dynamic_templates": [
- {
- "geo_location_point": {
- "match": "enrichments:geo:*:location_point",
- "match_mapping_type": "*",
- "mapping": {
- "type": "geo_point"
- }
- }
- },
- {
- "geo_country": {
- "match": "enrichments:geo:*:country",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_city": {
- "match": "enrichments:geo:*:city",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_location_id": {
- "match": "enrichments:geo:*:locID",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_dma_code": {
- "match": "enrichments:geo:*:dmaCode",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_postal_code": {
- "match": "enrichments:geo:*:postalCode",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- },
- {
- "geo_latitude": {
- "match": "enrichments:geo:*:latitude",
- "match_mapping_type": "*",
- "mapping": {
- "type": "float"
- }
- }
- },
- {
- "geo_longitude": {
- "match": "enrichments:geo:*:longitude",
- "match_mapping_type": "*",
- "mapping": {
- "type": "float"
- }
- }
- },
- {
- "timestamps": {
- "match": "*:ts",
- "match_mapping_type": "*",
- "mapping": {
- "type": "date",
- "format": "epoch_millis"
- }
- }
- }
- ],
- "properties": {
- "timestamp": {
- "type": "date",
- "format": "epoch_millis"
- },
- "source:type": {
- "type": "string",
- "index": "not_analyzed"
- },
- "ip_dst_addr": {
- "type": "ip"
- },
- "ip_dst_port": {
- "type": "integer"
- },
- "ip_src_addr": {
- "type": "ip"
- },
- "ip_src_port": {
- "type": "integer"
- },
- "start_time": {
- "type": "date",
- "format": "epoch_millis"
- },
- "end_time": {
- "type": "date",
- "format": "epoch_millis"
- },
- "duration": {
- "type": "double"
- },
- "rtt": {
- "type": "double"
- },
- "proto": {
- "type": "string",
- "index": "not_analyzed"
- },
- "sip": {
- "type": "string",
- "index": "not_analyzed"
- },
- "sp": {
- "type": "string",
- "index": "not_analyzed"
- },
- "dip": {
- "type": "string",
- "index": "not_analyzed"
- },
- "dp": {
- "type": "string",
- "index": "not_analyzed"
- },
- "iflags": {
- "type": "string",
- "index": "not_analyzed"
- },
- "uflags": {
- "type": "string",
- "index": "not_analyzed"
- },
- "riflags": {
- "type": "string"
- },
- "ruflags": {
- "type": "string",
- "index": "not_analyzed"
- },
- "isn": {
- "type": "string",
- "index": "not_analyzed"
- },
- "risn": {
- "type": "string",
- "index": "not_analyzed"
- },
- "tag": {
- "type": "string",
- "index": "not_analyzed"
- },
- "rtag": {
- "type": "string",
- "index": "not_analyzed"
- },
- "pkt": {
- "type": "integer"
- },
- "oct": {
- "type": "integer"
- },
- "rpkt": {
- "type": "integer"
- },
- "roct": {
- "type": "integer"
- },
- "app": {
- "type": "string",
- "index": "not_analyzed"
- },
- "end-reason": {
- "type": "string"
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py
deleted file mode 100755
index 027ffae..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py
+++ /dev/null
@@ -1,161 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-
-import os
-import time
-
-from resource_management.core.logger import Logger
-from resource_management.core.resources.system import Execute, File
-
-import metron_service
-
-
-# Wrap major operations and functionality in this class
-class EnrichmentCommands:
- __params = None
- __enrichment_topology = None
- __enrichment_topic = None
- __configured = False
-
- def __init__(self, params):
- if params is None:
- raise ValueError("params argument is required for initialization")
- self.__params = params
- self.__enrichment_topology = params.metron_enrichment_topology
- self.__enrichment_topic = params.metron_enrichment_topic
- self.__configured = os.path.isfile(self.__params.enrichment_configured_flag_file)
-
- def is_configured(self):
- return self.__configured
-
- def set_configured(self):
- File(self.__params.enrichment_configured_flag_file,
- content="",
- owner=self.__params.metron_user,
- mode=0775)
-
- def setup_repo(self):
- def local_repo():
- Logger.info("Setting up local repo")
- Execute("yum -y install createrepo")
- Execute("createrepo /localrepo")
- Execute("chmod -R o-w+r /localrepo")
- Execute("echo \"[METRON-0.2.0BETA]\n"
- "name=Metron 0.2.0BETA packages\n"
- "baseurl=file:///localrepo\n"
- "gpgcheck=0\n"
- "enabled=1\" > /etc/yum.repos.d/local.repo")
-
- def remote_repo():
- print('Using remote repo')
-
- yum_repo_types = {
- 'local': local_repo,
- 'remote': remote_repo
- }
- repo_type = self.__params.yum_repo_type
- if repo_type in yum_repo_types:
- yum_repo_types[repo_type]()
- else:
- raise ValueError("Unsupported repo type '{0}'".format(repo_type))
-
- def init_kafka_topics(self):
- Logger.info('Creating Kafka topics')
- command_template = """{0}/kafka-topics.sh \
- --zookeeper {1} \
- --create \
- --topic {2} \
- --partitions {3} \
- --replication-factor {4} \
- --config retention.bytes={5}"""
- num_partitions = 1
- replication_factor = 1
- retention_gigabytes = int(self.__params.metron_topic_retention)
- retention_bytes = retention_gigabytes * 1024 * 1024 * 1024
- Logger.info("Creating topics for enrichment")
-
- Logger.info("Creating topic'{0}'".format(self.__enrichment_topic))
- Execute(command_template.format(self.__params.kafka_bin_dir,
- self.__params.zookeeper_quorum,
- self.__enrichment_topic,
- num_partitions,
- replication_factor,
- retention_bytes))
- Logger.info("Done creating Kafka topics")
-
- def start_enrichment_topology(self):
- Logger.info("Starting Metron enrichment topology: {0}".format(self.__enrichment_topology))
- start_cmd_template = """{0}/bin/start_enrichment_topology.sh \
- -s {1} \
- -z {2}"""
- Logger.info('Starting ' + self.__enrichment_topology)
- Execute(start_cmd_template.format(self.__params.metron_home, self.__enrichment_topology, self.__params.zookeeper_quorum))
-
- Logger.info('Finished starting enrichment topology')
-
- def stop_enrichment_topology(self):
- Logger.info('Stopping ' + self.__enrichment_topology)
- stop_cmd = 'storm kill ' + self.__enrichment_topology
- Execute(stop_cmd)
- Logger.info('Done stopping enrichment topologies')
-
- def restart_enrichment_topology(self, env):
- Logger.info('Restarting the enrichment topologies')
- self.stop_enrichment_topology()
-
- # Wait for old topology to be cleaned up by Storm, before starting again.
- retries = 0
- topology_active = self.is_topology_active(env)
- while topology_active and retries < 3:
- Logger.info('Existing topology still active. Will wait and retry')
- time.sleep(40)
- topology_active = self.is_topology_active(env)
- retries += 1
-
- if not topology_active:
- self.start_enrichment_topology()
- Logger.info('Done restarting the enrichment topology')
- else:
- Logger.warning('Retries exhausted. Existing topology not cleaned up. Aborting topology start.')
-
- def is_topology_active(self, env):
- env.set_params(self.__params)
-
- active = True
- topologies = metron_service.get_running_topologies()
- is_running = False
- if self.__enrichment_topology in topologies:
- is_running = topologies[self.__enrichment_topology] in ['ACTIVE', 'REBALANCING']
- active &= is_running
- return active
-
- def create_hbase_tables(self):
- add_enrichment_cmd = "echo \"create '{0}','{1}'\" | hbase shell -n".format(self.__params.enrichment_table, self.__params.enrichment_cf)
- Execute(add_enrichment_cmd,
- tries=3,
- try_sleep=5,
- logoutput=False,
- path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'
- )
-
- add_threatintel_cmd = "echo \"create '{0}','{1}'\" | hbase shell -n".format(self.__params.threatintel_table, self.__params.threatintel_cf)
- Execute(add_threatintel_cmd,
- tries=3,
- try_sleep=5,
- logoutput=False,
- path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'
- )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_master.py
deleted file mode 100755
index 390576b..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_master.py
+++ /dev/null
@@ -1,87 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-
-from resource_management.core.exceptions import ComponentIsNotRunning
-from resource_management.core.logger import Logger
-from resource_management.core.resources.system import File
-from resource_management.core.source import Template
-from resource_management.libraries.functions.format import format
-from resource_management.libraries.script import Script
-
-from enrichment_commands import EnrichmentCommands
-
-
-class Enrichment(Script):
- def install(self, env):
- from params import params
- env.set_params(params)
- commands = EnrichmentCommands(params)
- commands.setup_repo()
- Logger.info('Install RPM packages')
- self.install_packages(env)
- self.configure(env)
-
- def configure(self, env, upgrade_type=None, config_dir=None):
- from params import params
- env.set_params(params)
-
- File(format("{metron_config_path}/enrichment.properties"),
- content=Template("enrichment.properties.j2"),
- owner=params.metron_user,
- group=params.metron_group
- )
-
- def start(self, env, upgrade_type=None):
- from params import params
- env.set_params(params)
- commands = EnrichmentCommands(params)
-
- if not commands.is_configured():
- commands.init_kafka_topics()
- commands.create_hbase_tables()
- commands.set_configured()
-
- commands.start_enrichment_topology()
-
- def stop(self, env, upgrade_type=None):
- from params import params
- env.set_params(params)
- commands = EnrichmentCommands(params)
- commands.stop_enrichment_topology()
-
- def status(self, env):
- from params import status_params
- env.set_params(status_params)
- commands = EnrichmentCommands(status_params)
-
- if not commands.is_topology_active(env):
- raise ComponentIsNotRunning()
-
- def restart(self, env):
- from params import params
- env.set_params(params)
- commands = EnrichmentCommands(params)
- commands.restart_enrichment_topology(env)
-
- def kafkabuild(self, env, upgrade_type=None):
- from params import params
- env.set_params(params)
- commands = EnrichmentCommands(params)
- commands.init_kafka_topics()
-
-
-if __name__ == "__main__":
- Enrichment().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py
deleted file mode 100755
index 8a7e3c6..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py
+++ /dev/null
@@ -1,142 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-
-import os
-import time
-
-from resource_management.core.logger import Logger
-from resource_management.core.resources.system import Execute, File
-
-import metron_service
-
-
-# Wrap major operations and functionality in this class
-class IndexingCommands:
- __params = None
- __indexing = None
- __configured = False
-
- def __init__(self, params):
- if params is None:
- raise ValueError("params argument is required for initialization")
- self.__params = params
- self.__indexing = params.metron_indexing_topology
- self.__configured = os.path.isfile(self.__params.indexing_configured_flag_file)
-
- def is_configured(self):
- return self.__configured
-
- def set_configured(self):
- File(self.__params.indexing_configured_flag_file,
- content="",
- owner=self.__params.metron_user,
- mode=0775)
-
- def setup_repo(self):
- def local_repo():
- Logger.info("Setting up local repo")
- Execute("yum -y install createrepo")
- Execute("createrepo /localrepo")
- Execute("chmod -R o-w+r /localrepo")
- Execute("echo \"[METRON-0.2.0BETA]\n"
- "name=Metron 0.2.0BETA packages\n"
- "baseurl=file:///localrepo\n"
- "gpgcheck=0\n"
- "enabled=1\" > /etc/yum.repos.d/local.repo")
-
- def remote_repo():
- print('Using remote repo')
-
- yum_repo_types = {
- 'local': local_repo,
- 'remote': remote_repo
- }
- repo_type = self.__params.yum_repo_type
- if repo_type in yum_repo_types:
- yum_repo_types[repo_type]()
- else:
- raise ValueError("Unsupported repo type '{0}'".format(repo_type))
-
- def init_kafka_topics(self):
- Logger.info('Creating Kafka topics')
- command_template = """{0}/kafka-topics.sh \
- --zookeeper {1} \
- --create \
- --topic {2} \
- --partitions {3} \
- --replication-factor {4} \
- --config retention.bytes={5}"""
- num_partitions = 1
- replication_factor = 1
- retention_gigabytes = int(self.__params.metron_topic_retention)
- retention_bytes = retention_gigabytes * 1024 * 1024 * 1024
- Logger.info("Creating topics for indexing")
-
- Logger.info("Creating topic'{0}'".format(self.__indexing))
- Execute(command_template.format(self.__params.kafka_bin_dir,
- self.__params.zookeeper_quorum,
- self.__indexing,
- num_partitions,
- replication_factor,
- retention_bytes))
- Logger.info("Done creating Kafka topics")
-
- def start_indexing_topology(self):
- Logger.info("Starting Metron indexing topology: {0}".format(self.__indexing))
- start_cmd_template = """{0}/bin/start_elasticsearch_topology.sh \
- -s {1} \
- -z {2}"""
- Logger.info('Starting ' + self.__indexing)
- Execute(start_cmd_template.format(self.__params.metron_home, self.__indexing, self.__params.zookeeper_quorum))
-
- Logger.info('Finished starting indexing topology')
-
- def stop_indexing_topology(self):
- Logger.info('Stopping ' + self.__indexing)
- stop_cmd = 'storm kill ' + self.__indexing
- Execute(stop_cmd)
- Logger.info('Done stopping indexing topologies')
-
- def restart_indexing_topology(self, env):
- Logger.info('Restarting the indexing topologies')
- self.stop_indexing_topology()
-
- # Wait for old topology to be cleaned up by Storm, before starting again.
- retries = 0
- topology_active = self.is_topology_active(env)
- while self.is_topology_active(env) and retries < 3:
- Logger.info('Existing topology still active. Will wait and retry')
- time.sleep(10)
- retries += 1
-
- if not topology_active:
- Logger.info('Waiting for storm kill to complete')
- time.sleep(30)
- self.start_indexing_topology()
- Logger.info('Done restarting the indexing topologies')
- else:
- Logger.warning('Retries exhausted. Existing topology not cleaned up. Aborting topology start.')
-
- def is_topology_active(self, env):
- env.set_params(self.__params)
- active = True
- topologies = metron_service.get_running_topologies()
- is_running = False
- if self.__indexing in topologies:
- is_running = topologies[self.__indexing] in ['ACTIVE', 'REBALANCING']
- active &= is_running
- return active
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py
deleted file mode 100755
index eabf5a1..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py
+++ /dev/null
@@ -1,120 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-
-from resource_management.core.exceptions import ComponentIsNotRunning
-from resource_management.core.logger import Logger
-from resource_management.core.resources.system import Execute
-from resource_management.core.resources.system import File
-from resource_management.core.source import StaticFile
-from resource_management.libraries.functions import format as ambari_format
-from resource_management.libraries.script import Script
-
-import metron_service
-from indexing_commands import IndexingCommands
-
-
-class Indexing(Script):
- __configured = False
-
- def install(self, env):
- from params import params
- env.set_params(params)
- commands = IndexingCommands(params)
- commands.setup_repo()
- Logger.info('Install RPM packages')
- self.install_packages(env)
-
- def configure(self, env, upgrade_type=None, config_dir=None):
- from params import params
- env.set_params(params)
-
- commands = IndexingCommands(params)
- metron_service.load_global_config(params)
-
- if not commands.is_configured():
- commands.init_kafka_topics()
- commands.set_configured()
-
- def start(self, env, upgrade_type=None):
- from params import params
- env.set_params(params)
- self.configure(env)
- commands = IndexingCommands(params)
- commands.start_indexing_topology()
-
- def stop(self, env, upgrade_type=None):
- from params import params
- env.set_params(params)
- commands = IndexingCommands(params)
- commands.stop_indexing_topology()
-
- def status(self, env):
- from params import status_params
- env.set_params(status_params)
- commands = IndexingCommands(status_params)
- if not commands.is_topology_active(env):
- raise ComponentIsNotRunning()
-
- def restart(self, env):
- from params import params
- env.set_params(params)
- self.configure(env)
- commands = IndexingCommands(params)
- commands.restart_indexing_topology(env)
-
- def elasticsearch_template_install(self, env):
- from params import params
- env.set_params(params)
-
- File(params.bro_index_path,
- mode=0755,
- content=StaticFile('bro_index.template')
- )
-
- File(params.snort_index_path,
- mode=0755,
- content=StaticFile('snort_index.template')
- )
-
- File(params.yaf_index_path,
- mode=0755,
- content=StaticFile('yaf_index.template')
- )
-
- bro_cmd = ambari_format(
- 'curl -s -XPOST http://{es_url}/_template/bro_index -d @{bro_index_path}')
- Execute(bro_cmd, logoutput=True)
- snort_cmd = ambari_format(
- 'curl -s -XPOST http://{es_url}/_template/snort_index -d @{snort_index_path}')
- Execute(snort_cmd, logoutput=True)
- yaf_cmd = ambari_format(
- 'curl -s -XPOST http://{es_url}/_template/yaf_index -d @{yaf_index_path}')
- Execute(yaf_cmd, logoutput=True)
-
- def elasticsearch_template_delete(self, env):
- from params import params
- env.set_params(params)
-
- bro_cmd = ambari_format('curl -s -XDELETE "http://{es_url}/bro_index*"')
- Execute(bro_cmd, logoutput=True)
- snort_cmd = ambari_format('curl -s -XDELETE "http://{es_url}/snort_index*"')
- Execute(snort_cmd, logoutput=True)
- yaf_cmd = ambari_format('curl -s -XDELETE "http://{es_url}/yaf_index*"')
- Execute(yaf_cmd, logoutput=True)
-
-
-if __name__ == "__main__":
- Indexing().execute()