You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by mm...@apache.org on 2016/09/29 16:04:53 UTC
[2/4] incubator-metron git commit: METRON-398: Bump release version
to 0.2.1BETA in master (mmiklavc) closes apache/incubator-metron#280
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/yaf_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/yaf_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/yaf_index.template
new file mode 100644
index 0000000..7743afc
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/files/yaf_index.template
@@ -0,0 +1,205 @@
+{
+ "template": "yaf_index*",
+ "mappings": {
+ "yaf_doc": {
+ "_timestamp": {
+ "enabled": true
+ },
+ "dynamic_templates": [
+ {
+ "geo_location_point": {
+ "match": "enrichments:geo:*:location_point",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "geo_point"
+ }
+ }
+ },
+ {
+ "geo_country": {
+ "match": "enrichments:geo:*:country",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_city": {
+ "match": "enrichments:geo:*:city",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_location_id": {
+ "match": "enrichments:geo:*:locID",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_dma_code": {
+ "match": "enrichments:geo:*:dmaCode",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_postal_code": {
+ "match": "enrichments:geo:*:postalCode",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_latitude": {
+ "match": "enrichments:geo:*:latitude",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "float"
+ }
+ }
+ },
+ {
+ "geo_longitude": {
+ "match": "enrichments:geo:*:longitude",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "float"
+ }
+ }
+ },
+ {
+ "timestamps": {
+ "match": "*:ts",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "date",
+ "format": "epoch_millis"
+ }
+ }
+ }
+ ],
+ "properties": {
+ "timestamp": {
+ "type": "date",
+ "format": "epoch_millis"
+ },
+ "source:type": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "ip_dst_addr": {
+ "type": "ip"
+ },
+ "ip_dst_port": {
+ "type": "integer"
+ },
+ "ip_src_addr": {
+ "type": "ip"
+ },
+ "ip_src_port": {
+ "type": "integer"
+ },
+ "start_time": {
+ "type": "date",
+ "format": "epoch_millis"
+ },
+ "end_time": {
+ "type": "date",
+ "format": "epoch_millis"
+ },
+ "duration": {
+ "type": "double"
+ },
+ "rtt": {
+ "type": "double"
+ },
+ "proto": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "sip": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "sp": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "dip": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "dp": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "iflags": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "uflags": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "riflags": {
+ "type": "string"
+ },
+ "ruflags": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "isn": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "risn": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "tag": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "rtag": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "pkt": {
+ "type": "integer"
+ },
+ "oct": {
+ "type": "integer"
+ },
+ "rpkt": {
+ "type": "integer"
+ },
+ "roct": {
+ "type": "integer"
+ },
+ "app": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "end-reason": {
+ "type": "string"
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/enrichment_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/enrichment_commands.py
new file mode 100755
index 0000000..65b834b
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/enrichment_commands.py
@@ -0,0 +1,161 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import os
+import time
+
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute, File
+
+import metron_service
+
+
+# Wrap major operations and functionality in this class
+class EnrichmentCommands:
+ __params = None
+ __enrichment_topology = None
+ __enrichment_topic = None
+ __configured = False
+
+ def __init__(self, params):
+ if params is None:
+ raise ValueError("params argument is required for initialization")
+ self.__params = params
+ self.__enrichment_topology = params.metron_enrichment_topology
+ self.__enrichment_topic = params.metron_enrichment_topic
+ self.__configured = os.path.isfile(self.__params.enrichment_configured_flag_file)
+
+ def is_configured(self):
+ return self.__configured
+
+ def set_configured(self):
+ File(self.__params.enrichment_configured_flag_file,
+ content="",
+ owner=self.__params.metron_user,
+ mode=0775)
+
+ def setup_repo(self):
+ def local_repo():
+ Logger.info("Setting up local repo")
+ Execute("yum -y install createrepo")
+ Execute("createrepo /localrepo")
+ Execute("chmod -R o-w+r /localrepo")
+ Execute("echo \"[METRON-0.2.1BETA]\n"
+ "name=Metron 0.2.1BETA packages\n"
+ "baseurl=file:///localrepo\n"
+ "gpgcheck=0\n"
+ "enabled=1\" > /etc/yum.repos.d/local.repo")
+
+ def remote_repo():
+ print('Using remote repo')
+
+ yum_repo_types = {
+ 'local': local_repo,
+ 'remote': remote_repo
+ }
+ repo_type = self.__params.yum_repo_type
+ if repo_type in yum_repo_types:
+ yum_repo_types[repo_type]()
+ else:
+ raise ValueError("Unsupported repo type '{0}'".format(repo_type))
+
+ def init_kafka_topics(self):
+ Logger.info('Creating Kafka topics')
+ command_template = """{0}/kafka-topics.sh \
+ --zookeeper {1} \
+ --create \
+ --topic {2} \
+ --partitions {3} \
+ --replication-factor {4} \
+ --config retention.bytes={5}"""
+ num_partitions = 1
+ replication_factor = 1
+ retention_gigabytes = int(self.__params.metron_topic_retention)
+ retention_bytes = retention_gigabytes * 1024 * 1024 * 1024
+ Logger.info("Creating topics for enrichment")
+
+ Logger.info("Creating topic'{0}'".format(self.__enrichment_topic))
+ Execute(command_template.format(self.__params.kafka_bin_dir,
+ self.__params.zookeeper_quorum,
+ self.__enrichment_topic,
+ num_partitions,
+ replication_factor,
+ retention_bytes))
+ Logger.info("Done creating Kafka topics")
+
+ def start_enrichment_topology(self):
+ Logger.info("Starting Metron enrichment topology: {0}".format(self.__enrichment_topology))
+ start_cmd_template = """{0}/bin/start_enrichment_topology.sh \
+ -s {1} \
+ -z {2}"""
+ Logger.info('Starting ' + self.__enrichment_topology)
+ Execute(start_cmd_template.format(self.__params.metron_home, self.__enrichment_topology, self.__params.zookeeper_quorum))
+
+ Logger.info('Finished starting enrichment topology')
+
+ def stop_enrichment_topology(self):
+ Logger.info('Stopping ' + self.__enrichment_topology)
+ stop_cmd = 'storm kill ' + self.__enrichment_topology
+ Execute(stop_cmd)
+ Logger.info('Done stopping enrichment topologies')
+
+ def restart_enrichment_topology(self, env):
+ Logger.info('Restarting the enrichment topologies')
+ self.stop_enrichment_topology()
+
+ # Wait for old topology to be cleaned up by Storm, before starting again.
+ retries = 0
+ topology_active = self.is_topology_active(env)
+ while topology_active and retries < 3:
+ Logger.info('Existing topology still active. Will wait and retry')
+ time.sleep(40)
+ topology_active = self.is_topology_active(env)
+ retries += 1
+
+ if not topology_active:
+ self.start_enrichment_topology()
+ Logger.info('Done restarting the enrichment topology')
+ else:
+ Logger.warning('Retries exhausted. Existing topology not cleaned up. Aborting topology start.')
+
+ def is_topology_active(self, env):
+ env.set_params(self.__params)
+
+ active = True
+ topologies = metron_service.get_running_topologies()
+ is_running = False
+ if self.__enrichment_topology in topologies:
+ is_running = topologies[self.__enrichment_topology] in ['ACTIVE', 'REBALANCING']
+ active &= is_running
+ return active
+
+ def create_hbase_tables(self):
+ add_enrichment_cmd = "echo \"create '{0}','{1}'\" | hbase shell -n".format(self.__params.enrichment_table, self.__params.enrichment_cf)
+ Execute(add_enrichment_cmd,
+ tries=3,
+ try_sleep=5,
+ logoutput=False,
+ path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'
+ )
+
+ add_threatintel_cmd = "echo \"create '{0}','{1}'\" | hbase shell -n".format(self.__params.threatintel_table, self.__params.threatintel_cf)
+ Execute(add_threatintel_cmd,
+ tries=3,
+ try_sleep=5,
+ logoutput=False,
+ path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'
+ )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/enrichment_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/enrichment_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/enrichment_master.py
new file mode 100755
index 0000000..390576b
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/enrichment_master.py
@@ -0,0 +1,87 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import File
+from resource_management.core.source import Template
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.script import Script
+
+from enrichment_commands import EnrichmentCommands
+
+
+class Enrichment(Script):
+ def install(self, env):
+ from params import params
+ env.set_params(params)
+ commands = EnrichmentCommands(params)
+ commands.setup_repo()
+ Logger.info('Install RPM packages')
+ self.install_packages(env)
+ self.configure(env)
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ from params import params
+ env.set_params(params)
+
+ File(format("{metron_config_path}/enrichment.properties"),
+ content=Template("enrichment.properties.j2"),
+ owner=params.metron_user,
+ group=params.metron_group
+ )
+
+ def start(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ commands = EnrichmentCommands(params)
+
+ if not commands.is_configured():
+ commands.init_kafka_topics()
+ commands.create_hbase_tables()
+ commands.set_configured()
+
+ commands.start_enrichment_topology()
+
+ def stop(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ commands = EnrichmentCommands(params)
+ commands.stop_enrichment_topology()
+
+ def status(self, env):
+ from params import status_params
+ env.set_params(status_params)
+ commands = EnrichmentCommands(status_params)
+
+ if not commands.is_topology_active(env):
+ raise ComponentIsNotRunning()
+
+ def restart(self, env):
+ from params import params
+ env.set_params(params)
+ commands = EnrichmentCommands(params)
+ commands.restart_enrichment_topology(env)
+
+ def kafkabuild(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ commands = EnrichmentCommands(params)
+ commands.init_kafka_topics()
+
+
+if __name__ == "__main__":
+ Enrichment().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/indexing_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/indexing_commands.py
new file mode 100755
index 0000000..fdb90e9
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/indexing_commands.py
@@ -0,0 +1,142 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import os
+import time
+
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute, File
+
+import metron_service
+
+
+# Wrap major operations and functionality in this class
+class IndexingCommands:
+ __params = None
+ __indexing = None
+ __configured = False
+
+ def __init__(self, params):
+ if params is None:
+ raise ValueError("params argument is required for initialization")
+ self.__params = params
+ self.__indexing = params.metron_indexing_topology
+ self.__configured = os.path.isfile(self.__params.indexing_configured_flag_file)
+
+ def is_configured(self):
+ return self.__configured
+
+ def set_configured(self):
+ File(self.__params.indexing_configured_flag_file,
+ content="",
+ owner=self.__params.metron_user,
+ mode=0775)
+
+ def setup_repo(self):
+ def local_repo():
+ Logger.info("Setting up local repo")
+ Execute("yum -y install createrepo")
+ Execute("createrepo /localrepo")
+ Execute("chmod -R o-w+r /localrepo")
+ Execute("echo \"[METRON-0.2.1BETA]\n"
+ "name=Metron 0.2.1BETA packages\n"
+ "baseurl=file:///localrepo\n"
+ "gpgcheck=0\n"
+ "enabled=1\" > /etc/yum.repos.d/local.repo")
+
+ def remote_repo():
+ print('Using remote repo')
+
+ yum_repo_types = {
+ 'local': local_repo,
+ 'remote': remote_repo
+ }
+ repo_type = self.__params.yum_repo_type
+ if repo_type in yum_repo_types:
+ yum_repo_types[repo_type]()
+ else:
+ raise ValueError("Unsupported repo type '{0}'".format(repo_type))
+
+ def init_kafka_topics(self):
+ Logger.info('Creating Kafka topics')
+ command_template = """{0}/kafka-topics.sh \
+ --zookeeper {1} \
+ --create \
+ --topic {2} \
+ --partitions {3} \
+ --replication-factor {4} \
+ --config retention.bytes={5}"""
+ num_partitions = 1
+ replication_factor = 1
+ retention_gigabytes = int(self.__params.metron_topic_retention)
+ retention_bytes = retention_gigabytes * 1024 * 1024 * 1024
+ Logger.info("Creating topics for indexing")
+
+ Logger.info("Creating topic'{0}'".format(self.__indexing))
+ Execute(command_template.format(self.__params.kafka_bin_dir,
+ self.__params.zookeeper_quorum,
+ self.__indexing,
+ num_partitions,
+ replication_factor,
+ retention_bytes))
+ Logger.info("Done creating Kafka topics")
+
+ def start_indexing_topology(self):
+ Logger.info("Starting Metron indexing topology: {0}".format(self.__indexing))
+ start_cmd_template = """{0}/bin/start_elasticsearch_topology.sh \
+ -s {1} \
+ -z {2}"""
+ Logger.info('Starting ' + self.__indexing)
+ Execute(start_cmd_template.format(self.__params.metron_home, self.__indexing, self.__params.zookeeper_quorum))
+
+ Logger.info('Finished starting indexing topology')
+
+ def stop_indexing_topology(self):
+ Logger.info('Stopping ' + self.__indexing)
+ stop_cmd = 'storm kill ' + self.__indexing
+ Execute(stop_cmd)
+ Logger.info('Done stopping indexing topologies')
+
+ def restart_indexing_topology(self, env):
+ Logger.info('Restarting the indexing topologies')
+ self.stop_indexing_topology()
+
+ # Wait for old topology to be cleaned up by Storm, before starting again.
+ retries = 0
+ topology_active = self.is_topology_active(env)
+ while self.is_topology_active(env) and retries < 3:
+ Logger.info('Existing topology still active. Will wait and retry')
+ time.sleep(10)
+ retries += 1
+
+ if not topology_active:
+ Logger.info('Waiting for storm kill to complete')
+ time.sleep(30)
+ self.start_indexing_topology()
+ Logger.info('Done restarting the indexing topologies')
+ else:
+ Logger.warning('Retries exhausted. Existing topology not cleaned up. Aborting topology start.')
+
+ def is_topology_active(self, env):
+ env.set_params(self.__params)
+ active = True
+ topologies = metron_service.get_running_topologies()
+ is_running = False
+ if self.__indexing in topologies:
+ is_running = topologies[self.__indexing] in ['ACTIVE', 'REBALANCING']
+ active &= is_running
+ return active
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/indexing_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/indexing_master.py
new file mode 100755
index 0000000..eabf5a1
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/indexing_master.py
@@ -0,0 +1,120 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute
+from resource_management.core.resources.system import File
+from resource_management.core.source import StaticFile
+from resource_management.libraries.functions import format as ambari_format
+from resource_management.libraries.script import Script
+
+import metron_service
+from indexing_commands import IndexingCommands
+
+
+class Indexing(Script):
+ __configured = False
+
+ def install(self, env):
+ from params import params
+ env.set_params(params)
+ commands = IndexingCommands(params)
+ commands.setup_repo()
+ Logger.info('Install RPM packages')
+ self.install_packages(env)
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ from params import params
+ env.set_params(params)
+
+ commands = IndexingCommands(params)
+ metron_service.load_global_config(params)
+
+ if not commands.is_configured():
+ commands.init_kafka_topics()
+ commands.set_configured()
+
+ def start(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ self.configure(env)
+ commands = IndexingCommands(params)
+ commands.start_indexing_topology()
+
+ def stop(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ commands = IndexingCommands(params)
+ commands.stop_indexing_topology()
+
+ def status(self, env):
+ from params import status_params
+ env.set_params(status_params)
+ commands = IndexingCommands(status_params)
+ if not commands.is_topology_active(env):
+ raise ComponentIsNotRunning()
+
+ def restart(self, env):
+ from params import params
+ env.set_params(params)
+ self.configure(env)
+ commands = IndexingCommands(params)
+ commands.restart_indexing_topology(env)
+
+ def elasticsearch_template_install(self, env):
+ from params import params
+ env.set_params(params)
+
+ File(params.bro_index_path,
+ mode=0755,
+ content=StaticFile('bro_index.template')
+ )
+
+ File(params.snort_index_path,
+ mode=0755,
+ content=StaticFile('snort_index.template')
+ )
+
+ File(params.yaf_index_path,
+ mode=0755,
+ content=StaticFile('yaf_index.template')
+ )
+
+ bro_cmd = ambari_format(
+ 'curl -s -XPOST http://{es_url}/_template/bro_index -d @{bro_index_path}')
+ Execute(bro_cmd, logoutput=True)
+ snort_cmd = ambari_format(
+ 'curl -s -XPOST http://{es_url}/_template/snort_index -d @{snort_index_path}')
+ Execute(snort_cmd, logoutput=True)
+ yaf_cmd = ambari_format(
+ 'curl -s -XPOST http://{es_url}/_template/yaf_index -d @{yaf_index_path}')
+ Execute(yaf_cmd, logoutput=True)
+
+ def elasticsearch_template_delete(self, env):
+ from params import params
+ env.set_params(params)
+
+ bro_cmd = ambari_format('curl -s -XDELETE "http://{es_url}/bro_index*"')
+ Execute(bro_cmd, logoutput=True)
+ snort_cmd = ambari_format('curl -s -XDELETE "http://{es_url}/snort_index*"')
+ Execute(snort_cmd, logoutput=True)
+ yaf_cmd = ambari_format('curl -s -XDELETE "http://{es_url}/yaf_index*"')
+ Execute(yaf_cmd, logoutput=True)
+
+
+if __name__ == "__main__":
+ Indexing().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/metron_service.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/metron_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/metron_service.py
new file mode 100644
index 0000000..57da2c7
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/metron_service.py
@@ -0,0 +1,76 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import json
+import subprocess
+
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Directory, File
+from resource_management.core.resources.system import Execute
+from resource_management.core.source import InlineTemplate
+from resource_management.libraries.functions import format as ambari_format
+
+
+def init_config():
+ Logger.info('Loading config into ZooKeeper')
+ Execute(ambari_format(
+ "{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_quorum}"),
+ path=ambari_format("{java_home}/bin")
+ )
+
+
+def get_running_topologies():
+ Logger.info('Getting Running Storm Topologies from Storm REST Server')
+
+ cmd = ambari_format('curl --max-time 3 {storm_rest_addr}/api/v1/topology/summary')
+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+ (stdout, stderr) = proc.communicate()
+
+ try:
+ stormjson = json.loads(stdout)
+ except ValueError:
+ return {}
+
+ topologiesDict = {}
+
+ for topology in stormjson['topologies']:
+ topologiesDict[topology['name']] = topology['status']
+
+ Logger.info("Topologies: " + str(topologiesDict))
+ return topologiesDict
+
+
+def load_global_config(params):
+ Logger.info('Create Metron Local Config Directory')
+ Logger.info("Configure Metron global.json")
+
+ directories = [params.metron_zookeeper_config_path]
+ Directory(directories,
+ mode=0755,
+ owner=params.metron_user,
+ group=params.metron_group
+ )
+
+ File("{0}/global.json".format(params.metron_zookeeper_config_path),
+ owner=params.metron_user,
+ content=InlineTemplate(params.global_json_template)
+ )
+
+ File("{0}/elasticsearch.properties".format(params.metron_zookeeper_config_path + '/..'),
+ owner=params.metron_user,
+ content=InlineTemplate(params.global_properties_template))
+
+ init_config()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_server.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_server.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_server.py
new file mode 100755
index 0000000..2fc363f
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_server.py
@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.libraries.script.script import Script
+
+import mysql_users
+from mysql_service import mysql_service
+from mysql_utils import mysql_configure
+
+
+class MysqlServer(Script):
+ def install(self, env):
+ self.install_packages(env)
+ self.configure(env)
+
+ def clean(self, env):
+ from params import params
+ env.set_params(params)
+ mysql_users.mysql_deluser()
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ from params import params
+ env.set_params(params)
+ mysql_configure()
+
+ def start(self, env, rolling_restart=False):
+ from params import params
+ env.set_params(params)
+ mysql_service(daemon_name=params.daemon_name, action='start')
+
+ def stop(self, env, rolling_restart=False):
+ from params import params
+ env.set_params(params)
+ mysql_service(daemon_name=params.daemon_name, action='stop')
+
+ def status(self, env):
+ from params import status_params
+ env.set_params(status_params)
+
+ mysql_service(daemon_name=status_params.daemon_name, action='status')
+
+
+if __name__ == "__main__":
+ MysqlServer().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_service.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_service.py
new file mode 100755
index 0000000..2e0ce8b
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_service.py
@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.exceptions import ComponentIsNotRunning, Fail
+from resource_management.core.resources.system import Execute
+from resource_management.libraries.functions.format import format
+
+
+def mysql_service(daemon_name=None, action='start'):
+ status_cmd = format("pgrep -l '^{mysql_process_name}$'")
+ cmd = ('service', daemon_name, action)
+
+ if action == 'status':
+ try:
+ Execute(status_cmd)
+ except Fail:
+ raise ComponentIsNotRunning()
+ elif action == 'stop':
+ Execute(cmd,
+ logoutput=True,
+ only_if=status_cmd,
+ sudo=True,
+ )
+ elif action == 'start':
+ Execute(cmd,
+ logoutput=True,
+ not_if=status_cmd,
+ sudo=True,
+ )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_users.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_users.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_users.py
new file mode 100755
index 0000000..1721eba
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_users.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.resources.system import Execute, File
+from resource_management.core.source import StaticFile
+from resource_management.libraries.functions.format import format
+
+
+# Used to add metron access to the needed components
+def mysql_adduser():
+ from params import params
+
+ File(params.mysql_adduser_path,
+ mode=0755,
+ content=StaticFile('addMysqlUser.sh')
+ )
+
+ add_user_cmd = format("bash -x {mysql_adduser_path} {daemon_name} {metron_user} {enrichment_metron_user_passwd!p} {enrichment_host}")
+ Execute(add_user_cmd,
+ tries=3,
+ try_sleep=5,
+ logoutput=False,
+ path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'
+ )
+
+
+# Removes hive metron from components
+def mysql_deluser():
+ from params import params
+
+ File(params.mysql_deluser_path,
+ mode=0755,
+ content=StaticFile('removeMysqlUser.sh')
+ )
+
+ del_user_cmd = format("bash -x {mysql_deluser_path} {daemon_name} {metron_user} {enrichment_host}")
+ Execute(del_user_cmd,
+ tries=3,
+ try_sleep=5,
+ path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
+ )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_utils.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_utils.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_utils.py
new file mode 100755
index 0000000..40b925d
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/mysql_utils.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.resources.system import Execute, File
+from resource_management.core.source import StaticFile
+from resource_management.libraries.functions.format import format
+
+import mysql_users
+
+
+def mysql_configure():
+ from params import params
+
+ # required for running hive
+ replace_bind_address = ('sed', '-i', 's|^bind-address[ \t]*=.*|bind-address = 0.0.0.0|', params.mysql_configname)
+ Execute(replace_bind_address,
+ sudo=True,
+ )
+
+ # this also will start mysql-server
+ mysql_users.mysql_adduser()
+
+ File(params.mysql_create_geoip_path,
+ mode=0755,
+ content=StaticFile('createMysqlGeoIp.sh')
+ )
+
+ geoip_setup_cmd = format("bash -x {mysql_create_geoip_path} {daemon_name} {geoip_ddl} {geoip_url}")
+ Execute(geoip_setup_cmd,
+ tries=3,
+ try_sleep=5,
+ path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
+ )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/__init__.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/__init__.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/__init__.py
new file mode 100755
index 0000000..242460e
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/__init__.py
@@ -0,0 +1,18 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params.py
new file mode 100755
index 0000000..953435d
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from ambari_commons import OSCheck
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.expect import expect
+
+if OSCheck.is_windows_family():
+ from params_windows import *
+else:
+ from params_linux import *
+
+java_home = config['hostLevelParams']['java_home']
+java_version = expect("/hostLevelParams/java_version", int)
+
+host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params_linux.py
new file mode 100755
index 0000000..5f914f3
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params_linux.py
@@ -0,0 +1,169 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import functools
+import os
+
+from ambari_commons.os_check import OSCheck
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions import get_kinit_path
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
+from resource_management.libraries.functions.is_empty import is_empty
+from resource_management.libraries.resources.hdfs_resource import HdfsResource
+from resource_management.libraries.script import Script
+
+import status_params
+
+# server configurations
+config = Script.get_config()
+tmp_dir = Script.get_tmp_dir()
+
+hostname = config['hostname']
+metron_home = status_params.metron_home
+parsers = status_params.parsers
+metron_ddl_dir = metron_home + '/ddl'
+geoip_ddl = metron_ddl_dir + '/geoip_ddl.sql'
+geoip_url = config['configurations']['metron-env']['geoip_url']
+metron_indexing_topology = status_params.metron_indexing_topology
+metron_user = config['configurations']['metron-env']['metron_user']
+metron_group = config['configurations']['metron-env']['metron_group']
+metron_config_path = metron_home + '/config'
+metron_zookeeper_config_dir = status_params.metron_zookeeper_config_dir
+metron_zookeeper_config_path = status_params.metron_zookeeper_config_path
+parsers_configured_flag_file = status_params.parsers_configured_flag_file
+enrichment_configured_flag_file = status_params.enrichment_configured_flag_file
+indexing_configured_flag_file = status_params.indexing_configured_flag_file
+global_json_template = config['configurations']['metron-env']['global-json']
+global_properties_template = config['configurations']['metron-env']['elasticsearch-properties']
+es_cluster_name = config['configurations']['metron-env']['es_cluster_name']
+es_url = config['configurations']['metron-env']['es_url']
+yum_repo_type = 'local'
+
+# hadoop params
+stack_root = Script.get_stack_root()
+hadoop_home_dir = stack_select.get_hadoop_dir("home")
+hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
+hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
+kafka_home = os.path.join(stack_root, "current", "kafka-broker")
+kafka_bin_dir = os.path.join(kafka_home, "bin")
+
+# zookeeper
+zk_hosts = default("/clusterHostInfo/zookeeper_hosts", [])
+has_zk_host = not len(zk_hosts) == 0
+zookeeper_quorum = None
+if has_zk_host:
+ if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']:
+ zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort']
+ else:
+ zookeeper_clientPort = '2181'
+ zookeeper_quorum = (':' + zookeeper_clientPort + ',').join(config['clusterHostInfo']['zookeeper_hosts'])
+ # last port config
+ zookeeper_quorum += ':' + zookeeper_clientPort
+
+# Storm
+storm_rest_addr = status_params.storm_rest_addr
+
+# Kafka
+kafka_hosts = default("/clusterHostInfo/kafka_broker_hosts", [])
+has_kafka_host = not len(kafka_hosts) == 0
+kafka_brokers = None
+if has_kafka_host:
+ if 'port' in config['configurations']['kafka-broker']:
+ kafka_broker_port = config['configurations']['kafka-broker']['port']
+ else:
+ kafka_broker_port = '6667'
+ kafka_brokers = (':' + kafka_broker_port + ',').join(config['clusterHostInfo']['kafka_broker_hosts'])
+ kafka_brokers += ':' + kafka_broker_port
+
+metron_apps_dir = config['configurations']['metron-env']['metron_apps_hdfs_dir']
+metron_topic_retention = config['configurations']['metron-env']['metron_topic_retention']
+
+local_grok_patterns_dir = format("{metron_home}/patterns")
+hdfs_grok_patterns_dir = format("{metron_apps_dir}/patterns")
+
+# for create_hdfs_directory
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
+smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
+kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
+hdfs_site = config['configurations']['hdfs-site']
+default_fs = config['configurations']['core-site']['fs.defaultFS']
+dfs_type = default("/commandParams/dfs_type", "")
+
+# MYSQL
+if OSCheck.is_ubuntu_family():
+ mysql_configname = '/etc/mysql/my.cnf'
+else:
+ mysql_configname = '/etc/my.cnf'
+
+daemon_name = status_params.daemon_name
+# There will always be exactly one mysql_host
+mysql_host = config['clusterHostInfo']['metron_enrichment_mysql_server_hosts'][0]
+mysql_port = config['configurations']['metron-env']['metron_enrichment_db_port']
+
+mysql_adduser_path = tmp_dir + "/addMysqlUser.sh"
+mysql_deluser_path = tmp_dir + "/removeMysqlUser.sh"
+mysql_create_geoip_path = tmp_dir + "/createMysqlGeoIp.sh"
+
+enrichment_hosts = default("/clusterHostInfo/enrichment_host", [])
+enrichment_host = enrichment_hosts[0] if len(enrichment_hosts) > 0 else None
+
+enrichment_metron_user = config['configurations']['metron-env']['metron_enrichment_db_user']
+enrichment_metron_user_passwd = config['configurations']['metron-env']['metron_enrichment_db_password']
+enrichment_metron_user_passwd = unicode(enrichment_metron_user_passwd) if not is_empty(
+ enrichment_metron_user_passwd) else enrichment_metron_user_passwd
+mysql_process_name = status_params.mysql_process_name
+
+# create partial functions with common arguments for every HdfsResource call
+# to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code
+HdfsResource = functools.partial(
+ HdfsResource,
+ user=hdfs_user,
+ hdfs_resource_ignore_file="/var/lib/ambari-agent/data/.hdfs_resource_ignore",
+ security_enabled=security_enabled,
+ keytab=hdfs_user_keytab,
+ kinit_path_local=kinit_path_local,
+ hadoop_bin_dir=hadoop_bin_dir,
+ hadoop_conf_dir=hadoop_conf_dir,
+ principal_name=hdfs_principal_name,
+ hdfs_site=hdfs_site,
+ default_fs=default_fs,
+ immutable_paths=get_not_managed_resources(),
+ dfs_type=dfs_type
+)
+
+# HBase
+enrichment_table = status_params.enrichment_table
+enrichment_cf = status_params.enrichment_cf
+threatintel_table = status_params.threatintel_table
+threatintel_cf = status_params.threatintel_cf
+
+metron_enrichment_topology = status_params.metron_enrichment_topology
+metron_enrichment_topic = status_params.metron_enrichment_topic
+
+# ES Templates
+bro_index_path = tmp_dir + "/bro_index.template"
+snort_index_path = tmp_dir + "/snort_index.template"
+yaf_index_path = tmp_dir + "/yaf_index.template"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params_windows.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params_windows.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params_windows.py
new file mode 100755
index 0000000..4d11b35
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/params_windows.py
@@ -0,0 +1,20 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+raise NotImplementedError
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/status_params.py
new file mode 100644
index 0000000..a0310d2
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/params/status_params.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from ambari_commons import OSCheck
+from resource_management.libraries.functions import format
+from resource_management.libraries.script import Script
+
+config = Script.get_config()
+
+# Parsers
+parsers = config['configurations']['metron-env']['parsers']
+metron_home = config['configurations']['metron-env']['metron_home']
+metron_zookeeper_config_dir = config['configurations']['metron-env']['metron_zookeeper_config_dir']
+metron_zookeeper_config_path = format('{metron_home}/{metron_zookeeper_config_dir}')
+parsers_configured_flag_file = metron_zookeeper_config_path + '/../metron_parsers_configured'
+
+# Enrichment
+metron_enrichment_topology = 'enrichment'
+metron_enrichment_topic = 'enrichments'
+
+enrichment_table = 'enrichment'
+enrichment_cf = 't'
+threatintel_table = 'threatintel'
+threatintel_cf = 't'
+
+mysql_process_name = 'mysqld'
+if OSCheck.is_suse_family() or OSCheck.is_ubuntu_family():
+ daemon_name = 'mysql'
+else:
+ daemon_name = 'mysqld'
+
+# ing
+metron_indexing_topology = config['configurations']['metron-env']['metron_indexing_topology']
+indexing_configured_flag_file = metron_zookeeper_config_path + '/../metron_indexing_configured'
+
+# Enrichment
+enrichment_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_is_configured'
+
+# Storm
+storm_rest_addr = config['configurations']['metron-env']['storm_rest_addr']
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/parser_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/parser_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/parser_commands.py
new file mode 100755
index 0000000..07e36db
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/parser_commands.py
@@ -0,0 +1,208 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+import re
+import subprocess
+import time
+
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute, File
+
+import metron_service
+
+
+# Wrap major operations and functionality in this class
+class ParserCommands:
+ __params = None
+ __parser_list = None
+ __configured = False
+
+ def __init__(self, params):
+ if params is None:
+ raise ValueError("params argument is required for initialization")
+ self.__params = params
+ self.__parser_list = self.__get_parsers(params)
+ self.__configured = os.path.isfile(self.__params.parsers_configured_flag_file)
+
+ # get list of parsers
+ def __get_parsers(self, params):
+ return params.parsers.replace(' ', '').split(',')
+
+ def is_configured(self):
+ return self.__configured
+
+ def set_configured(self):
+ File(self.__params.parsers_configured_flag_file,
+ content="",
+ owner=self.__params.metron_user,
+ mode=0775)
+
+ def init_parsers(self):
+ Logger.info(
+ "Copying grok patterns from local directory '{0}' to HDFS '{1}'".format(self.__params.local_grok_patterns_dir,
+ self.__params.metron_apps_dir))
+ self.__params.HdfsResource(self.__params.metron_apps_dir,
+ type="directory",
+ action="create_on_execute",
+ owner=self.__params.metron_user,
+ mode=0775,
+ source=self.__params.local_grok_patterns_dir)
+
+ Logger.info("Done initializing parser configuration")
+
+ def get_parser_list(self):
+ return self.__parser_list
+
+ def setup_repo(self):
+ def local_repo():
+ Logger.info("Setting up local repo")
+ Execute("yum -y install createrepo")
+ Execute("createrepo /localrepo")
+ Execute("chmod -R o-w+r /localrepo")
+ Execute("echo \"[METRON-0.2.1BETA]\n"
+ "name=Metron 0.2.1BETA packages\n"
+ "baseurl=file:///localrepo\n"
+ "gpgcheck=0\n"
+ "enabled=1\" > /etc/yum.repos.d/local.repo")
+
+ def remote_repo():
+ print('Using remote repo')
+
+ yum_repo_types = {
+ 'local': local_repo,
+ 'remote': remote_repo
+ }
+ repo_type = self.__params.yum_repo_type
+ if repo_type in yum_repo_types:
+ yum_repo_types[repo_type]()
+ else:
+ raise ValueError("Unsupported repo type '{0}'".format(repo_type))
+
+ def init_kafka_topics(self):
+ Logger.info('Creating Kafka topics')
+ command_template = """{0}/kafka-topics.sh \
+ --zookeeper {1} \
+ --create \
+ --topic {2} \
+ --partitions {3} \
+ --replication-factor {4} \
+ --config retention.bytes={5}"""
+ num_partitions = 1
+ replication_factor = 1
+ retention_gigabytes = int(self.__params.metron_topic_retention)
+ retention_bytes = retention_gigabytes * 1024 * 1024 * 1024
+ Logger.info("Creating main topics for parsers")
+ for parser_name in self.get_parser_list():
+ Logger.info("Creating topic'{0}'".format(parser_name))
+ Execute(command_template.format(self.__params.kafka_bin_dir,
+ self.__params.zookeeper_quorum,
+ parser_name,
+ num_partitions,
+ replication_factor,
+ retention_bytes))
+ Logger.info("Creating topics for error handling")
+ Execute(command_template.format(self.__params.kafka_bin_dir,
+ self.__params.zookeeper_quorum,
+ "parser_invalid",
+ num_partitions,
+ replication_factor,
+ retention_bytes))
+ Execute(command_template.format(self.__params.kafka_bin_dir,
+ self.__params.zookeeper_quorum,
+ "parser_error",
+ num_partitions, replication_factor,
+ retention_bytes))
+ Logger.info("Done creating Kafka topics")
+
+ def start_parser_topologies(self):
+ Logger.info("Starting Metron parser topologies: {0}".format(self.get_parser_list()))
+ start_cmd_template = """{0}/bin/start_parser_topology.sh \
+ -k {1} \
+ -z {2} \
+ -s {3}"""
+ for parser in self.get_parser_list():
+ Logger.info('Starting ' + parser)
+ Execute(start_cmd_template.format(self.__params.metron_home, self.__params.kafka_brokers,
+ self.__params.zookeeper_quorum, parser))
+
+ Logger.info('Finished starting parser topologies')
+
+ def stop_parser_topologies(self):
+ Logger.info('Stopping parsers')
+ for parser in self.get_parser_list():
+ Logger.info('Stopping ' + parser)
+ stop_cmd = 'storm kill ' + parser
+ Execute(stop_cmd)
+ Logger.info('Done stopping parser topologies')
+
+ def restart_parser_topologies(self, env):
+ Logger.info('Restarting the parser topologies')
+ self.stop_parser_topologies()
+ attempt_count = 0
+ while self.topologies_running(env):
+ if attempt_count > 2:
+ raise Exception("Unable to kill topologies")
+ attempt_count += 1
+ time.sleep(10)
+ self.start_parser_topologies()
+ Logger.info('Done restarting the parser topologies')
+
+ def topologies_exist(self):
+ cmd_open = subprocess.Popen(["storm", "list"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ (stdout, stderr) = cmd_open.communicate()
+ stdout_lines = stdout.splitlines()
+ if stdout_lines:
+ status_lines = self.__get_status_lines(stdout_lines)
+ for parser in self.get_parser_list():
+ for line in status_lines:
+ items = re.sub('[\s]+', ' ', line).split()
+ if items and items[0] == parser:
+ return True
+ return False
+
+ def topologies_running(self, env):
+ env.set_params(self.__params)
+ all_running = True
+ topologies = metron_service.get_running_topologies()
+ for parser in self.get_parser_list():
+ parser_found = False
+ is_running = False
+ if parser in topologies:
+ parser_found = True
+ is_running = topologies[parser] in ['ACTIVE', 'REBALANCING']
+ all_running &= parser_found and is_running
+ return all_running
+
+ def __get_status_lines(self, lines):
+ status_lines = []
+ do_stat = False
+ skipped = 0
+ for line in lines:
+ if line.startswith("Topology_name"):
+ do_stat = True
+ if do_stat and skipped == 2:
+ status_lines += [line]
+ elif do_stat:
+ skipped += 1
+ return status_lines
+
+ def __is_running(self, status):
+ return status in ['ACTIVE', 'REBALANCING']
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/parser_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/parser_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/parser_master.py
new file mode 100755
index 0000000..3758873
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/parser_master.py
@@ -0,0 +1,89 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.core.logger import Logger
+from resource_management.libraries.script import Script
+
+import metron_service
+from parser_commands import ParserCommands
+
+
+class ParserMaster(Script):
+ def get_component_name(self):
+ # TODO add this at some point - currently will cause problems with hdp-select
+ # return "parser-master"
+ pass
+
+ def install(self, env):
+ from params import params
+ env.set_params(params)
+ commands = ParserCommands(params)
+ commands.setup_repo()
+ Logger.info('Install RPM packages')
+ self.install_packages(env)
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ from params import params
+ env.set_params(params)
+ metron_service.load_global_config(params)
+ commands = ParserCommands(params)
+ if not commands.is_configured():
+ commands.init_parsers()
+ commands.init_kafka_topics()
+ commands.set_configured()
+
+ def start(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ self.configure(env)
+ commands = ParserCommands(params)
+ commands.start_parser_topologies()
+
+ def stop(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ commands = ParserCommands(params)
+ commands.stop_parser_topologies()
+
+ def status(self, env):
+ from params import status_params
+ env.set_params(status_params)
+ commands = ParserCommands(status_params)
+ if not commands.topologies_running(env):
+ raise ComponentIsNotRunning()
+
+ def restart(self, env):
+ from params import params
+ env.set_params(params)
+ self.configure(env)
+ commands = ParserCommands(params)
+ commands.restart_parser_topologies(env)
+
+ def servicechecktest(self, env):
+ from params import params
+ env.set_params(params)
+ from service_check import ServiceCheck
+ service_check = ServiceCheck()
+ Logger.info('Service Check Test')
+ service_check.service_check(env)
+
+
+if __name__ == "__main__":
+ ParserMaster().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/service_check.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/service_check.py
new file mode 100755
index 0000000..7dd9dfb
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/scripts/service_check.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from __future__ import print_function
+
+from resource_management.libraries.script import Script
+
+from indexing_commands import IndexingCommands
+from parser_commands import ParserCommands
+
+
+class ServiceCheck(Script):
+ def service_check(self, env):
+ from params import params
+ parsercommands = ParserCommands(params)
+ indexingcommands = IndexingCommands(params)
+ all_found = parsercommands.topologies_running(env) and indexingcommands.is_topology_active(env)
+ if all_found:
+ exit(0)
+ else:
+ exit(1)
+
+
+if __name__ == "__main__":
+ ServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/templates/enrichment.properties.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/templates/enrichment.properties.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/templates/enrichment.properties.j2
new file mode 100755
index 0000000..bab2f52
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/package/templates/enrichment.properties.j2
@@ -0,0 +1,88 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+##### Kafka #####
+
+kafka.zk={{zookeeper_quorum}}
+kafka.broker={{kafka_brokers}}
+enrichment.output.topic=indexing
+
+##### MySQL #####
+
+mysql.ip={{mysql_host}}
+mysql.port={{mysql_port}}
+mysql.username={{enrichment_metron_user}}
+mysql.password={{enrichment_metron_user_passwd}}
+
+##### Metrics #####
+
+#reporters
+org.apache.metron.metrics.reporter.graphite=true
+org.apache.metron.metrics.reporter.console=false
+org.apache.metron.metrics.reporter.jmx=false
+
+#Graphite Addresses
+
+org.apache.metron.metrics.graphite.address=localhost
+org.apache.metron.metrics.graphite.port=2023
+
+#TelemetryParserBolt
+org.apache.metron.metrics.TelemetryParserBolt.acks=true
+org.apache.metron.metrics.TelemetryParserBolt.emits=true
+org.apache.metron.metrics.TelemetryParserBolt.fails=true
+
+
+#GenericEnrichmentBolt
+org.apache.metron.metrics.GenericEnrichmentBolt.acks=true
+org.apache.metron.metrics.GenericEnrichmentBolt.emits=true
+org.apache.metron.metrics.GenericEnrichmentBolt.fails=true
+
+
+#TelemetryIndexingBolt
+org.apache.metron.metrics.TelemetryIndexingBolt.acks=true
+org.apache.metron.metrics.TelemetryIndexingBolt.emits=true
+org.apache.metron.metrics.TelemetryIndexingBolt.fails=true
+
+##### Host Enrichment #####
+
+hbase.provider.impl=org.apache.metron.hbase.HTableProvider
+enrichment.simple.hbase.table={{enrichment_table}}
+enrichment.simple.hbase.cf={{enrichment_cf}}
+org.apache.metron.enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\
+{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},\
+{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}]
+
+
+##### HBase #####
+bolt.hbase.table.name=pcap
+bolt.hbase.table.fields=t:value
+bolt.hbase.table.key.tuple.field.name=key
+bolt.hbase.table.timestamp.tuple.field.name=timestamp
+bolt.hbase.enable.batching=false
+bolt.hbase.write.buffer.size.in.bytes=2000000
+bolt.hbase.durability=SKIP_WAL
+bolt.hbase.partitioner.region.info.refresh.interval.mins=60
+
+##### Threat Intel #####
+
+threat.intel.tracker.table={{threatintel_table}}
+threat.intel.tracker.cf={{threatintel_cf}}
+threat.intel.simple.hbase.table={{threatintel_table}}
+threat.intel.simple.hbase.cf={{threatintel_cf}}
+threat.intel.ip.table=
+threat.intel.ip.cf=
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/quicklinks/quicklinks.json
new file mode 100755
index 0000000..ee1b225
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.1BETA/quicklinks/quicklinks.json
@@ -0,0 +1,28 @@
+{
+ "name": "default",
+ "description": "default quick links configuration",
+ "configuration": {
+ "protocol":
+ {
+ "type":"HTTP_ONLY"
+ },
+
+ "links": [
+ {
+ "name": "storm_ui",
+ "label": "Storm UI",
+ "requires_user_name": "false",
+ "component_name": "STORM_UI_SERVER",
+ "url":"%@://%@:%@/",
+ "port":{
+ "http_property": "ui.port",
+ "http_default_port": "8744",
+ "https_property": "ui.port",
+ "https_default_port": "8744",
+ "regex": "^(\\d+)$",
+ "site": "storm-site"
+ }
+ }
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json
index 99dec9b..303e84f 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json
@@ -62,7 +62,7 @@
},
{
"service_name" : "METRON",
- "service_version" : "0.2.0BETA",
+ "service_version" : "0.2.1BETA",
"applicable_stacks" : [
{
"stack_name" : "HDP",
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index 6b8f195..b0ef974 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -122,6 +122,7 @@ This package installs the Metron Parser files
%dir %{metron_home}/lib
%{metron_home}/bin/start_parser_topology.sh
%{metron_home}/config/zookeeper/parsers/bro.json
+%{metron_home}/config/zookeeper/parsers/jsonMap.json
%{metron_home}/config/zookeeper/parsers/snort.json
%{metron_home}/config/zookeeper/parsers/squid.json
%{metron_home}/config/zookeeper/parsers/websphere.json
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/packaging/docker/rpm-docker/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/rpm-docker/pom.xml b/metron-deployment/packaging/docker/rpm-docker/pom.xml
index 92414c2..30e2670 100644
--- a/metron-deployment/packaging/docker/rpm-docker/pom.xml
+++ b/metron-deployment/packaging/docker/rpm-docker/pom.xml
@@ -21,11 +21,11 @@
<artifactId>metron-rpm</artifactId>
<packaging>pom</packaging>
<name>metron-rpm</name>
- <version>0.2.0BETA</version>
+ <version>0.2.1BETA</version>
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-deployment</artifactId>
- <version>0.2.0BETA</version>
+ <version>0.2.1BETA</version>
<relativePath>../../..</relativePath>
</parent>
<description>RPM Builder for Apache Metron</description>
@@ -33,7 +33,7 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<rpm.docker.tag>rpm-docker</rpm.docker.tag>
- <metron_dir>${parent.relativePath}/..</metron_dir>
+ <metron_dir>${project.parent.relativePath}/..</metron_dir>
</properties>
<build>
<plugins>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/playbooks/docker_probe_install.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/playbooks/docker_probe_install.yml b/metron-deployment/playbooks/docker_probe_install.yml
index b23e72a..2ec087f 100644
--- a/metron-deployment/playbooks/docker_probe_install.yml
+++ b/metron-deployment/playbooks/docker_probe_install.yml
@@ -30,7 +30,7 @@
- hosts: sensors
vars:
- metron_version: 0.2.0BETA
+ metron_version: 0.2.1BETA
metron_directory: /usr/metron/{{ metron_version }}
bro_version: "2.4.1"
fixbuf_version: "1.7.1"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/pom.xml b/metron-deployment/pom.xml
index d4b05f1..717b1ce 100644
--- a/metron-deployment/pom.xml
+++ b/metron-deployment/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>Metron</artifactId>
- <version>0.2.0BETA</version>
+ <version>0.2.1BETA</version>
</parent>
<description>Building and deploying Metron</description>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-deployment/roles/metron_pcapservice/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_pcapservice/defaults/main.yml b/metron-deployment/roles/metron_pcapservice/defaults/main.yml
index 23770bd..5ab28b1 100644
--- a/metron-deployment/roles/metron_pcapservice/defaults/main.yml
+++ b/metron-deployment/roles/metron_pcapservice/defaults/main.yml
@@ -15,7 +15,7 @@
# limitations under the License.
#
---
-metron_version: 0.2.0BETA
+metron_version: 0.2.1BETA
metron_directory: /usr/metron/{{ metron_version }}
pcapservice_jar_name: metron-api-{{ metron_version }}.jar
pcapservice_jar_src: "{{ playbook_dir }}/../../metron-platform/metron-api/target/{{ pcapservice_jar_name }}"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-platform/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/README.md b/metron-platform/README.md
index cdb1c0b..360997c 100644
--- a/metron-platform/README.md
+++ b/metron-platform/README.md
@@ -17,7 +17,7 @@ limitations under the License.
# Current Build
-The latest build of metron-platform is 0.2.0BETA.
+The latest build of metron-platform is 0.2.1BETA.
We are still in the process of merging/porting additional features from our production code base into this open source release. This release will be followed by a number of additional beta releases until the port is complete. We will also work on getting additional documentation and user/developer guides to the community as soon as we can. At this time we offer no support for the beta software, but will try to respond to requests as promptly as we can.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-platform/elasticsearch-shaded/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/elasticsearch-shaded/pom.xml b/metron-platform/elasticsearch-shaded/pom.xml
index 7d6ce46..1e18375 100644
--- a/metron-platform/elasticsearch-shaded/pom.xml
+++ b/metron-platform/elasticsearch-shaded/pom.xml
@@ -17,7 +17,7 @@
<parent>
<artifactId>metron-platform</artifactId>
<groupId>org.apache.metron</groupId>
- <version>0.2.0BETA</version>
+ <version>0.2.1BETA</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>elasticsearch-shaded</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-platform/metron-api/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-api/pom.xml b/metron-platform/metron-api/pom.xml
index 5e5e8e1..d600bfd 100644
--- a/metron-platform/metron-api/pom.xml
+++ b/metron-platform/metron-api/pom.xml
@@ -17,7 +17,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.2.0BETA</version>
+ <version>0.2.1BETA</version>
</parent>
<artifactId>metron-api</artifactId>
<description>Metron API</description>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-platform/metron-common/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/pom.xml b/metron-platform/metron-common/pom.xml
index 7a31e84..c19a507 100644
--- a/metron-platform/metron-common/pom.xml
+++ b/metron-platform/metron-common/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.2.0BETA</version>
+ <version>0.2.1BETA</version>
</parent>
<artifactId>metron-common</artifactId>
<name>metron-common</name>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/823cd2a8/metron-platform/metron-data-management/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/pom.xml b/metron-platform/metron-data-management/pom.xml
index dd7a690..0263396 100644
--- a/metron-platform/metron-data-management/pom.xml
+++ b/metron-platform/metron-data-management/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.2.0BETA</version>
+ <version>0.2.1BETA</version>
</parent>
<artifactId>metron-data-management</artifactId>
<properties>