You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by rm...@apache.org on 2016/04/26 16:46:30 UTC
[42/51] [partial] incubator-metron git commit: METRON-113 Project
Reorganization (merrimanr) closes apache/incubator-metron#88
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/metron_streaming/files/config/sensors/pcap.json
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/files/config/sensors/pcap.json b/metron-deployment/roles/metron_streaming/files/config/sensors/pcap.json
new file mode 100644
index 0000000..7792165
--- /dev/null
+++ b/metron-deployment/roles/metron_streaming/files/config/sensors/pcap.json
@@ -0,0 +1,19 @@
+{
+ "index": "pcap",
+ "batchSize": 5,
+ "enrichmentFieldMap":
+ {
+ "geo": ["ip_src_addr", "ip_dst_addr"],
+ "host": ["ip_src_addr", "ip_dst_addr"]
+ },
+ "threatIntelFieldMap":
+ {
+ "hbaseThreatIntel": ["ip_dst_addr", "ip_src_addr"]
+ },
+ "fieldToThreatIntelTypeMap":
+ {
+ "ip_dst_addr" : [ "malicious_ip" ]
+ ,"ip_src_addr" : [ "malicious_ip" ]
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/metron_streaming/files/config/sensors/snort.json
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/files/config/sensors/snort.json b/metron-deployment/roles/metron_streaming/files/config/sensors/snort.json
new file mode 100644
index 0000000..c5b6dcc
--- /dev/null
+++ b/metron-deployment/roles/metron_streaming/files/config/sensors/snort.json
@@ -0,0 +1,18 @@
+{
+ "index": "snort",
+ "batchSize": 1,
+ "enrichmentFieldMap":
+ {
+ "geo": ["ip_dst_addr", "ip_src_addr"],
+ "host": ["host"]
+ },
+ "threatIntelFieldMap":
+ {
+ "hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"]
+ },
+ "fieldToThreatIntelTypeMap":
+ {
+ "ip_src_addr" : ["malicious_ip"],
+ "ip_dst_addr" : ["malicious_ip"]
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/metron_streaming/files/config/sensors/yaf.json
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/files/config/sensors/yaf.json b/metron-deployment/roles/metron_streaming/files/config/sensors/yaf.json
new file mode 100644
index 0000000..2b46c9a
--- /dev/null
+++ b/metron-deployment/roles/metron_streaming/files/config/sensors/yaf.json
@@ -0,0 +1,19 @@
+{
+ "index": "yaf",
+ "batchSize": 5,
+ "enrichmentFieldMap":
+ {
+ "geo": ["ip_dst_addr", "ip_src_addr"],
+ "host": ["host"]
+ },
+ "threatIntelFieldMap":
+ {
+ "hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"]
+ },
+ "fieldToThreatIntelTypeMap":
+ {
+ "ip_src_addr" : ["malicious_ip"],
+ "ip_dst_addr" : ["malicious_ip"]
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/metron_streaming/files/extractor.json
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/files/extractor.json b/metron-deployment/roles/metron_streaming/files/extractor.json
new file mode 100644
index 0000000..545202a
--- /dev/null
+++ b/metron-deployment/roles/metron_streaming/files/extractor.json
@@ -0,0 +1,12 @@
+{
+ "config": {
+ "columns": {
+ "ip": 0
+ },
+ "indicator_column": "ip",
+ "type" : "malicious_ip",
+ "separator": ","
+ },
+ "extractor": "CSV"
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/metron_streaming/files/yaf_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/files/yaf_index.template b/metron-deployment/roles/metron_streaming/files/yaf_index.template
new file mode 100644
index 0000000..c8c1702
--- /dev/null
+++ b/metron-deployment/roles/metron_streaming/files/yaf_index.template
@@ -0,0 +1,36 @@
+{
+ "template" : "yaf_index*",
+ "mappings" : {
+ "yaf_doc" : {
+ "properties": {
+ "message": {
+ "properties": {
+ "@timestamp":{"type":"date","format":"dateOptionalTime"},
+ "end-time":{"type":"string"},
+ "duration":{"type":"string"},
+ "rtt":{"type":"string"},
+ "proto":{"type":"string"},
+ "sip":{"type":"string"},
+ "sp":{"type":"string"},
+ "dip":{"type":"string"},
+ "dp":{"type":"string"},
+ "iflags":{"type":"string"},
+ "uflags":{"type":"string"},
+ "riflags":{"type":"string"},
+ "ruflags":{"type":"string"},
+ "isn":{"type":"string"},
+ "risn":{"type":"string"},
+ "tag":{"type":"string"},
+ "rtag":{"type":"string"},
+ "pkt":{"type":"string"},
+ "oct":{"type":"string"},
+ "rpkt":{"type":"string"},
+ "roct":{"type":"string"},
+ "app":{"type":"string"},
+ "end-reason":{"type":"string"}
+ }
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/metron_streaming/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/meta/main.yml b/metron-deployment/roles/metron_streaming/meta/main.yml
new file mode 100644
index 0000000..6820b2c
--- /dev/null
+++ b/metron-deployment/roles/metron_streaming/meta/main.yml
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+dependencies:
+ - ambari_gather_facts
+ - java_jdk
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/metron_streaming/tasks/es_purge.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/tasks/es_purge.yml b/metron-deployment/roles/metron_streaming/tasks/es_purge.yml
new file mode 100644
index 0000000..22616ca
--- /dev/null
+++ b/metron-deployment/roles/metron_streaming/tasks/es_purge.yml
@@ -0,0 +1,42 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Create Empty Log Files for ES Purge
+ file:
+ path: "{{ item }}"
+ state: touch
+ owner: hdfs
+ group: hdfs
+ mode: 0644
+ with_items:
+ - /var/log/bro-purge/cron-es-bro-purge.log
+ - /var/log/yaf-purge/cron-es-yaf-purge.log
+ - /var/log/snort-purge/cron-es-snort-purge.log
+
+
+- name: Purge Elasticsearch Indices every 30 days.
+ cron:
+ name: "{{ item.name }}"
+ job: "{{ item.job }}"
+ special_time: daily
+ user: hdfs
+ with_items:
+ - { name: "bro_es_purge", job: "{{ es_bro_purge_cronjob }}" }
+ - { name: "yaf_es_purge", job: "{{ es_yaf_purge_cronjob }}" }
+ - { name: "snort_es_purge", job: "{{ es_snort_purge_cronjob }}" }
+
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/metron_streaming/tasks/grok_upload.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/tasks/grok_upload.yml b/metron-deployment/roles/metron_streaming/tasks/grok_upload.yml
new file mode 100644
index 0000000..d857bf5
--- /dev/null
+++ b/metron-deployment/roles/metron_streaming/tasks/grok_upload.yml
@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Create HDFS directory for grok patterns
+ command: hdfs dfs -mkdir -p {{ metron_hdfs_output_dir }}/patterns
+ become: yes
+ become_user: hdfs
+
+- name: Assign hfds user as owner of {{ metron_hdfs_output_dir }}/patterns HDFS directory
+ command: hdfs dfs -chown -R hdfs:hadoop {{ metron_hdfs_output_dir }}/patterns
+ become: yes
+ become_user: hdfs
+
+- name: Assign permissions of HDFS {{ metron_hdfs_output_dir }}/patterns directory
+ command: hdfs dfs -chmod -R 775 {{ metron_hdfs_output_dir }}/patterns
+ become: yes
+ become_user: hdfs
+
+- name: Upload Grok Patterns to hdfs://{{ metron_hdfs_output_dir }}
+ command: hdfs dfs -put -f {{ metron_directory }}/patterns {{ metron_hdfs_output_dir }}
+ become: yes
+ become_user: hdfs
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/metron_streaming/tasks/hdfs_filesystem.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/tasks/hdfs_filesystem.yml b/metron-deployment/roles/metron_streaming/tasks/hdfs_filesystem.yml
new file mode 100644
index 0000000..252e671
--- /dev/null
+++ b/metron-deployment/roles/metron_streaming/tasks/hdfs_filesystem.yml
@@ -0,0 +1,41 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Create root user HDFS directory
+ command: hdfs dfs -mkdir -p /user/root
+ become: yes
+ become_user: hdfs
+
+- name: Assign root as owner of /user/root HDFS directory
+ command: hdfs dfs -chown root:root /user/root
+ become: yes
+ become_user: hdfs
+
+- name: Create Metron HDFS output directory
+ command: hdfs dfs -mkdir -p {{ metron_hdfs_output_dir }}
+ become: yes
+ become_user: hdfs
+
+- name: Assign hdfs as owner of HDFS output directory
+ command: hdfs dfs -chown hdfs:hadoop {{ metron_hdfs_output_dir }}
+ become: yes
+ become_user: hdfs
+
+- name: Assign permissions of HDFS output directory
+ command: hdfs dfs -chmod 775 {{ metron_hdfs_output_dir }}
+ become: yes
+ become_user: hdfs
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/metron_streaming/tasks/hdfs_purge.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/tasks/hdfs_purge.yml b/metron-deployment/roles/metron_streaming/tasks/hdfs_purge.yml
new file mode 100644
index 0000000..33442e4
--- /dev/null
+++ b/metron-deployment/roles/metron_streaming/tasks/hdfs_purge.yml
@@ -0,0 +1,52 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Create Log Directories for HDFS Purge
+ file:
+ path: "{{ item }}"
+ state: directory
+ mode: 0755
+ owner: hdfs
+ group: hdfs
+ with_items:
+ - /var/log/bro-purge
+ - /var/log/yaf-purge
+ - /var/log/snort-purge
+
+- name: Create Empty Log Files for HDFS Purge
+ file:
+ path: "{{ item }}"
+ state: touch
+ owner: hdfs
+ group: hdfs
+ mode: 0644
+ with_items:
+ - /var/log/bro-purge/cron-hdfs-bro-purge.log
+ - /var/log/yaf-purge/cron-hdfs-yaf-purge.log
+ - /var/log/snort-purge/cron-hdfs-snort-purge.log
+
+- name: Purge HDFS Sensor Data every 30 days.
+ cron:
+ name: "{{ item.name }}"
+ job: "{{ item.job }}"
+ special_time: daily
+ user: hdfs
+ with_items:
+ - { name: "bro_hdfs_purge", job: "{{ hdfs_bro_purge_cronjob }}" }
+ - { name: "yaf_hdfs_purge", job: "{{ hdfs_yaf_purge_cronjob }}" }
+ - { name: "snort_hdfs_purge", job: "{{ hdfs_snort_purge_cronjob }}" }
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/metron_streaming/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/tasks/main.yml b/metron-deployment/roles/metron_streaming/tasks/main.yml
new file mode 100644
index 0000000..2f22dba
--- /dev/null
+++ b/metron-deployment/roles/metron_streaming/tasks/main.yml
@@ -0,0 +1,155 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Create Metron streaming directories
+ file: path="{{ metron_directory }}/{{ item.name }}" state=directory mode=0755
+ with_items:
+ - { name: 'lib'}
+ - { name: 'scripts'}
+ - { name: 'config'}
+
+
+- name: Copy Metron Solr bundle
+ copy:
+ src: "{{ metron_solr_bundle_path }}"
+ dest: "{{ metron_directory }}"
+
+- name: Copy Metron Elasticsearch bundle
+ copy:
+ src: "{{ metron_elasticsearch_bundle_path }}"
+ dest: "{{ metron_directory }}"
+
+- name: Copy Metron Enrichment bundle
+ copy:
+ src: "{{ metron_enrichment_bundle_path }}"
+ dest: "{{ metron_directory }}"
+
+- name: Copy Metron Parsers bundle
+ copy:
+ src: "{{ metron_parsers_bundle_path }}"
+ dest: "{{ metron_directory }}"
+
+- name: Copy Metron DataLoads bundle
+ copy:
+ src: "{{ metron_data_management_bundle_path }}"
+ dest: "{{ metron_directory }}"
+
+- name: Copy Metron Common bundle
+ copy:
+ src: "{{ metron_common_bundle_path }}"
+ dest: "{{ metron_directory }}"
+
+- name: Unbundle Metron bundles
+ shell: cd {{ metron_directory }} && tar xzvf metron-solr*.tar.gz && tar xzvf metron-elasticsearch*.tar.gz && tar xzvf metron-enrichment*.tar.gz && tar xzvf metron-parsers*.tar.gz && tar xzvf metron-data-management*.tar.gz && tar xzvf metron-common*.tar.gz && rm *.tar.gz
+
+- name: Add *-site.xml files to topology jars
+ shell: cd {{ item.config_path }} && jar -uf {{ metron_directory }}/lib/{{ item.jar_name }} {{ item.file_name }}
+ with_items:
+ - { config_path: "{{ hbase_config_path }}", jar_name: "{{ metron_solr_jar_name }}", file_name: "hbase-site.xml" }
+ - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_solr_jar_name }}", file_name: "core-site.xml" }
+ - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_solr_jar_name }}", file_name: "hdfs-site.xml" }
+ - { config_path: "{{ hbase_config_path }}", jar_name: "{{ metron_elasticsearch_jar_name }}", file_name: "hbase-site.xml" }
+ - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_elasticsearch_jar_name }}", file_name: "core-site.xml" }
+ - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_elasticsearch_jar_name }}", file_name: "hdfs-site.xml" }
+
+- name: Get Default mysql passowrd
+ include_vars: "../roles/mysql_server/defaults/main.yml"
+ when: mysql_root_password is undefined
+
+- include: hdfs_filesystem.yml
+ run_once: true
+
+- include: grok_upload.yml
+ run_once: true
+
+- name: Configure Metron Parser Topologies
+ lineinfile:
+ dest: "{{ metron_parsers_properties_config_path }}"
+ regexp: "{{ item.regexp }}"
+ line: "{{ item.line }}"
+ with_items:
+ - { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" }
+ - { regexp: "kafka.broker=", line: "kafka.broker={{ kafka_broker_url }}" }
+
+- name: Configure Metron Solr topology
+ lineinfile: >
+ dest={{ metron_solr_properties_config_path }}
+ regexp="{{ item.regexp }}"
+ line="{{ item.line }}"
+ with_items:
+ - { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" }
+ - { regexp: "kafka.broker=", line: "kafka.broker={{ kafka_broker_url }}" }
+ - { regexp: "es.ip=", line: "es.ip={{ groups.search[0] }}" }
+ - { regexp: "es.port=", line: "es.port={{ elasticsearch_transport_port }}" }
+ - { regexp: "es.clustername=", line: "es.clustername={{ elasticsearch_cluster_name }}" }
+ - { regexp: "bolt.hdfs.file.system.url=", line: "bolt.hdfs.file.system.url={{ hdfs_url }}" }
+ - { regexp: "spout.kafka.topic.pcap=", line: "spout.kafka.topic.pcap={{ pycapa_topic }}" }
+ - { regexp: "spout.kafka.topic.bro=", line: "spout.kafka.topic.bro={{ bro_topic }}" }
+ - { regexp: "bolt.hbase.table.name=", line: "bolt.hbase.table.name={{ pcap_hbase_table }}" }
+ - { regexp: "threat.intel.tracker.table=", line: "threat.intel.tracker.table={{ tracker_hbase_table }}" }
+ - { regexp: "threat.intel.tracker.cf=", line: "threat.intel.tracker.cf=t" }
+ - { regexp: "threat.intel.simple.hbase.table=", line: "threat.intel.simple.hbase.table={{ threatintel_hbase_table }}" }
+ - { regexp: "threat.intel.simple.hbase.cf=", line: "threat.intel.simple.hbase.cf=t" }
+ - { regexp: "enrichment.simple.hbase.table=", line: "enrichment.simple.hbase.table={{ enrichment_hbase_table }}" }
+ - { regexp: "enrichment.simple.hbase.cf=", line: "enrichment.simple.hbase.cf=t" }
+ - { regexp: "mysql.ip=", line: "mysql.ip={{ groups.mysql[0] }}" }
+ - { regexp: "mysql.password=", line: "mysql.password={{ mysql_root_password }}" }
+ - { regexp: "index.hdfs.output=", line: "index.hdfs.output={{ metron_hdfs_output_dir }}/enrichment/indexed" }
+ - { regexp: "bolt.hdfs.rotation.policy=", line: "bolt.hdfs.rotation.policy={{ metron_hdfs_rotation_policy }}" }
+ - { regexp: "bolt.hdfs.rotation.policy.count=", line: "bolt.hdfs.rotation.policy.count={{ metron_hdfs_rotation_policy_count}}" }
+ - { regexp: "bolt.hdfs.rotation.policy.units=", line: "bolt.hdfs.rotation.policy.units={{ metron_hdfs_rotation_policy_units }}" }
+
+- name: Configure Metron Elasticsearch topology
+ lineinfile: >
+ dest={{ metron_elasticsearch_properties_config_path }}
+ regexp="{{ item.regexp }}"
+ line="{{ item.line }}"
+ with_items:
+ - { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" }
+ - { regexp: "kafka.broker=", line: "kafka.broker={{ kafka_broker_url }}" }
+ - { regexp: "es.ip=", line: "es.ip={{ groups.search[0] }}" }
+ - { regexp: "es.port=", line: "es.port={{ elasticsearch_transport_port }}" }
+ - { regexp: "es.clustername=", line: "es.clustername={{ elasticsearch_cluster_name }}" }
+ - { regexp: "bolt.hdfs.file.system.url=", line: "bolt.hdfs.file.system.url={{ hdfs_url }}" }
+ - { regexp: "spout.kafka.topic.pcap=", line: "spout.kafka.topic.pcap={{ pycapa_topic }}" }
+ - { regexp: "spout.kafka.topic.bro=", line: "spout.kafka.topic.bro={{ bro_topic }}" }
+ - { regexp: "bolt.hbase.table.name=", line: "bolt.hbase.table.name={{ pcap_hbase_table }}" }
+ - { regexp: "threat.intel.tracker.table=", line: "threat.intel.tracker.table={{ tracker_hbase_table }}" }
+ - { regexp: "threat.intel.tracker.cf=", line: "threat.intel.tracker.cf=t" }
+ - { regexp: "threat.intel.simple.hbase.table=", line: "threat.intel.simple.hbase.table={{ threatintel_hbase_table }}" }
+ - { regexp: "threat.intel.simple.hbase.cf=", line: "threat.intel.simple.hbase.cf=t" }
+ - { regexp: "enrichment.simple.hbase.table=", line: "enrichment.simple.hbase.table={{ enrichment_hbase_table }}" }
+ - { regexp: "enrichment.simple.hbase.cf=", line: "enrichment.simple.hbase.cf=t" }
+ - { regexp: "mysql.ip=", line: "mysql.ip={{ groups.mysql[0] }}" }
+ - { regexp: "mysql.password=", line: "mysql.password={{ mysql_root_password }}" }
+ - { regexp: "index.hdfs.output=", line: "index.hdfs.output={{ metron_hdfs_output_dir }}/enrichment/indexed" }
+ - { regexp: "bolt.hdfs.rotation.policy=", line: "bolt.hdfs.rotation.policy={{ metron_hdfs_rotation_policy }}" }
+ - { regexp: "bolt.hdfs.rotation.policy.count=", line: "bolt.hdfs.rotation.policy.count={{ metron_hdfs_rotation_policy_count}}" }
+ - { regexp: "bolt.hdfs.rotation.policy.units=", line: "bolt.hdfs.rotation.policy.units={{ metron_hdfs_rotation_policy_units }}" }
+
+- include: source_config.yml
+ run_once: true
+
+- include: threat_intel.yml
+ run_once: true
+ when: threat_intel_bulk_load == True
+
+- include: metron_topology.yml
+
+- include: hdfs_purge.yml
+
+- include: es_purge.yml
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/metron_streaming/tasks/metron_topology.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/tasks/metron_topology.yml b/metron-deployment/roles/metron_streaming/tasks/metron_topology.yml
new file mode 100644
index 0000000..3d64f2b
--- /dev/null
+++ b/metron-deployment/roles/metron_streaming/tasks/metron_topology.yml
@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Submit Metron Parser topologies
+ command: storm jar {{ metron_directory }}/lib/{{ metron_parsers_jar_name }} org.apache.storm.flux.Flux --filter {{ metron_parsers_properties_config_path }} --remote {{ item }}
+ with_items:
+ - "{{ storm_parser_topologies }}"
+
+- name: Submit Solr Metron Enrichment topology
+ command: storm jar {{ metron_directory }}/lib/{{ metron_solr_jar_name }} org.apache.storm.flux.Flux --filter {{ metron_solr_properties_config_path }} --remote {{ item }}
+ with_items:
+ - "{{ storm_enrichment_topology }}"
+ when: install_solr | default(False) == True
+
+- name: Submit Elasticsearch Metron Enrichment topology
+ command: storm jar {{ metron_directory }}/lib/{{ metron_elasticsearch_jar_name }} org.apache.storm.flux.Flux --filter {{ metron_elasticsearch_properties_config_path }} --remote {{ item }}
+ with_items:
+ - "{{ storm_enrichment_topology }}"
+ when: install_elasticsearch | default(False) == True
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/metron_streaming/tasks/source_config.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/tasks/source_config.yml b/metron-deployment/roles/metron_streaming/tasks/source_config.yml
new file mode 100644
index 0000000..897d0f1
--- /dev/null
+++ b/metron-deployment/roles/metron_streaming/tasks/source_config.yml
@@ -0,0 +1,51 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Create Source Config Directory
+ file:
+ path: "{{ zookeeper_config_path }}"
+ state: directory
+
+- name: Copy Elasticsearch Global Config File
+ template:
+ src: "templates/config/elasticsearch.global.json"
+ dest: "{{ zookeeper_global_config_path }}"
+ mode: 0644
+ when: install_elasticsearch | default(False) == True
+
+- name: Copy Solr Global Config File
+ template:
+ src: "../roles/metron_streaming/templates/config/solr.global.json"
+ dest: "{{ zookeeper_global_config_path }}"
+ mode: 0644
+ when: install_solr | default(False) == True
+
+- name: Copy Sensor Config Files
+ copy:
+ src: "{{ item }}"
+ dest: "{{ zookeeper_config_path }}"
+ mode: 0644
+ with_items:
+ - ../roles/metron_streaming/files/config/
+
+- name: Load Config
+ shell: "{{ metron_directory }}/scripts/zk_load_configs.sh -p {{ zookeeper_config_path }} -z {{ zookeeper_url }} && touch {{ zookeeper_config_path }}/configured"
+
+
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/metron_streaming/tasks/threat_intel.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/tasks/threat_intel.yml b/metron-deployment/roles/metron_streaming/tasks/threat_intel.yml
new file mode 100644
index 0000000..f1b7534
--- /dev/null
+++ b/metron-deployment/roles/metron_streaming/tasks/threat_intel.yml
@@ -0,0 +1,46 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+
+- name: Create Bulk load working Directory
+ file:
+ path: "{{ threat_intel_work_dir }}"
+ state: directory
+
+- name: Copy extractor.json to {{ inventory_hostname }}
+ copy:
+ src: ../roles/metron_streaming/files/extractor.json
+ dest: "{{ threat_intel_work_dir }}"
+ mode: 0644
+
+- name: Copy Bulk Load CSV File
+ template:
+ src: "{{ threat_intel_csv_filepath }}"
+ dest: "{{ threat_intel_work_dir }}/{{ threat_intel_csv_filename }}"
+ mode: 0644
+
+- name: Copy Bulk Load CSV File to HDFS
+ command: "hdfs dfs -put -f {{ threat_intel_work_dir }}/{{ threat_intel_csv_filename }} ."
+
+- name: Run Threat Intel Bulk Load
+ shell: "{{ threat_intel_bin }} -f t --table {{threatintel_hbase_table}} -e {{ threat_intel_work_dir }}/extractor.json -i /user/root && touch {{ threat_intel_work_dir }}/loaded"
+ args:
+ creates: "{{ threat_intel_work_dir }}/loaded"
+
+- name: Clean up HDFS File
+ command: "hdfs dfs -rm {{ threat_intel_csv_filename }}"
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/metron_streaming/templates/config/elasticsearch.global.json
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/templates/config/elasticsearch.global.json b/metron-deployment/roles/metron_streaming/templates/config/elasticsearch.global.json
new file mode 100644
index 0000000..8177102
--- /dev/null
+++ b/metron-deployment/roles/metron_streaming/templates/config/elasticsearch.global.json
@@ -0,0 +1,6 @@
+{
+ "es.clustername": "{{ elasticsearch_cluster_name }}",
+ "es.ip": "{{ groups.search[0] }}",
+ "es.port": "{{ elasticsearch_transport_port }}",
+ "es.date.format": "yyyy.MM.dd.HH"
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/metron_streaming/templates/config/solr.global.json
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/templates/config/solr.global.json b/metron-deployment/roles/metron_streaming/templates/config/solr.global.json
new file mode 100644
index 0000000..5cb7a4d
--- /dev/null
+++ b/metron-deployment/roles/metron_streaming/templates/config/solr.global.json
@@ -0,0 +1,6 @@
+{
+ "solr.zookeeper": "{{ zookeeper_url }}",
+ "solr.collection": "{{ solr_collection_name }}",
+ "solr.numShards": {{ solr_number_shards }},
+ "solr.replicationFactor": {{ solr_replication_factor }}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/metron_streaming/templates/threat_ip.csv
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/templates/threat_ip.csv b/metron-deployment/roles/metron_streaming/templates/threat_ip.csv
new file mode 100644
index 0000000..3ac38f3
--- /dev/null
+++ b/metron-deployment/roles/metron_streaming/templates/threat_ip.csv
@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#Add single column of ip address to alert
+#Public lists are available on the internet
+# example:
+23.113.113.105
+24.107.205.249
+24.108.62.255
+24.224.153.71
+27.4.1.212
+27.131.149.102
+31.24.30.31
+31.131.251.33
+31.186.99.250
+31.192.209.119
+31.192.209.150
+31.200.244.17
+37.34.52.185
+37.58.112.101
+37.99.146.27
+37.128.132.96
+37.140.195.177
+37.140.199.100
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/metron_ui/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_ui/defaults/main.yml b/metron-deployment/roles/metron_ui/defaults/main.yml
new file mode 100644
index 0000000..23aed40
--- /dev/null
+++ b/metron-deployment/roles/metron_ui/defaults/main.yml
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+metron_version: 0.1BETA
+metron_directory: /usr/metron/{{ metron_version }}
+metron_ui_directory: "{{ metron_directory }}/metron-ui"
+metron_temp_archive: /tmp/metron-ui.tar.gz
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/metron_ui/tasks/copy-source.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_ui/tasks/copy-source.yml b/metron-deployment/roles/metron_ui/tasks/copy-source.yml
new file mode 100644
index 0000000..703b7f9
--- /dev/null
+++ b/metron-deployment/roles/metron_ui/tasks/copy-source.yml
@@ -0,0 +1,46 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Verify {{ metron_ui_directory }} exists
+ file:
+ path: "{{ metron_ui_directory }}"
+ state: directory
+ mode: 0755
+ owner: root
+ group: root
+
+- name: Archive metron-ui on localhost
+ shell: tar --exclude='./node_modules' -czf {{ metron_temp_archive }} .
+ args:
+ chdir: "{{ playbook_dir }}/../../metron-ui"
+ creates: "{{ metron_temp_archive }}"
+ warn: false #Warns to use unarchive - unarchive does not archive
+ become: false
+ delegate_to: localhost
+ run_once: true
+
+- name: Extract metron-ui tarball
+ unarchive:
+ src: "{{ metron_temp_archive }}"
+ dest: "{{ metron_ui_directory }}"
+ creates: "{{ metron_ui_directory}}/config"
+
+- name: Delete {{ metron_temp_archive }}
+ local_action: file path="{{ metron_temp_archive }}" state=absent
+ become: false
+ run_once: true
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/metron_ui/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_ui/tasks/main.yml b/metron-deployment/roles/metron_ui/tasks/main.yml
new file mode 100644
index 0000000..fd3422b
--- /dev/null
+++ b/metron-deployment/roles/metron_ui/tasks/main.yml
@@ -0,0 +1,56 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Install Metron UI dependencies
+ yum:
+ pkg: "{{ item }}"
+ state: installed
+ with_items:
+ - libpcap-devel
+ - wireshark
+ - nodejs
+ - npm
+ register: result
+ until: result.rc == 0
+ retries: 5
+ delay: 10
+
+- include: copy-source.yml
+
+- name: Configure Metron UI
+ lineinfile:
+ dest="{{ metron_ui_directory }}/config.json"
+ regexp="{{ item.regexp }}"
+ line="{{ item.line }}"
+ state=present
+ with_items:
+ - { regexp: '"elasticsearch":', line: '"elasticsearch": { "url": "http://{{ groups.search[0] }}:{{ elasticsearch_web_port }}" },' }
+ - { regexp: '"pcap":', line: ' "pcap": { "url": "http://{{ groups.web[0] }}:{{ pcapservice_port }}/pcapGetter","mock": false }' }
+
+- name: Install Node dependencies
+ npm:
+ name: pm2
+ path: "{{ metron_ui_directory }}"
+ global: true
+
+- name: Install Metron UI
+ npm:
+ path: "{{ metron_ui_directory }}"
+ production: no
+
+- name: Start Metron UI
+ shell: "pm2 start {{ metron_ui_directory }}/lib/metron-ui.js --name metron"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/mysql_client/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/mysql_client/tasks/main.yml b/metron-deployment/roles/mysql_client/tasks/main.yml
new file mode 100644
index 0000000..8c54c23
--- /dev/null
+++ b/metron-deployment/roles/mysql_client/tasks/main.yml
@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+
+- name: Get Default mysql passowrd
+ include_vars: "../roles/mysql_server/defaults/main.yml"
+ when: mysql_root_password is undefined
+
+- name: Allow remote login to mysql
+ template:
+ src: "../roles/mysql_client/templates/db_config.sql"
+ dest: "/tmp/{{ansible_fqdn}}.sql"
+ delegate_to: "{{ groups.mysql[0] }}"
+
+- name: Import DB_Config
+ mysql_db:
+ name: "all"
+ state: "import"
+ target: "/tmp/{{ansible_fqdn}}.sql"
+ ignore_errors: True
+ delegate_to: "{{ groups.mysql[0] }}"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/mysql_client/templates/db_config.sql
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/mysql_client/templates/db_config.sql b/metron-deployment/roles/mysql_client/templates/db_config.sql
new file mode 100644
index 0000000..c407a13
--- /dev/null
+++ b/metron-deployment/roles/mysql_client/templates/db_config.sql
@@ -0,0 +1,21 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+CREATE USER 'root'@'{{ ansible_fqdn }}' IDENTIFIED BY '{{ mysql_root_password }}';
+SET PASSWORD FOR 'root'@'{{ ansible_fqdn }}' = PASSWORD('{{ mysql_root_password }}');
+GRANT ALL PRIVILEGES ON *.* to 'root'@'{{ ansible_fqdn }}' WITH GRANT OPTION;
+FLUSH PRIVILEGES;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/mysql_server/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/mysql_server/defaults/main.yml b/metron-deployment/roles/mysql_server/defaults/main.yml
new file mode 100644
index 0000000..0acbd17
--- /dev/null
+++ b/metron-deployment/roles/mysql_server/defaults/main.yml
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+mysql_rpm_version: mysql57-community-release-el6-7.noarch
+mysql_yum_repo_url: https://dev.mysql.com/get/{{ mysql_rpm_version }}.rpm
+mysql_root_password: P@ssw0rd
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/mysql_server/files/geoip_ddl.sql
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/mysql_server/files/geoip_ddl.sql b/metron-deployment/roles/mysql_server/files/geoip_ddl.sql
new file mode 100644
index 0000000..02616c6
--- /dev/null
+++ b/metron-deployment/roles/mysql_server/files/geoip_ddl.sql
@@ -0,0 +1,49 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+CREATE DATABASE IF NOT EXISTS GEO;
+
+USE GEO;
+
+DROP TABLE IF EXISTS `blocks`;
+CREATE TABLE `blocks` ( `startIPNum` int(10) unsigned NOT NULL,`endIPNum` int(10) unsigned NOT NULL,`locID`
+int(10) unsigned NOT NULL, PRIMARY KEY (`startIPNum`,`endIPNum`) )
+ENGINE=MyISAM DEFAULT CHARSET=latin1 PACK_KEYS=1 DELAY_KEY_WRITE=1;
+
+DROP TABLE IF EXISTS `location`;
+CREATE TABLE `location` (`locID` int(10) unsigned NOT NULL,`country` char(2) default NULL,`region` char(2)
+ default NULL,`city` varchar(45) default NULL,`postalCode` char(7) default NULL,`latitude` double default
+NULL,`longitude` double default NULL,`dmaCode` char(3) default NULL,`areaCode` char(3) default NULL,PRIMARY KEY
+ (`locID`),KEY `Index_Country` (`country`) ) ENGINE=MyISAM DEFAULT CHARSET=latin1 ROW_FORMAT=FIXED;
+
+load data infile '/var/lib/mysql-files/GeoLiteCity-Blocks.csv' into table `blocks` fields terminated by ',' optionally enclosed by '"' lines terminated by '\n' ignore 2 lines;
+load data infile '/var/lib/mysql-files/GeoLiteCity-Location.csv' into table `location` fields terminated by ',' optionally enclosed by '"' lines terminated by '\n' ignore 2 lines;
+
+
+DELIMITER $$
+DROP FUNCTION IF EXISTS `IPTOLOCID` $$
+CREATE FUNCTION `IPTOLOCID`( ip VARCHAR(15)) RETURNS int(10) unsigned
+ BEGIN
+ DECLARE ipn INTEGER UNSIGNED;
+ DECLARE locID_var INTEGER;
+ IF ip LIKE '192.168.%' OR ip LIKE '10.%' THEN RETURN 0;
+ END IF;
+ SET ipn = INET_ATON(ip);
+ SELECT locID INTO locID_var FROM `blocks` INNER JOIN (SELECT MAX(startIPNum) AS start FROM `blocks` WHERE startIPNum <= ipn) AS s ON (startIPNum = s.start) WHERE endIPNum >= ipn;
+ RETURN locID_var;
+ END
+$$
+DELIMITER ;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/mysql_server/handlers/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/mysql_server/handlers/main.yml b/metron-deployment/roles/mysql_server/handlers/main.yml
new file mode 100644
index 0000000..112c5ca
--- /dev/null
+++ b/metron-deployment/roles/mysql_server/handlers/main.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: restart elasticsearch
+ service: name=elasticsearch state=restarted
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/mysql_server/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/mysql_server/tasks/main.yml b/metron-deployment/roles/mysql_server/tasks/main.yml
new file mode 100644
index 0000000..a484ed0
--- /dev/null
+++ b/metron-deployment/roles/mysql_server/tasks/main.yml
@@ -0,0 +1,93 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Create temporary directories
+ file:
+ path: "/tmp/geoip"
+ state: directory
+ mode: 0755
+
+- name: Install Mysql Community Release Repo Def
+ get_url:
+ dest: /tmp/{{ mysql_rpm_version }}.rpm
+ url: "{{ mysql_yum_repo_url }}"
+
+- name: Install Mysql Community Release Repo
+ yum:
+ pkg: /tmp/{{ mysql_rpm_version }}.rpm
+ state: installed
+ register: result
+ until: result.rc == 0
+ retries: 5
+ delay: 10
+
+- name: Install MySQL
+ yum:
+ name: "{{ item }}"
+ state: latest
+ with_items:
+ - "mysql-community-server"
+ - "MySQL-python"
+ register: result
+ until: result.rc == 0
+ retries: 5
+ delay: 10
+
+- name: Start MySQL
+ service:
+ name: mysqld
+ state: started
+ enabled: yes
+
+- name: Retrieve temporary root password
+ shell: "grep 'temporary password' /var/log/mysqld.log | sed 's/.*root@localhost: //'"
+ args:
+ creates: ~/.my.cnf
+ register: temp_root_password
+
+- name: Update mysql root password
+ command: "mysqladmin --user=root --password='{{ temp_root_password.stdout }}' password '{{ mysql_root_password }}'"
+ ignore_errors: yes
+ args:
+ creates: ~/.my.cnf
+
+- name: Create .my.cnf
+ template:
+ src: "../roles/mysql_server/templates/.my.cnf"
+ dest: ~/.my.cnf
+
+
+- name: Download GeoIP databases
+ unarchive:
+ src: http://geolite.maxmind.com/download/geoip/database/GeoLiteCity_CSV/GeoLiteCity-latest.tar.xz
+ dest: /tmp/geoip
+ copy: no
+ creates: /tmp/geopip/*/GeoLiteCity-Blocks.csv
+
+- name: Copy to MySQL import directory
+ shell: "cp /tmp/geoip/*/*.csv /var/lib/mysql-files/"
+
+- name: Copy DDL
+ copy:
+ src: geoip_ddl.sql
+ dest: /tmp/geoip_ddl.sql
+
+- name: Import GeoIP DDL
+ mysql_db:
+ name: all
+ state: import
+ target: /tmp/geoip_ddl.sql
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/mysql_server/templates/.my.cnf
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/mysql_server/templates/.my.cnf b/metron-deployment/roles/mysql_server/templates/.my.cnf
new file mode 100644
index 0000000..d5c0825
--- /dev/null
+++ b/metron-deployment/roles/mysql_server/templates/.my.cnf
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+[client]
+user=root
+password={{ mysql_root_password }}
+host=localhost
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/ntp/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ntp/tasks/main.yml b/metron-deployment/roles/ntp/tasks/main.yml
new file mode 100644
index 0000000..7b1b9a8
--- /dev/null
+++ b/metron-deployment/roles/ntp/tasks/main.yml
@@ -0,0 +1,31 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Install ntp
+ yum:
+ name: ntp
+ state: present
+ register: result
+ until: result.rc == 0
+ retries: 5
+ delay: 10
+
+- name: Ensure ntp is running and enabled
+ service:
+ name: ntpd
+ state: started
+ enabled: yes
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/packet-capture/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/packet-capture/defaults/main.yml b/metron-deployment/roles/packet-capture/defaults/main.yml
new file mode 100644
index 0000000..3e6358c
--- /dev/null
+++ b/metron-deployment/roles/packet-capture/defaults/main.yml
@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+# dpdk
+dpdk_home: "/usr/local/dpdk"
+dpdk_version: "2.2.0"
+dpdk_sdk: "/root/dpdk-{{ dpdk_version }}"
+dpdk_target: "x86_64-native-linuxapp-gcc"
+num_huge_pages: 512
+extra_cflags: -g
+
+# pcapture
+pcapture_work_dir: /root/packet-capture
+pcapture_prefix: /usr/local/bin
+pcapture_ld_library_path: /usr/local/lib
+pcapture_portmask: 0x01
+pcapture_kafka_config: /etc/pcapture.conf
+pcapture_bin: pcapture
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/packet-capture/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/packet-capture/meta/main.yml b/metron-deployment/roles/packet-capture/meta/main.yml
new file mode 100644
index 0000000..d253e88
--- /dev/null
+++ b/metron-deployment/roles/packet-capture/meta/main.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+dependencies:
+ - librdkafka
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/packet-capture/tasks/debug.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/packet-capture/tasks/debug.yml b/metron-deployment/roles/packet-capture/tasks/debug.yml
new file mode 100644
index 0000000..06f1526
--- /dev/null
+++ b/metron-deployment/roles/packet-capture/tasks/debug.yml
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+ - name: Install debug utilities
+ yum: name=yum-utils
+ tags:
+ - debug
+
+ - name: Install debug symbols
+ shell: debuginfo-install -y glibc glib2 zlib
+ tags:
+ - debug
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/packet-capture/tasks/dependencies.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/packet-capture/tasks/dependencies.yml b/metron-deployment/roles/packet-capture/tasks/dependencies.yml
new file mode 100644
index 0000000..4d6edc4
--- /dev/null
+++ b/metron-deployment/roles/packet-capture/tasks/dependencies.yml
@@ -0,0 +1,38 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+ - name: Install dependencies
+ yum: name={{ item }}
+ with_items:
+ - "@Development tools"
+ - pciutils
+ - net-tools
+ - glib2
+ - glib2-devel
+ - git
+
+ #
+ # install prerequisite packages and the latest kernel headers. need to
+ # ensure that the kernel headers match the current running kernel version.
+ # if this is not the case, the DPDK build process will fail
+ #
+ - name: Install latest kernel headers and source
+ yum: name={{ item }} state=latest
+ with_items:
+ - kernel
+ - kernel-devel
+ - kernel-headers
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/packet-capture/tasks/dpdk.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/packet-capture/tasks/dpdk.yml b/metron-deployment/roles/packet-capture/tasks/dpdk.yml
new file mode 100644
index 0000000..3780be7
--- /dev/null
+++ b/metron-deployment/roles/packet-capture/tasks/dpdk.yml
@@ -0,0 +1,59 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+ - name: "Download DPDK version {{ dpdk_version }}"
+ unarchive:
+ src: "http://dpdk.org/browse/dpdk/snapshot/dpdk-{{ dpdk_version }}.tar.gz"
+ dest: "/root"
+ creates: "{{ dpdk_sdk }}"
+ copy: no
+
+ - name: "Configure DPDK for the target environment: {{ dpdk_target }}"
+ shell: "make config T={{ dpdk_target }} DESTDIR={{ dpdk_home }}"
+ args:
+ chdir: "{{ dpdk_sdk }}"
+ creates: "{{ dpdk_home }}"
+
+ - name: "Turn on debug flags"
+ lineinfile:
+ dest: "{{ dpdk_sdk }}/config/common_linuxapp"
+ regexp: 'DEBUG=n'
+ line: 'DEBUG=y'
+ tags:
+ - debug
+
+ - name: "Build DPDK for the target environment: {{ dpdk_target }}"
+ shell: "make install T={{ dpdk_target }} DESTDIR={{ dpdk_home }} EXTRA_CFLAGS={{ extra_cflags }}"
+ args:
+ chdir: "{{ dpdk_sdk }}"
+ creates: "{{ dpdk_home }}"
+
+ - name: Load kernel modules to enable userspace IO
+ shell: "{{ item }}"
+ with_items:
+ - modprobe uio_pci_generic
+ - modprobe vfio-pci
+
+ - name: Bind the device to the loaded kernel module(s)
+ shell: "{{ dpdk_home }}/sbin/dpdk_nic_bind --force --bind=uio_pci_generic {{ item }}"
+ with_items: "{{ dpdk_device }}"
+
+ - name: Set useful environment variables
+ lineinfile: "dest=/root/.bash_profile line={{ item }}"
+ with_items:
+ - "export RTE_SDK={{ dpdk_sdk }}"
+ - "export RTE_TARGET={{ dpdk_target }}"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/packet-capture/tasks/kernel.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/packet-capture/tasks/kernel.yml b/metron-deployment/roles/packet-capture/tasks/kernel.yml
new file mode 100644
index 0000000..cd4abe6
--- /dev/null
+++ b/metron-deployment/roles/packet-capture/tasks/kernel.yml
@@ -0,0 +1,51 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+# DPDK requires specific kernel boot parameters. set the params and reboot
+# the host, if the actual params differ from what is expected.
+#
+---
+ - set_fact:
+ expected_kernel_params: "default_hugepagesz=1G hugepagesz=1G hugepages={{ num_huge_pages }} iommu=pt intel_iommu=on"
+
+ - name: Check kernel boot parameters
+ shell: "cat /proc/cmdline"
+ register: actual_kernel_params
+
+ - name: Alter kernel boot parameters
+ lineinfile:
+ dest: /etc/default/grub
+ regexp: '^(GRUB_CMDLINE_LINUX=\"[^\"]+)\"$'
+ line: '\1 {{ expected_kernel_params }}"'
+ backrefs: yes
+ when: not expected_kernel_params in actual_kernel_params.stdout
+
+ - name: Update grub with kernel boot parameters
+ shell: /sbin/grub2-mkconfig -o /boot/grub2/grub.cfg
+ when: not expected_kernel_params in actual_kernel_params.stdout
+
+ - name: Restart for modified kernel params
+ command: shutdown -r now "modified kernel params"
+ async: 0
+ poll: 0
+ ignore_errors: true
+ when: not expected_kernel_params in actual_kernel_params.stdout
+
+ - name: Wait for reboot of '{{ inventory_hostname }}'
+ local_action: wait_for host={{ inventory_hostname }} state=started port=22 timeout=300 delay=10
+ become: false
+ when: not expected_kernel_params in actual_kernel_params.stdout
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/packet-capture/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/packet-capture/tasks/main.yml b/metron-deployment/roles/packet-capture/tasks/main.yml
new file mode 100644
index 0000000..f096178
--- /dev/null
+++ b/metron-deployment/roles/packet-capture/tasks/main.yml
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+ - include: dependencies.yml
+ - include: kernel.yml
+ - include: dpdk.yml
+ - include: pcapture.yml
+ - include: debug.yml
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/packet-capture/tasks/pcapture.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/packet-capture/tasks/pcapture.yml b/metron-deployment/roles/packet-capture/tasks/pcapture.yml
new file mode 100644
index 0000000..d00d379
--- /dev/null
+++ b/metron-deployment/roles/packet-capture/tasks/pcapture.yml
@@ -0,0 +1,49 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Distribute pcapture
+ copy: src=../../../metron-sensors/packet-capture dest={{ pcapture_work_dir | dirname }} mode=0755
+
+- name: Build pcapture
+ shell: "{{ item }}"
+ args:
+ chdir: "{{ pcapture_work_dir }}"
+ with_items:
+ - make
+ environment:
+ RTE_SDK: "{{ dpdk_sdk }}"
+ RTE_TARGET: "{{ dpdk_target }}"
+ LD_LIBRARY_PATH: "{{ pcapture_ld_library_path }}"
+
+- name: Install pcapture
+ shell: "cp {{ pcapture_work_dir }}/src/build/app/{{ pcapture_bin }} {{ pcapture_prefix }}"
+ args:
+ chdir: "{{ pcapture_work_dir }}"
+ creates: "{{ pcapture_prefix }}/{{ pcapture_bin }}"
+
+- name: Deploy configuration
+ template: src=pcapture.conf dest={{ pcapture_kafka_config }} mode=0755
+
+- name: Deploy service
+ template: src=pcapture dest=/etc/init.d/ mode=0755
+
+- name: Register the service with systemd
+ shell: systemctl enable pcapture
+ when: ansible_distribution == "CentOS" and ansible_distribution_major_version == "7"
+
+- name: Run pcapture
+ service: name=pcapture state=restarted
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/packet-capture/templates/pcapture
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/packet-capture/templates/pcapture b/metron-deployment/roles/packet-capture/templates/pcapture
new file mode 100644
index 0000000..8c2221a
--- /dev/null
+++ b/metron-deployment/roles/packet-capture/templates/pcapture
@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# pcapture daemon
+# chkconfig: 345 20 80
+# description: Packet capture probe
+# processname: pcapture
+#
+
+export RTE_SDK="{{ dpdk_sdk }}"
+export RTE_TARGET="{{ dpdk_target }}"
+export LD_LIBRARY_PATH="{{ pcapture_ld_library_path }}"
+
+DAEMON_PATH="{{ dpdk_sdk }}"
+DAEMON="{{ pcapture_prefix }}/{{ pcapture_bin }}"
+DAEMONOPTS+=" -- "
+DAEMONOPTS+="-p {{ pcapture_portmask }} "
+DAEMONOPTS+="-t {{ pcapture_topic }} "
+DAEMONOPTS+="-c {{ pcapture_kafka_config }} "
+
+NAME="pcapture"
+DESC="Metron network packet capture probe"
+PIDFILE=/var/run/$NAME.pid
+SCRIPTNAME=/etc/init.d/$NAME
+DAEMONLOG=/var/log/$NAME.log
+NOW=`date`
+
+case "$1" in
+ start)
+ printf "%-50s" "Starting $NAME..."
+ echo "$NOW: Starting $NAME..." >> $DAEMONLOG
+ cd $DAEMON_PATH
+ PID=`$DAEMON $DAEMONOPTS >> $DAEMONLOG 2>&1 & echo $!`
+ if [ -z $PID ]; then
+ printf "%s\n" "Fail"
+ else
+ echo $PID > $PIDFILE
+ printf "%s\n" "Ok"
+ fi
+ ;;
+
+ status)
+ printf "%-50s" "Checking $NAME..."
+ if [ -f $PIDFILE ]; then
+ PID=`cat $PIDFILE`
+ if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then
+ printf "%s\n" "Process dead but pidfile exists"
+ else
+ echo "Running"
+ fi
+ else
+ printf "%s\n" "Service not running"
+ fi
+ ;;
+
+ stop)
+ printf "%-50s" "Stopping $NAME"
+ PID=`cat $PIDFILE`
+ cd $DAEMON_PATH
+ if [ -f $PIDFILE ]; then
+ echo "$NOW: Stopping $NAME with pid=$PID" >> $DAEMONLOG
+ kill -HUP $PID
+ printf "%s\n" "Ok"
+ rm -f $PIDFILE
+ else
+ printf "%s\n" "pidfile not found"
+ fi
+ ;;
+
+ restart)
+ $0 stop
+ $0 start
+ ;;
+
+ *)
+ echo "Usage: $0 {status|start|stop|restart}"
+ exit 1
+esac
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/packet-capture/templates/pcapture.conf
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/packet-capture/templates/pcapture.conf b/metron-deployment/roles/packet-capture/templates/pcapture.conf
new file mode 100644
index 0000000..e404476
--- /dev/null
+++ b/metron-deployment/roles/packet-capture/templates/pcapture.conf
@@ -0,0 +1,67 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# kafka global settings
+#
+[kafka-global]
+
+# initial list of kafka brokers
+metadata.broker.list = {{ kafka_broker_url }}
+
+# identifies the client to kafka
+client.id = metron-packet-capture
+
+# max number of messages allowed on the producer queue
+queue.buffering.max.messages = 1000
+
+# maximum time, in milliseconds, for buffering data on the producer queue
+queue.buffering.max.ms = 3000
+
+# compression codec = none, gzip or snappy
+compression.codec = snappy
+
+# maximum number of messages batched in one MessageSet (increase for better compression)
+batch.num.messages = 10
+
+# max times to retry sending a failed message set
+message.send.max.retries = 5
+
+# backoff time before retrying a message send
+retry.backoff.ms = 250
+
+# how often statistics are emitted; 0 = never
+statistics.interval.ms = 0
+
+# only provide delivery reports for failed messages
+delivery.report.only.error = false
+
+#
+# kafka topic settings
+#
+[kafka-topic]
+
+# broker acks { 1 = leader ack, 0 = no acks, -1 = in sync replica ack }
+request.required.acks = 1
+
+# local message timeout. This value is only enforced locally and limits the time a
+# produced message waits for successful delivery. A time of 0 is infinite.
+message.timeout.ms = 10000
+
+# report offset of produced message back to application. The application must be
+# use the dr_msg_cb to retrieve the offset from rd_kafka_message_t.offset
+produce.offset.report = false
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/pcap_replay/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/pcap_replay/README.md b/metron-deployment/roles/pcap_replay/README.md
new file mode 100644
index 0000000..8bc92c9
--- /dev/null
+++ b/metron-deployment/roles/pcap_replay/README.md
@@ -0,0 +1,44 @@
+Pcap Replay
+===========
+
+This project enables packet capture data to be replayed through a network interface to simulate live network traffic. This can be used to support functional, performance, and load testing of Apache Metron.
+
+Getting Started
+---------------
+
+To replay packet capture data, simply start the `pcap-replay` SysV service. To do this run the following command.
+
+```
+service pcap-replay start
+```
+
+All additional options accepted by `tcpreplay` can be passed to the service script to modify how the network data is replayed. For example, this makes it simple to control the amount and rate of data replayed during functional, performance and load testing.
+
+Example: Replay data at a rate of 10 mbps.
+
+```
+service pcap-replay start --mbps 10
+```
+
+Example: Replay data at a rate of 10 packets per second.
+
+```
+service pcap-replay start --pps 10
+```
+
+All nodes on the same subnet with their network interface set to promiscuous mode will then be able to capture the network traffic being replayed. To validate, simply run something like the following.
+
+```
+tcpdump -i eth1
+```
+
+Data
+----
+
+An example packet capture file has been installed at `/opt/pcap-replay/example.pcap`. By default, the network traffic contained within this file is continually replayed.
+
+To replay your own packet capture data, simply add any number of files containing `libpcap` formatted packet capture data to `/opt/pcap-replay`. The files must end with the `.pcap` extension. To pick up newly installed files, simply restart the service.
+
+```
+service pcap-replay restart
+```
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/pcap_replay/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/pcap_replay/defaults/main.yml b/metron-deployment/roles/pcap_replay/defaults/main.yml
new file mode 100644
index 0000000..b1fae1e
--- /dev/null
+++ b/metron-deployment/roles/pcap_replay/defaults/main.yml
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+pcap_replay_interface: eth0
+pcap_path: /opt/pcap-replay
+tcpreplay_version: 4.1.1
+tcpreplay_prefix: /opt
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/pcap_replay/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/pcap_replay/meta/main.yml b/metron-deployment/roles/pcap_replay/meta/main.yml
new file mode 100644
index 0000000..0c47853
--- /dev/null
+++ b/metron-deployment/roles/pcap_replay/meta/main.yml
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+dependencies:
+ - libselinux-python
+ - build-tools
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/pcap_replay/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/pcap_replay/tasks/main.yml b/metron-deployment/roles/pcap_replay/tasks/main.yml
new file mode 100644
index 0000000..06919ed
--- /dev/null
+++ b/metron-deployment/roles/pcap_replay/tasks/main.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- include: tcpreplay.yml
+- include: service.yml
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/pcap_replay/tasks/service.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/pcap_replay/tasks/service.yml b/metron-deployment/roles/pcap_replay/tasks/service.yml
new file mode 100644
index 0000000..9e13e7f
--- /dev/null
+++ b/metron-deployment/roles/pcap_replay/tasks/service.yml
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Create pcap directory
+ file: path={{ pcap_path }} state=directory mode=0755
+
+- name: Install init.d service script
+ template: src=pcap-replay dest=/etc/init.d/pcap-replay mode=0755
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/pcap_replay/tasks/tcpreplay.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/pcap_replay/tasks/tcpreplay.yml b/metron-deployment/roles/pcap_replay/tasks/tcpreplay.yml
new file mode 100644
index 0000000..e24dcf1
--- /dev/null
+++ b/metron-deployment/roles/pcap_replay/tasks/tcpreplay.yml
@@ -0,0 +1,38 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Download tcpreplay
+ get_url:
+ url: "https://github.com/appneta/tcpreplay/releases/download/v{{ tcpreplay_version }}/tcpreplay-{{ tcpreplay_version }}.tar.gz"
+ dest: "/tmp/tcpreplay-{{ tcpreplay_version }}.tar.gz"
+
+- name: Extract tcpreplay tarball
+ unarchive:
+ src: "/tmp/tcpreplay-{{ tcpreplay_version }}.tar.gz"
+ dest: /opt
+ copy: no
+ creates: "/opt/tcpreplay-{{ tcpreplay_version }}"
+
+- name: Compile and install tcpreplay
+ shell: "{{ item }}"
+ args:
+ chdir: "/opt/tcpreplay-{{ tcpreplay_version }}"
+ creates: "{{ tcpreplay_prefix }}/bin/tcpreplay"
+ with_items:
+ - "./configure --prefix={{ tcpreplay_prefix }}"
+ - make
+ - make install
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/pcap_replay/templates/pcap-replay
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/pcap_replay/templates/pcap-replay b/metron-deployment/roles/pcap_replay/templates/pcap-replay
new file mode 100644
index 0000000..b9ae0c3
--- /dev/null
+++ b/metron-deployment/roles/pcap_replay/templates/pcap-replay
@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# pcap replay daemon
+# chkconfig: 345 20 80
+# description: Replays packet capture data stored in libpcap format
+# processname: pcap-replay
+#
+
+DAEMON_PATH="{{ pcap_path }}"
+PCAPIN=`ls $DAEMON_PATH/*.pcap 2> /dev/null`
+IFACE="{{ pcap_replay_interface }}"
+EXTRA_ARGS="${@:2}"
+DAEMON="{{ tcpreplay_prefix }}/bin/tcpreplay"
+DAEMONOPTS="--intf1=$IFACE --loop=0 $EXTRA_ARGS $PCAPIN"
+
+NAME=pcap-replay
+DESC="Replay packet capture data"
+PIDFILE=/var/run/$NAME.pid
+SCRIPTNAME=/etc/init.d/$NAME
+
+case "$1" in
+ start)
+ printf "%-50s" "Starting $NAME..."
+
+ # ensure that a pcap file exists to replay
+ if [ -z "$PCAPIN" ]; then
+ printf "%s: %s\n" "Fail: No pcap files found at " $DAEMON_PATH
+ else
+ # kick-off the daemon
+ cd $DAEMON_PATH
+ PID=`$DAEMON $DAEMONOPTS > /dev/null 2>&1 & echo $!`
+ if [ -z $PID ]; then
+ printf "%s\n" "Fail"
+ else
+ echo $PID > $PIDFILE
+ printf "%s\n" "Ok"
+ fi
+ fi
+ ;;
+
+ status)
+ printf "%-50s" "Checking $NAME..."
+ if [ -f $PIDFILE ]; then
+ PID=`cat $PIDFILE`
+ if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then
+ printf "%s\n" "Process dead but pidfile exists"
+ else
+ echo "Running"
+ fi
+ else
+ printf "%s\n" "Service not running"
+ fi
+ ;;
+
+ stop)
+ printf "%-50s" "Stopping $NAME"
+ PID=`cat $PIDFILE`
+ cd $DAEMON_PATH
+ if [ -f $PIDFILE ]; then
+ kill -HUP $PID
+ printf "%s\n" "Ok"
+ rm -f $PIDFILE
+ else
+ printf "%s\n" "pidfile not found"
+ fi
+ ;;
+
+ restart)
+ $0 stop
+ $0 start
+ ;;
+
+ *)
+ echo "Usage: $0 {status|start|stop|restart}"
+ exit 1
+esac
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/pycapa/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/pycapa/meta/main.yml b/metron-deployment/roles/pycapa/meta/main.yml
new file mode 100644
index 0000000..3aaa18d
--- /dev/null
+++ b/metron-deployment/roles/pycapa/meta/main.yml
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+dependencies:
+ - ambari_gather_facts
+ - epel
+ - python-pip
+ - kafka-client