You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2016/03/21 18:06:07 UTC
[25/43] incubator-metron git commit: METRON-58 Remediate Deployment
Integration Testing Issues (dlyle65535 via cestella) closes
apache/incubator-metron#36
METRON-58 Remediate Deployment Integration Testing Issues (dlyle65535 via cestella) closes apache/incubator-metron#36
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/2e9f2c6c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/2e9f2c6c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/2e9f2c6c
Branch: refs/heads/Metron_0.1BETA
Commit: 2e9f2c6ceac70fb19d38281a1abe1d3ee0d088bb
Parents: 6638a71
Author: dlyle65535 <dl...@gmail.com>
Authored: Mon Mar 7 18:24:31 2016 -0500
Committer: cstella <ce...@gmail.com>
Committed: Mon Mar 7 18:24:31 2016 -0500
----------------------------------------------------------------------
.../inventory/metron_example/group_vars/all | 6 -
.../inventory/multinode-vagrant/group_vars/all | 6 -
.../inventory/singlenode-vagrant/group_vars/all | 11 +-
deployment/playbooks/ambari_install.yml | 14 +-
deployment/playbooks/metron_full_install.yml | 4 +
deployment/playbooks/metron_install.yml | 17 +-
deployment/roles/ambari_common/tasks/main.yml | 2 +-
.../roles/ambari_config/defaults/main.yml | 30 +++
.../roles/ambari_config/vars/single_node_vm.yml | 40 ++--
.../roles/ambari_master/defaults/main.yml | 19 ++
deployment/roles/ambari_master/tasks/main.yml | 7 +
deployment/roles/bro/defaults/main.yml | 20 ++
deployment/roles/bro/tasks/main.yml | 8 +
.../roles/elasticsearch/defaults/main.yml | 20 ++
deployment/roles/elasticsearch/tasks/main.yml | 4 +-
deployment/roles/hadoop_setup/defaults/main.yml | 25 ++
deployment/roles/hadoop_setup/tasks/main.yml | 3 +-
.../roles/metron_common/defaults/main.yml | 19 ++
deployment/roles/metron_common/vars/main.yml | 19 --
.../roles/metron_streaming/defaults/main.yml | 31 +++
.../roles/metron_streaming/files/extractor.json | 11 +
.../files/source/bro-config.json | 14 ++
.../files/source/pcap-config.json | 14 ++
.../files/source/snort-config.json | 14 ++
.../files/source/yaf-config.json | 14 ++
.../roles/metron_streaming/handlers/main.yml | 4 +-
.../metron_streaming/tasks/full_topology.yml | 26 +++
.../roles/metron_streaming/tasks/main.yml | 24 +-
.../metron_streaming/tasks/small_topology.yml | 26 +++
.../metron_streaming/tasks/source_config.yml | 31 +++
.../metron_streaming/tasks/threat_intel.yml | 48 ++++
.../metron_streaming/templates/threat_ip.csv | 37 +++
deployment/roles/mysql/files/geoip_ddl.sql | 49 ----
deployment/roles/mysql/files/mylogin.cnf | 19 --
.../mysql57-community-release-el6-7.noarch.rpm | Bin 8848 -> 0 bytes
deployment/roles/mysql/handlers/main.yml | 19 --
deployment/roles/mysql/tasks/main.yml | 85 -------
deployment/roles/mysql/templates/.my.cnf | 20 --
deployment/roles/mysql/vars/main.yml | 20 --
deployment/roles/mysql_client/tasks/main.yml | 34 +++
.../roles/mysql_client/templates/db_config.sql | 21 ++
.../roles/mysql_server/files/geoip_ddl.sql | 49 ++++
.../mysql57-community-release-el6-7.noarch.rpm | Bin 0 -> 8848 bytes
deployment/roles/mysql_server/handlers/main.yml | 19 ++
deployment/roles/mysql_server/tasks/main.yml | 86 +++++++
deployment/roles/mysql_server/templates/.my.cnf | 20 ++
deployment/roles/mysql_server/vars/main.yml | 20 ++
deployment/roles/pcap_replay/defaults/main.yml | 21 ++
.../roles/pcap_replay/templates/pcap-replay | 2 +-
deployment/roles/pcap_replay/vars/main.yml | 21 --
.../roles/tap_interface/defaults/main.yml | 19 ++
deployment/roles/tap_interface/tasks/main.yml | 30 +++
deployment/roles/yaf/defaults/main.yml | 29 +++
deployment/roles/yaf/tasks/main.yml | 9 +-
deployment/roles/yaf/vars/main.yml | 22 --
metron-streaming/Metron-Common/pom.xml | 5 +
.../metron/bolt/BulkMessageWriterBolt.java | 5 +-
.../java/org/apache/metron/bolt/JoinBolt.java | 1 +
.../org/apache/metron/domain/Enrichment.java | 11 +
.../java/org/apache/metron/utils/JSONUtils.java | 70 ++++++
.../resources/config/source/bro-config.json | 9 +-
.../resources/config/source/snort-config.json | 9 +-
.../resources/config/source/yaf-config.json | 9 +-
.../dataloads/bulk/ThreatIntelBulkLoader.java | 2 +-
.../enrichment/bolt/EnrichmentJoinBolt.java | 19 +-
.../enrichment/bolt/EnrichmentSplitterBolt.java | 2 +
.../enrichment/bolt/GenericEnrichmentBolt.java | 6 +-
.../enrichment/bolt/ThreatIntelJoinBolt.java | 13 +-
.../metron/threatintel/ThreatIntelAdapter.java | 2 +-
.../metron/indexing/TelemetryIndexingBolt.java | 11 +
.../metron/writer/ElasticsearchWriter.java | 10 +-
.../metron/parsing/parsers/BasicBroParser.java | 7 +-
.../parsing/parsers/BasicSnortParser.java | 1 +
.../metron/parsing/test/BasicBroParserTest.java | 22 ++
.../src/test/resources/BroParserTest.log | 3 +-
.../util/integration/ComponentRunner.java | 33 ++-
metron-streaming/Metron-Topologies/pom.xml | 17 +-
.../apache/metron/utils/SourceConfigUtils.java | 37 ++-
.../Metron_Configs/etc/env/config.properties | 3 +-
.../Metron_Configs/topologies/bro/remote.yaml | 15 +-
.../Metron_Configs/topologies/bro/test.yaml | 10 -
.../topologies/enrichment/remote.yaml | 13 +-
.../topologies/paloalto/test.yaml | 2 +-
.../Metron_Configs/topologies/pcap/parse.yaml | 4 +-
.../Metron_Configs/topologies/pcap/remote.yaml | 97 +++-----
.../Metron_Configs/topologies/snort/remote.yaml | 10 -
.../Metron_Configs/topologies/snort/test.yaml | 12 +-
.../Metron_Configs/topologies/yaf/remote.yaml | 15 +-
.../Metron_Configs/topologies/yaf/test.yaml | 10 -
.../src/main/resources/SampleIndexed/YafIndexed | 20 +-
.../src/main/resources/SampleParsed/SnortParsed | 6 +-
.../integration/EnrichmentIntegrationTest.java | 226 +++++++++++++++++--
.../integration/ParserIntegrationTest.java | 15 +-
.../metron/integration/util/TestUtils.java | 1 -
.../integration/util/mock/MockGeoAdapter.java | 26 ++-
pom.xml | 3 +-
96 files changed, 1382 insertions(+), 592 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/inventory/metron_example/group_vars/all
----------------------------------------------------------------------
diff --git a/deployment/inventory/metron_example/group_vars/all b/deployment/inventory/metron_example/group_vars/all
index b8cf9dc..3a26769 100644
--- a/deployment/inventory/metron_example/group_vars/all
+++ b/deployment/inventory/metron_example/group_vars/all
@@ -31,12 +31,6 @@ pcap_hbase_table: pcap
tracker_hbase_table: access_tracker
threatintel_ip_hbase_table: malicious_ip
-# kafka
-pycapa_topic: pcap
-bro_topic: bro
-yaf_topic: ipfix
-snort_topic: snort
-
#elasticsearch
elasticsearch_transport_port: 9300
elasticsearch_network_interface: eth0
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/inventory/multinode-vagrant/group_vars/all
----------------------------------------------------------------------
diff --git a/deployment/inventory/multinode-vagrant/group_vars/all b/deployment/inventory/multinode-vagrant/group_vars/all
index bb41e89..fc3b56d 100644
--- a/deployment/inventory/multinode-vagrant/group_vars/all
+++ b/deployment/inventory/multinode-vagrant/group_vars/all
@@ -28,12 +28,6 @@ pcap_hbase_table: pcap
tracker_hbase_table: access_tracker
threatintel_ip_hbase_table: malicious_ip
-# kafka
-pycapa_topic: pcap
-bro_topic: bro
-yaf_topic: ipfix
-snort_topic: snort
-
#elasticsearch
elasticsearch_transport_port: 9300
elasticsearch_network_interface: eth1
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/inventory/singlenode-vagrant/group_vars/all
----------------------------------------------------------------------
diff --git a/deployment/inventory/singlenode-vagrant/group_vars/all b/deployment/inventory/singlenode-vagrant/group_vars/all
index 1e08a6a..6405eea 100644
--- a/deployment/inventory/singlenode-vagrant/group_vars/all
+++ b/deployment/inventory/singlenode-vagrant/group_vars/all
@@ -28,12 +28,6 @@ pcap_hbase_table: pcap
tracker_hbase_table: access_tracker
threatintel_ip_hbase_table: malicious_ip
-# kafka
-pycapa_topic: pcap
-bro_topic: bro
-yaf_topic: ipfix
-snort_topic: snort
-
#elasticsearch
elasticsearch_transport_port: 9300
elasticsearch_network_interface: eth1
@@ -55,7 +49,7 @@ snort_version: "2.9.8.0-1"
snort_alert_csv_path: "/var/log/snort/alert.csv"
#PCAP Replay
-pcap_replay: True
+pcap_replay: False
pcap_replay_interface: eth1
#data directories - only required to override defaults
@@ -73,3 +67,6 @@ storm_local_dir: "/data1/hadoop/storm"
kafka_log_dirs: "/data1/kafka-log"
elasticsearch_data_dir: "/data1/elasticsearch,/data2/elasticsearch"
+ambari_server_mem: 512
+topology_name: small_topology.yml
+threat_intel_bulk_load: False
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/playbooks/ambari_install.yml
----------------------------------------------------------------------
diff --git a/deployment/playbooks/ambari_install.yml b/deployment/playbooks/ambari_install.yml
index e1da427..c7f8249 100644
--- a/deployment/playbooks/ambari_install.yml
+++ b/deployment/playbooks/ambari_install.yml
@@ -19,17 +19,29 @@
sudo: yes
roles:
- role: ambari_common
+ tags:
+ - ambari-prereqs
+ - hdp-install
- hosts: ambari_master
sudo: yes
roles:
- - role: ambari_master
+ - role: ambari_master
+ tags:
+ - ambari-server
+ - hdp-install
- hosts: ambari_slave
sudo: yes
roles:
- role: ambari_slave
+ tags:
+ - ambari-agent
+ - hdp-install
- hosts: ambari_master
roles:
- role: ambari_config
+ tags:
+ - hdp-install
+ - hdp-deploy
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/playbooks/metron_full_install.yml
----------------------------------------------------------------------
diff --git a/deployment/playbooks/metron_full_install.yml b/deployment/playbooks/metron_full_install.yml
index 38203da..26ffd62 100644
--- a/deployment/playbooks/metron_full_install.yml
+++ b/deployment/playbooks/metron_full_install.yml
@@ -16,4 +16,8 @@
#
---
- include: ambari_install.yml
+ tags:
+ - ambari
- include: metron_install.yml
+ tags:
+ - metron
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/playbooks/metron_install.yml
----------------------------------------------------------------------
diff --git a/deployment/playbooks/metron_install.yml b/deployment/playbooks/metron_install.yml
index ad070c9..b8646fc 100644
--- a/deployment/playbooks/metron_install.yml
+++ b/deployment/playbooks/metron_install.yml
@@ -19,12 +19,16 @@
sudo: yes
roles:
- role: metron_common
+ tags:
+ - metron-prereqs
- hosts: hadoop_client
sudo: yes
roles:
- role: ambari_gather_facts
- role: hadoop_setup
+ tags:
+ - metron-prereqs
- hosts: search
sudo: yes
@@ -38,13 +42,22 @@
- hosts: mysql
sudo: yes
roles:
- - role: mysql
+ - role: mysql_server
tags:
- - mysql
+ - mysql-server
+
+- hosts: ambari_slave
+ sudo: yes
+ roles:
+ - role: mysql_client
+ tags:
+ - mysql-client
+
- hosts: sensors
sudo: yes
roles:
+ - { role: tap_interface, when: install_tap | default(False) == True }
- role: ambari_gather_facts
- role: flume
- role: pycapa
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/ambari_common/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_common/tasks/main.yml b/deployment/roles/ambari_common/tasks/main.yml
index 992468e..35f3fce 100644
--- a/deployment/roles/ambari_common/tasks/main.yml
+++ b/deployment/roles/ambari_common/tasks/main.yml
@@ -59,7 +59,7 @@
- name: install epel-repo rpm
yum: pkg=/tmp/epel-release.rpm state=installed
-- name: Download HDP repo
+- name: Download Ambari repo
get_url: url="{{ rhel_ambari_install_url }}" dest=/etc/yum.repos.d/ambari.repo
- name: Clean yum
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/ambari_config/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_config/defaults/main.yml b/deployment/roles/ambari_config/defaults/main.yml
new file mode 100644
index 0000000..507b6e3
--- /dev/null
+++ b/deployment/roles/ambari_config/defaults/main.yml
@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+zookeeper_data_dir: /hadoop/zookeeper
+namenode_checkpoint_dir: /hadoop/hdfs/namesecondary
+namenode_name_dir: /hadoop/hdfs/namenode
+datanode_data_dir: /hadoop/hdfs/data
+journalnode_edits_dir: /hadoop/hdfs/journalnode
+jhs_recovery_store_ldb_path: /hadoop/mapreduce/jhs
+nodemanager_local_dirs: /hadoop/yarn/local
+timeline_ldb_store_path: /hadoop/yarn/timeline
+timeline_ldb_state_path: /hadoop/yarn/timeline
+nodemanager_log_dirs: /hadoop/yarn/log
+storm_local_dir: /hadoop/storm
+kafka_log_dirs: /kafka-log
+cluster_type: small_cluster
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/ambari_config/vars/single_node_vm.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_config/vars/single_node_vm.yml b/deployment/roles/ambari_config/vars/single_node_vm.yml
index d0d3b78..6b18825 100644
--- a/deployment/roles/ambari_config/vars/single_node_vm.yml
+++ b/deployment/roles/ambari_config/vars/single_node_vm.yml
@@ -29,47 +29,49 @@ zookeeper_slave: [ZOOKEEPER_CLIENT]
hbase_master: [HBASE_MASTER, HBASE_CLIENT]
hbase_slave: [HBASE_REGIONSERVER]
-metron_components: "{{ hadoop_master | union(zookeeper_master) | union(storm_master) | union(spark_master) | union(hbase_master) | union(hadoop_slave) | union(zookeeper_slave) | union(storm_slave) | union(spark_slave) | union(kafka_broker) | union(hbase_slave) }}"
+metron_components: "{{ hadoop_master | union(zookeeper_master) | union(storm_master) | union(hbase_master) | union(hadoop_slave) | union(zookeeper_slave) | union(storm_slave) | union(kafka_broker) | union(hbase_slave) }}"
cluster_name: "metron_cluster"
blueprint_name: "metron_blueprint"
configurations:
- zoo.cfg:
- dataDir: '{{ zookeeper_data_dir | default("/hadoop/zookeeper") }}'
+ dataDir: '{{ zookeeper_data_dir }}'
- hadoop-env:
- namenode_heapsize: 1024
- dtnode_heapsize: 1024
+ hadoop_heapsize: 1024
+ namenode_heapsize: 512
+ dtnode_heapsize: 512
+ namenode_opt_permsize: 128m
- hbase-env:
- hbase_regionserver_heapsize: 1024
- hbase_master_heapsize: 1024
+ hbase_regionserver_heapsize: 512
+ hbase_master_heapsize: 512
+ hbase_regionserver_xmn_max: 512
- hdfs-site:
- dfs.namenode.checkpoint.dir: '{{ namenode_checkpoint_dir | default("/hadoop/hdfs/namesecondary") }}'
- dfs.namenode.name.dir: '{{ namenode_name_dir | default("/hadoop/hdfs/namenode") }}'
- dfs.datanode.data.dir: '{{ datanode_data_dir | default("/hadoop/hdfs/data" ) }}'
- dfs.journalnode.edits.dir: '{{ journalnode_edits_dir | default("/hadoop/hdfs/journalnode") }}'
+ dfs.namenode.checkpoint.dir: '{{ namenode_checkpoint_dir }}'
+ dfs.namenode.name.dir: '{{ namenode_name_dir }}'
+ dfs.datanode.data.dir: '{{ datanode_data_dir }}'
+ dfs.journalnode.edits.dir: '{{ journalnode_edits_dir }}'
- yarn-env:
nodemanager_heapsize: 512
yarn_heapsize: 512
apptimelineserver_heapsize : 512
+ resourcemanager_heapsize: 1024
- mapred-env:
jobhistory_heapsize: 256
- mapred-site:
- mapreduce.jobhistory.recovery.store.leveldb.path : '{{ jhs_recovery_store_ldb_path | default("/hadoop/mapreduce/jhs") }}'
+ mapreduce.jobhistory.recovery.store.leveldb.path : '{{ jhs_recovery_store_ldb_path }}'
- yarn-site:
- yarn.nodemanager.resource.memory-mb: 1024
- yarn.scheduler.maximum-allocation-mb: 1024
- yarn.nodemanager.local-dirs : '{{ nodemanager_local_dirs| default("/hadoop/yarn/local") }}'
- yarn.timeline-service.leveldb-timeline-store.path: '{{ timeline_ldb_store_path | default("/hadoop/yarn/timeline") }}'
- yarn.timeline-service.leveldb-state-store.path: '{{ timeline_ldb_state_path| default("/hadoop/yarn/timeline") }}'
- yarn.nodemanager.log-dirs: '{{ nodemanager_log_dirs| default("/hadoop/yarn/log") }}'
+ yarn.nodemanager.local-dirs : '{{ nodemanager_local_dirs }}'
+ yarn.timeline-service.leveldb-timeline-store.path: '{{ timeline_ldb_store_path }}'
+ yarn.timeline-service.leveldb-state-store.path: '{{ timeline_ldb_state_path }}'
+ yarn.nodemanager.log-dirs: '{{ nodemanager_log_dirs }}'
- storm-site:
supervisor.slots.ports: "[6700, 6701, 6702, 6703]"
- storm.local.dir: '{{ storm_local_dir | default("/hadoop/storm") }}'
+ storm.local.dir: '{{ storm_local_dir }}'
- kafka-env:
content: "{% raw %}\n#!/bin/bash\n\n# Set KAFKA specific environment variables here.\n\n# The java implementation to use.\nexport KAFKA_HEAP_OPTS=\"-Xms256M -Xmx256M\"\nexport KAFKA_JVM_PERFORMANCE_OPTS=\"-server -XX:+UseG1GC -XX:+DisableExplicitGC -Djava.awt.headless=true\"\nexport JAVA_HOME={{java64_home}}\nexport PATH=$PATH:$JAVA_HOME/bin\nexport PID_DIR={{kafka_pid_dir}}\nexport LOG_DIR={{kafka_log_dir}}\nexport KAFKA_KERBEROS_PARAMS={{kafka_kerberos_params}}\n# Add kafka sink to classpath and related depenencies\nif [ -e \"/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar\" ]; then\n export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar\n export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/lib/*\nfi\nif [ -f /etc/kafka/conf/kafka-ranger-env.sh ]; then\n . /etc/kafka/conf/kafka-ranger-env.sh\nfi{% endraw %}"
- kafka-broker:
- log.dirs: '{{ kafka_log_dirs | default("/kafka-log") }}'
+ log.dirs: '{{ kafka_log_dirs }}'
blueprint:
stack_name: HDP
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/ambari_master/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_master/defaults/main.yml b/deployment/roles/ambari_master/defaults/main.yml
new file mode 100644
index 0000000..3b8cc73
--- /dev/null
+++ b/deployment/roles/ambari_master/defaults/main.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+ambari_server_mem: 2048
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/ambari_master/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_master/tasks/main.yml b/deployment/roles/ambari_master/tasks/main.yml
index 8c78f06..daf4e41 100644
--- a/deployment/roles/ambari_master/tasks/main.yml
+++ b/deployment/roles/ambari_master/tasks/main.yml
@@ -24,6 +24,13 @@
register: ambari_server_setup
failed_when: ambari_server_setup.stderr
+- name: Set Ambari Server Max Memory
+ replace:
+ dest: /var/lib/ambari-server/ambari-env.sh
+ regexp: "\ -Xmx2048m\ "
+ replace: " -Xmx{{ ambari_server_mem }}m "
+ backup: no
+
- name: start ambari server
service: name=ambari-server state=restarted
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/bro/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/bro/defaults/main.yml b/deployment/roles/bro/defaults/main.yml
new file mode 100644
index 0000000..c7a2c1f
--- /dev/null
+++ b/deployment/roles/bro/defaults/main.yml
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+bro_crontab_minutes: 0-59/5
+bro_crontab_job: /usr/local/bro/bin/broctl cron
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/bro/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/bro/tasks/main.yml b/deployment/roles/bro/tasks/main.yml
index 04dfe8f..0191052 100644
--- a/deployment/roles/bro/tasks/main.yml
+++ b/deployment/roles/bro/tasks/main.yml
@@ -31,6 +31,7 @@
- python-devel
- swig
- zlib-devel
+ - perl
- include: librdkafka.yml
@@ -46,3 +47,10 @@
- name: Start bro
shell: /usr/local/bro/bin/broctl start
+
+- name: Bro Cronjob
+ cron:
+ name: Bro Cron
+ minute: "{{ bro_crontab_minutes }}"
+ job: "{{ bro_crontab_job }}"
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/elasticsearch/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/elasticsearch/defaults/main.yml b/deployment/roles/elasticsearch/defaults/main.yml
new file mode 100644
index 0000000..d91fa1a
--- /dev/null
+++ b/deployment/roles/elasticsearch/defaults/main.yml
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+elasticsearch_data_dir: /var/lib/elasticsearch
+elasticsearch_network_interface: eth0
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/elasticsearch/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/elasticsearch/tasks/main.yml b/deployment/roles/elasticsearch/tasks/main.yml
index 555666b..fa8d4f3 100644
--- a/deployment/roles/elasticsearch/tasks/main.yml
+++ b/deployment/roles/elasticsearch/tasks/main.yml
@@ -55,10 +55,10 @@
with_items:
- { regexp: '#cluster\.name', line: 'cluster.name: metron' }
- { regexp: '#network\.host:', line: 'network.host: _{{
- elasticsearch_network_interface | default("eth0") }}:ipv4_' }
+ elasticsearch_network_interface }}:ipv4_' }
- { regexp: '#discovery\.zen\.ping\.unicast\.hosts',
line: 'discovery.zen.ping.unicast.hosts: [ {{ es_hosts }} ]'}
- - { regexp: '#path\.data', line: 'path.data: {{ elasticsearch_data_dir | default("/var/lib/elasticsearch")}}' }
+ - { regexp: '#path\.data', line: 'path.data: {{ elasticsearch_data_dir }}' }
notify: restart elasticsearch
- name: Start Elasticsearch.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/hadoop_setup/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/hadoop_setup/defaults/main.yml b/deployment/roles/hadoop_setup/defaults/main.yml
new file mode 100644
index 0000000..c783cea
--- /dev/null
+++ b/deployment/roles/hadoop_setup/defaults/main.yml
@@ -0,0 +1,25 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+num_partitions: 1
+retention_in_gb: 10
+pycapa_topic: pcap
+bro_topic: bro
+yaf_topic: ipfix
+snort_topic: snort
+enrichments_topic: enrichments
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/hadoop_setup/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/hadoop_setup/tasks/main.yml b/deployment/roles/hadoop_setup/tasks/main.yml
index 5e77b99..5b6c47c 100644
--- a/deployment/roles/hadoop_setup/tasks/main.yml
+++ b/deployment/roles/hadoop_setup/tasks/main.yml
@@ -26,10 +26,11 @@
#if kafka topic
- name: Create Kafka topics
- shell: "{{ kafka_home }}/bin/kafka-topics.sh --zookeeper {{ zookeeper_url }} --create --topic {{ item }} --partitions 1 --replication-factor 1"
+ shell: "{{ kafka_home }}/bin/kafka-topics.sh --zookeeper {{ zookeeper_url }} --create --topic {{ item }} --partitions {{ num_partitions }} --replication-factor 1 --config retention.bytes={{ retention_in_gb * 1024 * 1024 * 1024}}"
ignore_errors: yes
with_items:
- "{{ pycapa_topic }}"
- "{{ bro_topic }}"
- "{{ yaf_topic }}"
- "{{ snort_topic }}"
+ - "{{ enrichments_topic }}"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_common/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_common/defaults/main.yml b/deployment/roles/metron_common/defaults/main.yml
new file mode 100644
index 0000000..50aaefd
--- /dev/null
+++ b/deployment/roles/metron_common/defaults/main.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+metron_jar_name: Metron-Topologies-{{ metron_version }}.jar
+metron_jar_path: "{{ playbook_dir }}/../../metron-streaming/Metron-Topologies/target/{{ metron_jar_name }}"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_common/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_common/vars/main.yml b/deployment/roles/metron_common/vars/main.yml
deleted file mode 100644
index 50aaefd..0000000
--- a/deployment/roles/metron_common/vars/main.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-metron_jar_name: Metron-Topologies-{{ metron_version }}.jar
-metron_jar_path: "{{ playbook_dir }}/../../metron-streaming/Metron-Topologies/target/{{ metron_jar_name }}"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/defaults/main.yml b/deployment/roles/metron_streaming/defaults/main.yml
new file mode 100644
index 0000000..cb425f9
--- /dev/null
+++ b/deployment/roles/metron_streaming/defaults/main.yml
@@ -0,0 +1,31 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+source_config_path: "{{ metron_directory }}/config/source"
+threat_intel_bulk_load: True
+threat_intel_bin: "{{ metron_directory }}/bin/threatintel_bulk_load.sh"
+threat_intel_host: "{{ groups.ambari_master[0] }}"
+threat_intel_work_dir: /tmp/ti_bulk
+threat_intel_csv_filename: "threat_ip.csv"
+threat_intel_csv_filepath: "../roles/metron_streaming/templates/{{ threat_intel_csv_filename }}"
+
+topology_name: full_topology.yml
+pycapa_topic: pcap
+bro_topic: bro
+yaf_topic: ipfix
+snort_topic: snort
+enrichments_topic: enrichments
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/files/extractor.json
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/files/extractor.json b/deployment/roles/metron_streaming/files/extractor.json
new file mode 100644
index 0000000..81429e8
--- /dev/null
+++ b/deployment/roles/metron_streaming/files/extractor.json
@@ -0,0 +1,11 @@
+{
+ "config": {
+ "columns": {
+ "ip": 0
+ },
+ "indicator_column": "ip",
+ "separator": ","
+ },
+ "extractor": "CSV"
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/files/source/bro-config.json
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/files/source/bro-config.json b/deployment/roles/metron_streaming/files/source/bro-config.json
new file mode 100644
index 0000000..34109b8
--- /dev/null
+++ b/deployment/roles/metron_streaming/files/source/bro-config.json
@@ -0,0 +1,14 @@
+{
+ "index": "bro",
+ "batchSize": 5,
+ "enrichmentFieldMap":
+ {
+ "geo": ["ip_dst_addr", "ip_src_addr"],
+ "host": ["host"]
+ },
+ "threatIntelFieldMap":
+ {
+ "ip": ["ip_dst_addr", "ip_src_addr"]
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/files/source/pcap-config.json
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/files/source/pcap-config.json b/deployment/roles/metron_streaming/files/source/pcap-config.json
new file mode 100644
index 0000000..4b9c639
--- /dev/null
+++ b/deployment/roles/metron_streaming/files/source/pcap-config.json
@@ -0,0 +1,14 @@
+{
+ "index": "pcap",
+ "batchSize": 5,
+ "enrichmentFieldMap":
+ {
+ "geo": ["ip_src_addr", "ip_dst_addr"],
+ "host": ["ip_src_addr", "ip_dst_addr"]
+ },
+ "threatIntelFieldMap":
+ {
+ "ip": ["ip_src_addr", "ip_dst_addr"]
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/files/source/snort-config.json
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/files/source/snort-config.json b/deployment/roles/metron_streaming/files/source/snort-config.json
new file mode 100644
index 0000000..1208637
--- /dev/null
+++ b/deployment/roles/metron_streaming/files/source/snort-config.json
@@ -0,0 +1,14 @@
+{
+ "index": "snort",
+ "batchSize": 1,
+ "enrichmentFieldMap":
+ {
+ "geo": ["ip_dst_addr", "ip_src_addr"],
+ "host": ["host"]
+ },
+ "threatIntelFieldMap":
+ {
+ "ip": ["ip_dst_addr", "ip_src_addr"]
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/files/source/yaf-config.json
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/files/source/yaf-config.json b/deployment/roles/metron_streaming/files/source/yaf-config.json
new file mode 100644
index 0000000..65de961
--- /dev/null
+++ b/deployment/roles/metron_streaming/files/source/yaf-config.json
@@ -0,0 +1,14 @@
+{
+ "index": "yaf",
+ "batchSize": 5,
+ "enrichmentFieldMap":
+ {
+ "geo": ["ip_dst_addr", "ip_src_addr"],
+ "host": ["host"]
+ },
+ "threatIntelFieldMap":
+ {
+ "ip": ["ip_dst_addr", "ip_src_addr"]
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/handlers/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/handlers/main.yml b/deployment/roles/metron_streaming/handlers/main.yml
index 112c5ca..634d591 100644
--- a/deployment/roles/metron_streaming/handlers/main.yml
+++ b/deployment/roles/metron_streaming/handlers/main.yml
@@ -15,5 +15,5 @@
# limitations under the License.
#
---
-- name: restart elasticsearch
- service: name=elasticsearch state=restarted
+- name: Load Source Config
+ shell: java -cp {{ metron_directory }}/lib/{{ metron_jar_name }}::/usr/hdp/current/hadoop-client/lib/slf4j-api-1.7.10.jar org.apache.metron.utils.SourceConfigUtils -p {{ source_config_path }} -z {{ zookeeper_url }} && touch {{ source_config_path }}/configured
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/tasks/full_topology.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/tasks/full_topology.yml b/deployment/roles/metron_streaming/tasks/full_topology.yml
new file mode 100644
index 0000000..060caf8
--- /dev/null
+++ b/deployment/roles/metron_streaming/tasks/full_topology.yml
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+
+- name: Submit Metron topologies
+ command: storm jar {{ metron_directory }}/lib/{{ metron_jar_name }} org.apache.storm.flux.Flux --remote {{ item }} --filter {{ metron_properties_config_path }}
+ with_items:
+ - "{{ metron_directory }}/config/topologies/bro/remote.yaml"
+ - "{{ metron_directory }}/config/topologies/snort/remote.yaml"
+ - "{{ metron_directory }}/config/topologies/yaf/remote.yaml"
+ - "{{ metron_directory }}/config/topologies/pcap/parse.yaml"
+ - "{{ metron_directory }}/config/topologies/enrichment/remote.yaml"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/tasks/main.yml b/deployment/roles/metron_streaming/tasks/main.yml
index 7d6fe9c..c1e1642 100644
--- a/deployment/roles/metron_streaming/tasks/main.yml
+++ b/deployment/roles/metron_streaming/tasks/main.yml
@@ -52,6 +52,10 @@
- "etc"
- "topologies"
+- name: Get Default mysql passowrd
+ include_vars: "../roles/mysql_server/vars/main.yml"
+ when: mysql_root_password is undefined
+
- name: Configure Metron topologies
lineinfile: >
dest={{ metron_properties_config_path }}
@@ -59,6 +63,7 @@
line="{{ item.line }}"
with_items:
- { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" }
+ - { regexp: "kafka.broker=", line: "kafka.broker={{ kafka_broker_url }}" }
- { regexp: "es.ip=", line: "es.ip={{ groups.search[0] }}" }
- { regexp: "es.port=", line: "es.port={{ elasticsearch_transport_port }}" }
- { regexp: "es.clustername=", line: "es.clustername={{ elasticsearch_cluster_name }}" }
@@ -70,7 +75,8 @@
- { regexp: "threat.intel.tracker.cf=", line: "threat.intel.tracker.cf=t" }
- { regexp: "threat.intel.ip.table=", line: "threat.intel.ip.table={{ threatintel_ip_hbase_table }}" }
- { regexp: "threat.intel.ip.cf=", line: "threat.intel.ip.cf=t" }
- - { regexp: "mysql.ip=", line: "mysql.ip={{ groups.search[0] }}" }
+ - { regexp: "mysql.ip=", line: "mysql.ip={{ groups.mysql[0] }}" }
+ - { regexp: "mysql.password=", line: "mysql.password={{ mysql_root_password }}" }
- name: Add Elasticsearch templates for topologies
uri:
@@ -80,11 +86,11 @@
status_code: 200
body_format: json
-- name: Submit Metron topologies
- command: storm jar {{ metron_directory }}/lib/{{ metron_jar_name }} org.apache.storm.flux.Flux --remote {{ item }} --filter {{ metron_properties_config_path }}
- ignore_errors: yes
- with_items:
- - "{{ metron_directory }}/config/topologies/pcap/remote.yaml"
- - "{{ metron_directory }}/config/topologies/bro/remote.yaml"
- - "{{ metron_directory }}/config/topologies/snort/remote.yaml"
- - "{{ metron_directory }}/config/topologies/yaf/remote.yaml"
+- include: source_config.yml
+ run_once: true
+- include: threat_intel.yml
+ run_once: true
+ when: threat_intel_bulk_load == True
+
+- include: "{{ topology_name }}"
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/tasks/small_topology.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/tasks/small_topology.yml b/deployment/roles/metron_streaming/tasks/small_topology.yml
new file mode 100644
index 0000000..6707210
--- /dev/null
+++ b/deployment/roles/metron_streaming/tasks/small_topology.yml
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+
+- name: Submit Metron topologies
+ command: storm jar {{ metron_directory }}/lib/{{ metron_jar_name }} org.apache.storm.flux.Flux --remote {{ item }} --filter {{ metron_properties_config_path }}
+ with_items:
+ - "{{ metron_directory }}/config/topologies/bro/remote.yaml"
+ - "{{ metron_directory }}/config/topologies/pcap/parse.yaml"
+ - "{{ metron_directory }}/config/topologies/yaf/remote.yaml"
+ - "{{ metron_directory }}/config/topologies/enrichment/remote.yaml"
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/tasks/source_config.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/tasks/source_config.yml b/deployment/roles/metron_streaming/tasks/source_config.yml
new file mode 100644
index 0000000..9233bac
--- /dev/null
+++ b/deployment/roles/metron_streaming/tasks/source_config.yml
@@ -0,0 +1,31 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Create Source Config Directory
+ file:
+ path: "{{ source_config_path }}"
+ state: directory
+
+- name: Copy Source Config Files
+ copy:
+ src: "{{ item }}"
+ dest: "{{ source_config_path }}"
+ mode: 0644
+ with_fileglob:
+ - ../roles/metron_streaming/files/source/*.json
+ notify: Load Source Config
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/tasks/threat_intel.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/tasks/threat_intel.yml b/deployment/roles/metron_streaming/tasks/threat_intel.yml
new file mode 100644
index 0000000..0439e46
--- /dev/null
+++ b/deployment/roles/metron_streaming/tasks/threat_intel.yml
@@ -0,0 +1,48 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Create root user HDFS directory
+ command: su - hdfs -c "hdfs dfs -mkdir -p /user/root && hdfs dfs -chown root:root /user/root"
+
+- name: Create Bulk load working Directory
+ file:
+ path: "{{ threat_intel_work_dir }}"
+ state: directory
+
+- name: Copy extractor.json to {{ inventory_hostname }}
+ copy:
+ src: ../roles/metron_streaming/files/extractor.json
+ dest: "{{ threat_intel_work_dir }}"
+ mode: 0644
+
+- name: Copy Bulk Load CSV File
+ template:
+ src: "{{ threat_intel_csv_filepath }}"
+ dest: "{{ threat_intel_work_dir }}/{{ threat_intel_csv_filename }}"
+ mode: 0644
+
+- name: Copy Bulk Load CSV File to HDFS
+ command: "hdfs dfs -put {{ threat_intel_work_dir }}/{{ threat_intel_csv_filename }} ."
+
+- name: Run Threat Intel Bulk Load
+ shell: "{{ threat_intel_bin }} -f t --table malicious_ip -e {{ threat_intel_work_dir }}/extractor.json -i /user/root && touch {{ threat_intel_work_dir }}/loaded"
+ args:
+ creates: "{{ threat_intel_work_dir }}/loaded"
+
+- name: Clean up HDFS File
+ command: "hdfs dfs -rm {{ threat_intel_csv_filename }}"
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/templates/threat_ip.csv
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/templates/threat_ip.csv b/deployment/roles/metron_streaming/templates/threat_ip.csv
new file mode 100644
index 0000000..3ac38f3
--- /dev/null
+++ b/deployment/roles/metron_streaming/templates/threat_ip.csv
@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#Add single column of ip address to alert
+#Public lists are available on the internet
+# example:
+23.113.113.105
+24.107.205.249
+24.108.62.255
+24.224.153.71
+27.4.1.212
+27.131.149.102
+31.24.30.31
+31.131.251.33
+31.186.99.250
+31.192.209.119
+31.192.209.150
+31.200.244.17
+37.34.52.185
+37.58.112.101
+37.99.146.27
+37.128.132.96
+37.140.195.177
+37.140.199.100
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql/files/geoip_ddl.sql
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql/files/geoip_ddl.sql b/deployment/roles/mysql/files/geoip_ddl.sql
deleted file mode 100644
index 02616c6..0000000
--- a/deployment/roles/mysql/files/geoip_ddl.sql
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-CREATE DATABASE IF NOT EXISTS GEO;
-
-USE GEO;
-
-DROP TABLE IF EXISTS `blocks`;
-CREATE TABLE `blocks` ( `startIPNum` int(10) unsigned NOT NULL,`endIPNum` int(10) unsigned NOT NULL,`locID`
-int(10) unsigned NOT NULL, PRIMARY KEY (`startIPNum`,`endIPNum`) )
-ENGINE=MyISAM DEFAULT CHARSET=latin1 PACK_KEYS=1 DELAY_KEY_WRITE=1;
-
-DROP TABLE IF EXISTS `location`;
-CREATE TABLE `location` (`locID` int(10) unsigned NOT NULL,`country` char(2) default NULL,`region` char(2)
- default NULL,`city` varchar(45) default NULL,`postalCode` char(7) default NULL,`latitude` double default
-NULL,`longitude` double default NULL,`dmaCode` char(3) default NULL,`areaCode` char(3) default NULL,PRIMARY KEY
- (`locID`),KEY `Index_Country` (`country`) ) ENGINE=MyISAM DEFAULT CHARSET=latin1 ROW_FORMAT=FIXED;
-
-load data infile '/var/lib/mysql-files/GeoLiteCity-Blocks.csv' into table `blocks` fields terminated by ',' optionally enclosed by '"' lines terminated by '\n' ignore 2 lines;
-load data infile '/var/lib/mysql-files/GeoLiteCity-Location.csv' into table `location` fields terminated by ',' optionally enclosed by '"' lines terminated by '\n' ignore 2 lines;
-
-
-DELIMITER $$
-DROP FUNCTION IF EXISTS `IPTOLOCID` $$
-CREATE FUNCTION `IPTOLOCID`( ip VARCHAR(15)) RETURNS int(10) unsigned
- BEGIN
- DECLARE ipn INTEGER UNSIGNED;
- DECLARE locID_var INTEGER;
- IF ip LIKE '192.168.%' OR ip LIKE '10.%' THEN RETURN 0;
- END IF;
- SET ipn = INET_ATON(ip);
- SELECT locID INTO locID_var FROM `blocks` INNER JOIN (SELECT MAX(startIPNum) AS start FROM `blocks` WHERE startIPNum <= ipn) AS s ON (startIPNum = s.start) WHERE endIPNum >= ipn;
- RETURN locID_var;
- END
-$$
-DELIMITER ;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql/files/mylogin.cnf
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql/files/mylogin.cnf b/deployment/roles/mysql/files/mylogin.cnf
deleted file mode 100644
index b8d5781..0000000
--- a/deployment/roles/mysql/files/mylogin.cnf
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-[client]
-user=root
-password=P@ssw0rd
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql/files/mysql57-community-release-el6-7.noarch.rpm
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql/files/mysql57-community-release-el6-7.noarch.rpm b/deployment/roles/mysql/files/mysql57-community-release-el6-7.noarch.rpm
deleted file mode 100644
index 8603602..0000000
Binary files a/deployment/roles/mysql/files/mysql57-community-release-el6-7.noarch.rpm and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql/handlers/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql/handlers/main.yml b/deployment/roles/mysql/handlers/main.yml
deleted file mode 100644
index 112c5ca..0000000
--- a/deployment/roles/mysql/handlers/main.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: restart elasticsearch
- service: name=elasticsearch state=restarted
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql/tasks/main.yml b/deployment/roles/mysql/tasks/main.yml
deleted file mode 100644
index 91db896..0000000
--- a/deployment/roles/mysql/tasks/main.yml
+++ /dev/null
@@ -1,85 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Create temporary directories
- file:
- path: "/tmp/{{ item }}"
- state: directory
- mode: 0755
- with_items:
- - "geoip"
-
-- name: Distribute Mysql
- copy:
- src: "{{ mysql_rpm_version }}.rpm"
- dest: /tmp
-
-- name: Install Msyql Yum Repository
- yum:
- name: "/tmp/{{ mysql_rpm_version }}.rpm"
-
-- name: Install MySQL
- yum:
- name: "{{ item }}"
- state: latest
- with_items:
- - "mysql-community-server"
- - "MySQL-python"
-
-- name: Start MySQL
- service:
- name: mysqld
- state: started
- enabled: yes
-
-- name: Retrieve temporary root password
- shell: "grep 'temporary password' /var/log/mysqld.log | sed 's/.*root@localhost: //'"
- args:
- creates: ~/.my.cnf
- register: temp_root_password
-
-- name: Update mysql root password
- command: "mysqladmin --user=root --password='{{ temp_root_password.stdout }}' password '{{ mysql_root_password }}'"
- ignore_errors: yes
- args:
- creates: ~/.my.cnf
-
-- name: Copy mylogin.cnf
- copy:
- src: mylogin.cnf
- dest: ~/.my.cnf
-
-- name: Download GeoIP databases
- unarchive:
- src: http://geolite.maxmind.com/download/geoip/database/GeoLiteCity_CSV/GeoLiteCity-latest.tar.xz
- dest: /tmp/geoip
- copy: no
- creates: /tmp/geopip/*/GeoLiteCity-Blocks.csv
-
-- name: Copy to MySQL import directory
- shell: "cp /tmp/geoip/*/*.csv /var/lib/mysql-files/"
-
-- name: Copy DDL
- copy:
- src: geoip_ddl.sql
- dest: /tmp/geoip_ddl.sql
-
-- name: Import GeoIP DDL
- mysql_db:
- name: all
- state: import
- target: /tmp/geoip_ddl.sql
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql/templates/.my.cnf
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql/templates/.my.cnf b/deployment/roles/mysql/templates/.my.cnf
deleted file mode 100644
index d5c0825..0000000
--- a/deployment/roles/mysql/templates/.my.cnf
+++ /dev/null
@@ -1,20 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-[client]
-user=root
-password={{ mysql_root_password }}
-host=localhost
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql/vars/main.yml b/deployment/roles/mysql/vars/main.yml
deleted file mode 100644
index ccf2426..0000000
--- a/deployment/roles/mysql/vars/main.yml
+++ /dev/null
@@ -1,20 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-mysql_rpm_version: mysql57-community-release-el6-7.noarch
-mysql_root_password: P@ssw0rd
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql_client/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql_client/tasks/main.yml b/deployment/roles/mysql_client/tasks/main.yml
new file mode 100644
index 0000000..5c98eb9
--- /dev/null
+++ b/deployment/roles/mysql_client/tasks/main.yml
@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+
+- name: Get Default mysql passowrd
+ include_vars: "../roles/mysql_server/vars/main.yml"
+ when: mysql_root_password is undefined
+
+- name: Allow remote login to mysql
+ template:
+ src: "../roles/mysql_client/templates/db_config.sql"
+ dest: "/tmp/{{ansible_fqdn}}.sql"
+ delegate_to: "{{ groups.mysql[0] }}"
+
+- name: Import DB_Config
+ mysql_db:
+ name: "all"
+ state: "import"
+ target: "/tmp/{{ansible_fqdn}}.sql"
+ delegate_to: "{{ groups.mysql[0] }}"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql_client/templates/db_config.sql
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql_client/templates/db_config.sql b/deployment/roles/mysql_client/templates/db_config.sql
new file mode 100644
index 0000000..c407a13
--- /dev/null
+++ b/deployment/roles/mysql_client/templates/db_config.sql
@@ -0,0 +1,21 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+CREATE USER 'root'@'{{ ansible_fqdn }}' IDENTIFIED BY '{{ mysql_root_password }}';
+SET PASSWORD FOR 'root'@'{{ ansible_fqdn }}' = PASSWORD('{{ mysql_root_password }}');
+GRANT ALL PRIVILEGES ON *.* to 'root'@'{{ ansible_fqdn }}' WITH GRANT OPTION;
+FLUSH PRIVILEGES;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql_server/files/geoip_ddl.sql
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql_server/files/geoip_ddl.sql b/deployment/roles/mysql_server/files/geoip_ddl.sql
new file mode 100644
index 0000000..02616c6
--- /dev/null
+++ b/deployment/roles/mysql_server/files/geoip_ddl.sql
@@ -0,0 +1,49 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+CREATE DATABASE IF NOT EXISTS GEO;
+
+USE GEO;
+
+DROP TABLE IF EXISTS `blocks`;
+CREATE TABLE `blocks` ( `startIPNum` int(10) unsigned NOT NULL,`endIPNum` int(10) unsigned NOT NULL,`locID`
+int(10) unsigned NOT NULL, PRIMARY KEY (`startIPNum`,`endIPNum`) )
+ENGINE=MyISAM DEFAULT CHARSET=latin1 PACK_KEYS=1 DELAY_KEY_WRITE=1;
+
+DROP TABLE IF EXISTS `location`;
+CREATE TABLE `location` (`locID` int(10) unsigned NOT NULL,`country` char(2) default NULL,`region` char(2)
+ default NULL,`city` varchar(45) default NULL,`postalCode` char(7) default NULL,`latitude` double default
+NULL,`longitude` double default NULL,`dmaCode` char(3) default NULL,`areaCode` char(3) default NULL,PRIMARY KEY
+ (`locID`),KEY `Index_Country` (`country`) ) ENGINE=MyISAM DEFAULT CHARSET=latin1 ROW_FORMAT=FIXED;
+
+load data infile '/var/lib/mysql-files/GeoLiteCity-Blocks.csv' into table `blocks` fields terminated by ',' optionally enclosed by '"' lines terminated by '\n' ignore 2 lines;
+load data infile '/var/lib/mysql-files/GeoLiteCity-Location.csv' into table `location` fields terminated by ',' optionally enclosed by '"' lines terminated by '\n' ignore 2 lines;
+
+
+DELIMITER $$
+DROP FUNCTION IF EXISTS `IPTOLOCID` $$
+CREATE FUNCTION `IPTOLOCID`( ip VARCHAR(15)) RETURNS int(10) unsigned
+ BEGIN
+ DECLARE ipn INTEGER UNSIGNED;
+ DECLARE locID_var INTEGER;
+ IF ip LIKE '192.168.%' OR ip LIKE '10.%' THEN RETURN 0;
+ END IF;
+ SET ipn = INET_ATON(ip);
+ SELECT locID INTO locID_var FROM `blocks` INNER JOIN (SELECT MAX(startIPNum) AS start FROM `blocks` WHERE startIPNum <= ipn) AS s ON (startIPNum = s.start) WHERE endIPNum >= ipn;
+ RETURN locID_var;
+ END
+$$
+DELIMITER ;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql_server/files/mysql57-community-release-el6-7.noarch.rpm
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql_server/files/mysql57-community-release-el6-7.noarch.rpm b/deployment/roles/mysql_server/files/mysql57-community-release-el6-7.noarch.rpm
new file mode 100644
index 0000000..8603602
Binary files /dev/null and b/deployment/roles/mysql_server/files/mysql57-community-release-el6-7.noarch.rpm differ
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql_server/handlers/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql_server/handlers/main.yml b/deployment/roles/mysql_server/handlers/main.yml
new file mode 100644
index 0000000..112c5ca
--- /dev/null
+++ b/deployment/roles/mysql_server/handlers/main.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: restart elasticsearch
+ service: name=elasticsearch state=restarted
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql_server/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql_server/tasks/main.yml b/deployment/roles/mysql_server/tasks/main.yml
new file mode 100644
index 0000000..987c160
--- /dev/null
+++ b/deployment/roles/mysql_server/tasks/main.yml
@@ -0,0 +1,86 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Create temporary directories
+ file:
+ path: "/tmp/{{ item }}"
+ state: directory
+ mode: 0755
+ with_items:
+ - "geoip"
+
+- name: Distribute Mysql
+ copy:
+ src: "{{ mysql_rpm_version }}.rpm"
+ dest: /tmp
+
+- name: Install Msyql Yum Repository
+ yum:
+ name: "/tmp/{{ mysql_rpm_version }}.rpm"
+
+- name: Install MySQL
+ yum:
+ name: "{{ item }}"
+ state: latest
+ with_items:
+ - "mysql-community-server"
+ - "MySQL-python"
+
+- name: Start MySQL
+ service:
+ name: mysqld
+ state: started
+ enabled: yes
+
+- name: Retrieve temporary root password
+ shell: "grep 'temporary password' /var/log/mysqld.log | sed 's/.*root@localhost: //'"
+ args:
+ creates: ~/.my.cnf
+ register: temp_root_password
+
+- name: Update mysql root password
+ command: "mysqladmin --user=root --password='{{ temp_root_password.stdout }}' password '{{ mysql_root_password }}'"
+ ignore_errors: yes
+ args:
+ creates: ~/.my.cnf
+
+- name: Create .my.cnf
+ template:
+ src: "../roles/mysql_server/templates/.my.cnf"
+ dest: ~/.my.cnf
+
+
+- name: Download GeoIP databases
+ unarchive:
+ src: http://geolite.maxmind.com/download/geoip/database/GeoLiteCity_CSV/GeoLiteCity-latest.tar.xz
+ dest: /tmp/geoip
+ copy: no
+ creates: /tmp/geopip/*/GeoLiteCity-Blocks.csv
+
+- name: Copy to MySQL import directory
+ shell: "cp /tmp/geoip/*/*.csv /var/lib/mysql-files/"
+
+- name: Copy DDL
+ copy:
+ src: geoip_ddl.sql
+ dest: /tmp/geoip_ddl.sql
+
+- name: Import GeoIP DDL
+ mysql_db:
+ name: all
+ state: import
+ target: /tmp/geoip_ddl.sql
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql_server/templates/.my.cnf
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql_server/templates/.my.cnf b/deployment/roles/mysql_server/templates/.my.cnf
new file mode 100644
index 0000000..d5c0825
--- /dev/null
+++ b/deployment/roles/mysql_server/templates/.my.cnf
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+[client]
+user=root
+password={{ mysql_root_password }}
+host=localhost
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql_server/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql_server/vars/main.yml b/deployment/roles/mysql_server/vars/main.yml
new file mode 100644
index 0000000..ccf2426
--- /dev/null
+++ b/deployment/roles/mysql_server/vars/main.yml
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+mysql_rpm_version: mysql57-community-release-el6-7.noarch
+mysql_root_password: P@ssw0rd
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/pcap_replay/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/pcap_replay/defaults/main.yml b/deployment/roles/pcap_replay/defaults/main.yml
new file mode 100644
index 0000000..b1fae1e
--- /dev/null
+++ b/deployment/roles/pcap_replay/defaults/main.yml
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+pcap_replay_interface: eth0
+pcap_path: /opt/pcap-replay
+tcpreplay_version: 4.1.1
+tcpreplay_prefix: /opt
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/pcap_replay/templates/pcap-replay
----------------------------------------------------------------------
diff --git a/deployment/roles/pcap_replay/templates/pcap-replay b/deployment/roles/pcap_replay/templates/pcap-replay
index 56dc40c..b9ae0c3 100644
--- a/deployment/roles/pcap_replay/templates/pcap-replay
+++ b/deployment/roles/pcap_replay/templates/pcap-replay
@@ -24,7 +24,7 @@
DAEMON_PATH="{{ pcap_path }}"
PCAPIN=`ls $DAEMON_PATH/*.pcap 2> /dev/null`
-IFACE="{{ pcap_replay_interface | default("eth0") }}"
+IFACE="{{ pcap_replay_interface }}"
EXTRA_ARGS="${@:2}"
DAEMON="{{ tcpreplay_prefix }}/bin/tcpreplay"
DAEMONOPTS="--intf1=$IFACE --loop=0 $EXTRA_ARGS $PCAPIN"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/pcap_replay/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/pcap_replay/vars/main.yml b/deployment/roles/pcap_replay/vars/main.yml
deleted file mode 100644
index b1fae1e..0000000
--- a/deployment/roles/pcap_replay/vars/main.yml
+++ /dev/null
@@ -1,21 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-pcap_replay_interface: eth0
-pcap_path: /opt/pcap-replay
-tcpreplay_version: 4.1.1
-tcpreplay_prefix: /opt
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/tap_interface/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/tap_interface/defaults/main.yml b/deployment/roles/tap_interface/defaults/main.yml
new file mode 100644
index 0000000..ca752b4
--- /dev/null
+++ b/deployment/roles/tap_interface/defaults/main.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+tap_if: tap0
+tap_ip: 10.0.0.1
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/tap_interface/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/tap_interface/tasks/main.yml b/deployment/roles/tap_interface/tasks/main.yml
new file mode 100644
index 0000000..d4590f7
--- /dev/null
+++ b/deployment/roles/tap_interface/tasks/main.yml
@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Install tunctl
+ yum: name=tunctl
+
+#TODO - only run when tap_if does not exist
+- name: Create {{ tap_if }}
+ command: tunctl -p
+
+- name: Bring up {{ tap_if }} on {{ tap_ip }}
+ command: ifconfig {{ tap_if }} {{ tap_ip }} up
+
+- name: Put {{ tap_if }} in PROMISC
+ command: ip link set {{ tap_if }} promisc on
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/yaf/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/yaf/defaults/main.yml b/deployment/roles/yaf/defaults/main.yml
new file mode 100644
index 0000000..f804cb5
--- /dev/null
+++ b/deployment/roles/yaf/defaults/main.yml
@@ -0,0 +1,29 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+fixbuf_version: 1.7.1
+yaf_version: 2.8.0
+yaf_home: /opt/yaf
+yaf_topic: ipfix
+hdp_repo_def: http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.2.0/hdp.repo
+yaf: /usr/local/bin/yaf
+yaf_args: ""
+yafscii: /usr/local/bin/yafscii
+yaf_log: /var/log/yaf.log
+yaf_lock: /var/lock/subsys/yaf
+kafka_prod: /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/yaf/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/yaf/tasks/main.yml b/deployment/roles/yaf/tasks/main.yml
index 1e1194d..468a4f9 100644
--- a/deployment/roles/yaf/tasks/main.yml
+++ b/deployment/roles/yaf/tasks/main.yml
@@ -43,13 +43,6 @@
- name: Install kafka
yum: name=kafka
-- set_fact:
- yaf: /usr/local/bin/yaf
- yafscii: /usr/local/bin/yafscii
- yaf_log: /var/log/yaf.log
- yaf_lock: /var/lock/subsys/yaf
- kafka_prod: /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh
-
- name: Check for Java at "{{ java_home }}"
stat: path="{{ java_home }}"
register: jdk_dir
@@ -69,6 +62,6 @@
when: not jdk_dir.stat.exists
- name: Start yaf
- shell: "daemonize -c {{ yaf_home }} -e {{ yaf_log }} -o {{ yaf_log }} -l {{ yaf_lock }} {{ yaf }} --in {{ sniff_interface }} --live pcap | {{ yafscii }} --tabular | {{ kafka_prod }} --broker-list {{ kafka_broker_url }} --topic {{ yaf_topic }}"
+ shell: "daemonize -c {{ yaf_home }} -e {{ yaf_log }} -o {{ yaf_log }} -l {{ yaf_lock }} {{ yaf }} --in {{ sniff_interface }} --live pcap {{ yaf_args }} | {{ yafscii }} --tabular | {{ kafka_prod }} --broker-list {{ kafka_broker_url }} --topic {{ yaf_topic }}"
args:
creates: "{{ yaf_lock }}"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/yaf/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/yaf/vars/main.yml b/deployment/roles/yaf/vars/main.yml
deleted file mode 100644
index 1d53958..0000000
--- a/deployment/roles/yaf/vars/main.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-fixbuf_version: 1.7.1
-yaf_version: 2.8.0
-yaf_home: /opt/yaf
-yaf_topic: ipfix
-hdp_repo_def: http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.2.0/hdp.repo
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/metron-streaming/Metron-Common/pom.xml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/pom.xml b/metron-streaming/Metron-Common/pom.xml
index c4fc5aa..605c7ed 100644
--- a/metron-streaming/Metron-Common/pom.xml
+++ b/metron-streaming/Metron-Common/pom.xml
@@ -222,6 +222,11 @@
<version>1.4</version>
<configuration>
<createDependencyReducedPom>true</createDependencyReducedPom>
+ <artifactSet>
+ <excludes>
+ <exclude>*slf4j*</exclude>
+ </excludes>
+ </artifactSet>
</configuration>
<executions>
<execution>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/BulkMessageWriterBolt.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/BulkMessageWriterBolt.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/BulkMessageWriterBolt.java
index 6d094ee..a8fda69 100644
--- a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/BulkMessageWriterBolt.java
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/BulkMessageWriterBolt.java
@@ -66,6 +66,7 @@ public class BulkMessageWriterBolt extends ConfiguredBolt {
@Override
public void execute(Tuple tuple) {
JSONObject message = (JSONObject) tuple.getValueByField("message");
+ message.put("index." + bulkMessageWriter.getClass().getSimpleName().toLowerCase() + ".ts", "" + System.currentTimeMillis());
String sourceType = TopologyUtils.getSourceType(message);
SourceConfig configuration = configurations.get(sourceType);
int batchSize = configuration != null ? configuration.getBatchSize() : 1;
@@ -80,7 +81,9 @@ public class BulkMessageWriterBolt extends ConfiguredBolt {
sourceMessageMap.put(sourceType, messageList);
} else {
try {
- bulkMessageWriter.write(sourceType, configuration, tupleList, messageList);
+
+ String esType = sourceType + "_doc";
+ bulkMessageWriter.write(esType, configuration, tupleList, messageList);
for(Tuple t: tupleList) {
collector.ack(t);
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/JoinBolt.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/JoinBolt.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/JoinBolt.java
index dac1c0a..653eade 100644
--- a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/JoinBolt.java
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/JoinBolt.java
@@ -27,6 +27,7 @@ import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.Sets;
+import org.json.simple.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/metron-streaming/Metron-Common/src/main/java/org/apache/metron/domain/Enrichment.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/domain/Enrichment.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/domain/Enrichment.java
index 7079d5c..6f43739 100644
--- a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/domain/Enrichment.java
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/domain/Enrichment.java
@@ -20,10 +20,12 @@ package org.apache.metron.domain;
import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
import java.io.Serializable;
+import java.util.List;
public class Enrichment<T extends EnrichmentAdapter> implements Serializable {
private String type;
+ private List<String> fields;
private T adapter;
public Enrichment() {}
@@ -33,6 +35,15 @@ public class Enrichment<T extends EnrichmentAdapter> implements Serializable {
this.adapter = adapter;
}
+
+ public List<String> getFields() {
+ return fields;
+ }
+
+ public void setFields(List<String> fields) {
+ this.fields = fields;
+ }
+
public String getType() {
return type;
}