You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ni...@apache.org on 2016/04/15 21:30:22 UTC
incubator-metron git commit: METRON-83 Create sensor test mode
(nickwallen) closes apache/incubator-metron#58
Repository: incubator-metron
Updated Branches:
refs/heads/master 9ca109b11 -> 3bc804fa2
METRON-83 Create sensor test mode (nickwallen) closes apache/incubator-metron#58
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/3bc804fa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/3bc804fa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/3bc804fa
Branch: refs/heads/master
Commit: 3bc804fa227ff3152cbe5454736d8e84bc7329ce
Parents: 9ca109b
Author: nickwallen <ni...@nickallen.org>
Authored: Fri Apr 15 15:29:38 2016 -0400
Committer: Nick Allen <ni...@nickallen.org>
Committed: Fri Apr 15 15:29:38 2016 -0400
----------------------------------------------------------------------
deployment/amazon-ec2/conf/defaults.yml | 7 ++-
deployment/amazon-ec2/playbook.yml | 1 +
.../inventory/metron_example/group_vars/all | 5 +-
.../inventory/multinode-vagrant/group_vars/all | 5 +-
.../inventory/singlenode-vagrant/group_vars/all | 3 +-
deployment/playbooks/metron_install.yml | 3 +-
deployment/roles/pcap_replay/files/example.pcap | Bin 507865 -> 0 bytes
deployment/roles/pcap_replay/tasks/main.yml | 4 --
deployment/roles/pcap_replay/tasks/service.yml | 3 -
deployment/roles/sensor-test-mode/README.md | 27 +++++++++
.../roles/sensor-test-mode/files/example.pcap | Bin 0 -> 507865 bytes
deployment/roles/sensor-test-mode/meta/main.yml | 19 +++++++
.../roles/sensor-test-mode/tasks/main.yml | 56 +++++++++++++++++++
deployment/roles/yaf/templates/start-yaf.sh | 2 +-
deployment/roles/yaf/templates/yaf | 1 +
.../vagrant/multinode-vagrant/ansible.cfg | 2 +-
.../vagrant/singlenode-vagrant/ansible.cfg | 1 -
17 files changed, 122 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3bc804fa/deployment/amazon-ec2/conf/defaults.yml
----------------------------------------------------------------------
diff --git a/deployment/amazon-ec2/conf/defaults.yml b/deployment/amazon-ec2/conf/defaults.yml
index 5f581f6..76c4b98 100644
--- a/deployment/amazon-ec2/conf/defaults.yml
+++ b/deployment/amazon-ec2/conf/defaults.yml
@@ -47,14 +47,15 @@ retention_in_gb: 25
metron_version: 0.1BETA
java_home: /usr/jdk64/jdk1.8.0_40
pcapservice_port: 8081
+
+# sensors
+sensor_test_mode: True
sniff_interface: tap0
snort_alert_csv_path: "/var/log/snort/alert.csv"
-yaf_args: "--idle-timeout 0"
-
-# pcap-replay
pcap_replay: True
pcap_replay_interface: tap0
install_tap: True
+pcap_path: /opt/pcap-replay
# data directories
zookeeper_data_dir: "/data1/hadoop/zookeeper"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3bc804fa/deployment/amazon-ec2/playbook.yml
----------------------------------------------------------------------
diff --git a/deployment/amazon-ec2/playbook.yml b/deployment/amazon-ec2/playbook.yml
index 2b2d060..16d281d 100644
--- a/deployment/amazon-ec2/playbook.yml
+++ b/deployment/amazon-ec2/playbook.yml
@@ -48,6 +48,7 @@
tags:
- ec2
- wait
+
#
# mount additional data volumes on all ec2 hosts
#
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3bc804fa/deployment/inventory/metron_example/group_vars/all
----------------------------------------------------------------------
diff --git a/deployment/inventory/metron_example/group_vars/all b/deployment/inventory/metron_example/group_vars/all
index 4eba586..097516d 100644
--- a/deployment/inventory/metron_example/group_vars/all
+++ b/deployment/inventory/metron_example/group_vars/all
@@ -36,6 +36,9 @@ enrichment_hbase_table: enrichment
metron_version: 0.1BETA
java_home: /usr/jdk64/jdk1.8.0_40
pcapservice_port: 8081
+
+# sensors
+sensor_test_mode: True
sniff_interface: eth0
bro_version: "2.4.1"
fixbuf_version: "1.7.1"
@@ -71,4 +74,4 @@ install_elasticsearch: True
install_solr: False
elasticsearch_transport_port: 9300
elasticsearch_network_interface: eth1
-elasticsearch_web_port: 9200
\ No newline at end of file
+elasticsearch_web_port: 9200
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3bc804fa/deployment/inventory/multinode-vagrant/group_vars/all
----------------------------------------------------------------------
diff --git a/deployment/inventory/multinode-vagrant/group_vars/all b/deployment/inventory/multinode-vagrant/group_vars/all
index 8f4e5ff..a4a6af5 100644
--- a/deployment/inventory/multinode-vagrant/group_vars/all
+++ b/deployment/inventory/multinode-vagrant/group_vars/all
@@ -38,6 +38,9 @@ elasticsearch_web_port: 9200
metron_version: 0.1BETA
java_home: /usr/jdk64/jdk1.8.0_40
pcapservice_port: 8081
+
+# sensors
+sensor_test_mode: True
sniff_interface: eth1
bro_version: "2.4.1"
fixbuf_version: "1.7.1"
@@ -69,4 +72,4 @@ install_elasticsearch: True
install_solr: False
elasticsearch_transport_port: 9300
elasticsearch_network_interface: eth1
-elasticsearch_web_port: 9200
\ No newline at end of file
+elasticsearch_web_port: 9200
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3bc804fa/deployment/inventory/singlenode-vagrant/group_vars/all
----------------------------------------------------------------------
diff --git a/deployment/inventory/singlenode-vagrant/group_vars/all b/deployment/inventory/singlenode-vagrant/group_vars/all
index 6d96d5d..54dc168 100644
--- a/deployment/inventory/singlenode-vagrant/group_vars/all
+++ b/deployment/inventory/singlenode-vagrant/group_vars/all
@@ -60,7 +60,8 @@ elasticsearch_data_dir: "/data1/elasticsearch,/data2/elasticsearch"
ambari_server_mem: 512
threat_intel_bulk_load: False
-#Sensors
+# sensors
+sensor_test_mode: True
install_pycapa: False
install_bro: True
install_snort: True
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3bc804fa/deployment/playbooks/metron_install.yml
----------------------------------------------------------------------
diff --git a/deployment/playbooks/metron_install.yml b/deployment/playbooks/metron_install.yml
index 89db980..f6bc492 100644
--- a/deployment/playbooks/metron_install.yml
+++ b/deployment/playbooks/metron_install.yml
@@ -75,7 +75,8 @@
- { role: flume, when: install_snort | default(True) == True }
- { role: snort , when: install_snort | default(True) == True }
- { role: yaf, when: install_yaf | default(True) == True }
- - { role: pcap_replay , when: pcap_replay | default(False) == True }
+ - { role: pcap_replay , when: (pcap_replay | default(False)) or (sensor_test_mode | default(False)) == True }
+ - { role: sensor-test-mode, when: sensor_test_mode | default(False) == True }
tags:
- sensors
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3bc804fa/deployment/roles/pcap_replay/files/example.pcap
----------------------------------------------------------------------
diff --git a/deployment/roles/pcap_replay/files/example.pcap b/deployment/roles/pcap_replay/files/example.pcap
deleted file mode 100644
index 06594ec..0000000
Binary files a/deployment/roles/pcap_replay/files/example.pcap and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3bc804fa/deployment/roles/pcap_replay/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/pcap_replay/tasks/main.yml b/deployment/roles/pcap_replay/tasks/main.yml
index 68005b9..06919ed 100644
--- a/deployment/roles/pcap_replay/tasks/main.yml
+++ b/deployment/roles/pcap_replay/tasks/main.yml
@@ -16,8 +16,4 @@
#
---
- include: tcpreplay.yml
-
- include: service.yml
-
-- name: Start the pcap-replay service
- service: name=pcap-replay state=restarted
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3bc804fa/deployment/roles/pcap_replay/tasks/service.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/pcap_replay/tasks/service.yml b/deployment/roles/pcap_replay/tasks/service.yml
index 8a549f7..9e13e7f 100644
--- a/deployment/roles/pcap_replay/tasks/service.yml
+++ b/deployment/roles/pcap_replay/tasks/service.yml
@@ -20,6 +20,3 @@
- name: Install init.d service script
template: src=pcap-replay dest=/etc/init.d/pcap-replay mode=0755
-
-- name: Install example pcap file
- copy: src=example.pcap dest={{ pcap_path }}/
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3bc804fa/deployment/roles/sensor-test-mode/README.md
----------------------------------------------------------------------
diff --git a/deployment/roles/sensor-test-mode/README.md b/deployment/roles/sensor-test-mode/README.md
new file mode 100644
index 0000000..76a3220
--- /dev/null
+++ b/deployment/roles/sensor-test-mode/README.md
@@ -0,0 +1,27 @@
+Sensor Test Mode
+================
+
+A role that configures each of the sensors to produce the maximum amount of telemetry data. This role is useful only for testing. It can be useful to support functional, performance, and load testing of Apache Metron.
+
+The role does the following to maximize the amount of telemetry data produced by each Metron sensor.
+
+- Plays a packet capture file through a network interface to simulate live network traffic.
+- Configures [YAF](https://tools.netsa.cert.org/yaf/yaf.html) with `idle-timeout=0`. This causes a flow record to be produced for every network packet received.
+- Configures [Snort](https://www.snort.org/) to produce an alert for every network packet received.
+
+Getting Started
+---------------
+
+To enable the `sensor-test-mode` role apply the role to the `sensors` host group in your Ansible playbook.
+
+```
+- hosts: sensors
+ roles:
+ - role: sensor-test-mode
+```
+
+The role has also been added to the default `metron_install.yml` playbook so that it can be turned on/off with a property in both the local Virtualbox and the remote EC2 deployments.
+
+```
+sensor_test_mode: True
+```
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3bc804fa/deployment/roles/sensor-test-mode/files/example.pcap
----------------------------------------------------------------------
diff --git a/deployment/roles/sensor-test-mode/files/example.pcap b/deployment/roles/sensor-test-mode/files/example.pcap
new file mode 100644
index 0000000..06594ec
Binary files /dev/null and b/deployment/roles/sensor-test-mode/files/example.pcap differ
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3bc804fa/deployment/roles/sensor-test-mode/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/sensor-test-mode/meta/main.yml b/deployment/roles/sensor-test-mode/meta/main.yml
new file mode 100644
index 0000000..0e9e5b3
--- /dev/null
+++ b/deployment/roles/sensor-test-mode/meta/main.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+dependencies:
+ - pcap_replay
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3bc804fa/deployment/roles/sensor-test-mode/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/sensor-test-mode/tasks/main.yml b/deployment/roles/sensor-test-mode/tasks/main.yml
new file mode 100644
index 0000000..26b4e2a
--- /dev/null
+++ b/deployment/roles/sensor-test-mode/tasks/main.yml
@@ -0,0 +1,56 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+#
+# load example pcap data to replay
+#
+- name: Install example pcap file
+ copy: src=example.pcap dest={{ pcap_path }}/
+
+- name: Start the pcap-replay service
+ service: name=pcap-replay state=restarted
+
+#
+# configure yaf to generate a flow record for every packet
+#
+- name: Stop running instances of yaf
+ become: True
+ service: name=yaf state=stopped
+
+- name: Configure yaf to generate a flow record for every network packet
+ become: True
+ service: name=yaf state=started args="--idle-timeout 0"
+
+#
+# configure snort to alert on every packet
+#
+- name: Configure snort to use a set of test rules
+ become: True
+ lineinfile:
+ dest: /etc/snort/snort.conf
+ line: "include $RULE_PATH/test.rules"
+
+- name: Create a snort alert for testing that alerts on every packet
+ become: True
+ lineinfile:
+ dest: /etc/snort/rules/test.rules
+ line: "alert tcp any any -> any any (msg:'snort test alert'; sid:999158; )"
+ create: yes
+
+- name: Restart snort
+ become: True
+ service: name=snortd state=restarted
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3bc804fa/deployment/roles/yaf/templates/start-yaf.sh
----------------------------------------------------------------------
diff --git a/deployment/roles/yaf/templates/start-yaf.sh b/deployment/roles/yaf/templates/start-yaf.sh
index adaf1c4..9660e72 100644
--- a/deployment/roles/yaf/templates/start-yaf.sh
+++ b/deployment/roles/yaf/templates/start-yaf.sh
@@ -22,4 +22,4 @@
# flowmeter - and sends the output to kafka so that it can be consumed
# by metron
#
-{{ yaf_bin }} --in {{ sniff_interface }} --live pcap ${@:2} | {{ yafscii_bin }} --tabular | {{ kafka_prod }} --broker-list {{ kafka_broker_url }} --topic {{ yaf_topic }}
+{{ yaf_bin }} --in {{ sniff_interface }} --live pcap "${@:1}" | {{ yafscii_bin }} --tabular | {{ kafka_prod }} --broker-list {{ kafka_broker_url }} --topic {{ yaf_topic }}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3bc804fa/deployment/roles/yaf/templates/yaf
----------------------------------------------------------------------
diff --git a/deployment/roles/yaf/templates/yaf b/deployment/roles/yaf/templates/yaf
index 0550c2a..18bc4ac 100644
--- a/deployment/roles/yaf/templates/yaf
+++ b/deployment/roles/yaf/templates/yaf
@@ -64,6 +64,7 @@ case "$1" in
cd $DAEMON_PATH
if [ -f $PIDFILE ]; then
kill -HUP $PID
+ killall $NAME
printf "%s\n" "Ok"
rm -f $PIDFILE
else
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3bc804fa/deployment/vagrant/multinode-vagrant/ansible.cfg
----------------------------------------------------------------------
diff --git a/deployment/vagrant/multinode-vagrant/ansible.cfg b/deployment/vagrant/multinode-vagrant/ansible.cfg
index 9c650c2..7a41ec8 100644
--- a/deployment/vagrant/multinode-vagrant/ansible.cfg
+++ b/deployment/vagrant/multinode-vagrant/ansible.cfg
@@ -14,9 +14,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-
[defaults]
host_key_checking = false
library = ../../extra_modules
roles_path = ../../roles
pipelining = True
+log_path = ./ansible.log
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3bc804fa/deployment/vagrant/singlenode-vagrant/ansible.cfg
----------------------------------------------------------------------
diff --git a/deployment/vagrant/singlenode-vagrant/ansible.cfg b/deployment/vagrant/singlenode-vagrant/ansible.cfg
index 0a75321..7a41ec8 100644
--- a/deployment/vagrant/singlenode-vagrant/ansible.cfg
+++ b/deployment/vagrant/singlenode-vagrant/ansible.cfg
@@ -14,7 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-
[defaults]
host_key_checking = false
library = ../../extra_modules