You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2016/03/08 22:29:24 UTC
incubator-metron git commit: METRON-61 Allow selection of topologies
and sensors in deployment. (dlyle65535 via cestella) closes
apache/incubator-metron#38
Repository: incubator-metron
Updated Branches:
refs/heads/master f2c82c68c -> 560f7abe7
METRON-61 Allow selection of topologies and sensors in deployment. (dlyle65535 via cestella) closes apache/incubator-metron#38
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/560f7abe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/560f7abe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/560f7abe
Branch: refs/heads/master
Commit: 560f7abe7c4d9395c2471bf75cbd0e3c0e8f21bc
Parents: f2c82c6
Author: dlyle65535 <dl...@gmail.com>
Authored: Tue Mar 8 16:29:15 2016 -0500
Committer: cstella <ce...@gmail.com>
Committed: Tue Mar 8 16:29:15 2016 -0500
----------------------------------------------------------------------
.../inventory/singlenode-vagrant/group_vars/all | 23 +++++++++++------
deployment/playbooks/metron_install.yml | 12 ++++-----
deployment/roles/bro/defaults/main.yml | 2 +-
deployment/roles/bro/tasks/main.yml | 6 +++--
.../roles/metron_streaming/defaults/main.yml | 7 +++++-
.../metron_streaming/tasks/full_topology.yml | 26 --------------------
.../roles/metron_streaming/tasks/main.yml | 2 +-
.../metron_streaming/tasks/metron_topology.yml | 22 +++++++++++++++++
.../metron_streaming/tasks/small_topology.yml | 26 --------------------
9 files changed, 55 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/560f7abe/deployment/inventory/singlenode-vagrant/group_vars/all
----------------------------------------------------------------------
diff --git a/deployment/inventory/singlenode-vagrant/group_vars/all b/deployment/inventory/singlenode-vagrant/group_vars/all
index 6405eea..2d1157e 100644
--- a/deployment/inventory/singlenode-vagrant/group_vars/all
+++ b/deployment/inventory/singlenode-vagrant/group_vars/all
@@ -35,23 +35,17 @@ elasticsearch_web_port: 9200
# metron variables
metron_version: 0.1BETA
+metron_directory: /usr/metron/{{ metron_version }}
java_home: /usr/jdk64/jdk1.8.0_40
-pcapservice_port: 8081
-sniff_interface: eth1
bro_version: "2.4.1"
fixbuf_version: "1.7.1"
yaf_version: "2.8.0"
daq_version: "2.0.6-1"
-iface: "eth0"
pycapa_repo: "https://github.com/OpenSOC/pycapa.git"
pycapa_home: "/opt/pycapa"
snort_version: "2.9.8.0-1"
snort_alert_csv_path: "/var/log/snort/alert.csv"
-#PCAP Replay
-pcap_replay: False
-pcap_replay_interface: eth1
-
#data directories - only required to override defaults
zookeeper_data_dir: "/data1/hadoop/zookeeper"
namenode_checkpoint_dir: "/data1/hadoop/hdfs/namesecondary"
@@ -68,5 +62,18 @@ kafka_log_dirs: "/data1/kafka-log"
elasticsearch_data_dir: "/data1/elasticsearch,/data2/elasticsearch"
ambari_server_mem: 512
-topology_name: small_topology.yml
threat_intel_bulk_load: False
+
+#Sensors
+install_pycapa: False
+install_bro: True
+install_snort: True
+install_yaf: False
+pcap_replay: True
+sniff_interface: eth1
+pcap_replay_interface: "{{ sniff_interface }}"
+storm_topologies:
+ - "{{ metron_directory }}/config/topologies/bro/remote.yaml"
+ - "{{ metron_directory }}/config/topologies/snort/remote.yaml"
+ - "{{ metron_directory }}/config/topologies/enrichment/remote.yaml"
+pcapservice_port: 8081
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/560f7abe/deployment/playbooks/metron_install.yml
----------------------------------------------------------------------
diff --git a/deployment/playbooks/metron_install.yml b/deployment/playbooks/metron_install.yml
index b8646fc..46565cf 100644
--- a/deployment/playbooks/metron_install.yml
+++ b/deployment/playbooks/metron_install.yml
@@ -57,13 +57,13 @@
- hosts: sensors
sudo: yes
roles:
- - { role: tap_interface, when: install_tap | default(False) == True }
- role: ambari_gather_facts
- - role: flume
- - role: pycapa
- - role: bro
- - role: snort
- - role: yaf
+ - { role: tap_interface, when: install_tap | default(False) == True }
+ - { role: pycapa, when: install_pycapa | default(True) == True }
+ - { role: bro, when: install_bro | default(True) == True }
+ - { role: flume, when: install_snort | default(True) == True }
+ - { role: snort , when: install_snort | default(True) == True }
+ - { role: yaf, when: install_yaf | default(True) == True }
- { role: pcap_replay , when: pcap_replay | default(False) == True }
tags:
- sensors
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/560f7abe/deployment/roles/bro/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/bro/defaults/main.yml b/deployment/roles/bro/defaults/main.yml
index c7a2c1f..11a39ed 100644
--- a/deployment/roles/bro/defaults/main.yml
+++ b/deployment/roles/bro/defaults/main.yml
@@ -17,4 +17,4 @@
---
bro_crontab_minutes: 0-59/5
bro_crontab_job: /usr/local/bro/bin/broctl cron
-
+bro_clean_job: rm -rf /usr/local/bro/spool/tmp/*
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/560f7abe/deployment/roles/bro/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/bro/tasks/main.yml b/deployment/roles/bro/tasks/main.yml
index 0191052..3539240 100644
--- a/deployment/roles/bro/tasks/main.yml
+++ b/deployment/roles/bro/tasks/main.yml
@@ -52,5 +52,7 @@
cron:
name: Bro Cron
minute: "{{ bro_crontab_minutes }}"
- job: "{{ bro_crontab_job }}"
-
+ job: "{{ item }}"
+ with_items:
+ - "{{ bro_crontab_job }}"
+ - "{{ bro_clean_job }}"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/560f7abe/deployment/roles/metron_streaming/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/defaults/main.yml b/deployment/roles/metron_streaming/defaults/main.yml
index cb425f9..21f9c7b 100644
--- a/deployment/roles/metron_streaming/defaults/main.yml
+++ b/deployment/roles/metron_streaming/defaults/main.yml
@@ -23,9 +23,14 @@ threat_intel_work_dir: /tmp/ti_bulk
threat_intel_csv_filename: "threat_ip.csv"
threat_intel_csv_filepath: "../roles/metron_streaming/templates/{{ threat_intel_csv_filename }}"
-topology_name: full_topology.yml
pycapa_topic: pcap
bro_topic: bro
yaf_topic: ipfix
snort_topic: snort
enrichments_topic: enrichments
+storm_topologies:
+ - "{{ metron_directory }}/config/topologies/bro/remote.yaml"
+ - "{{ metron_directory }}/config/topologies/snort/remote.yaml"
+ - "{{ metron_directory }}/config/topologies/yaf/remote.yaml"
+ - "{{ metron_directory }}/config/topologies/pcap/parse.yaml"
+ - "{{ metron_directory }}/config/topologies/enrichment/remote.yaml"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/560f7abe/deployment/roles/metron_streaming/tasks/full_topology.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/tasks/full_topology.yml b/deployment/roles/metron_streaming/tasks/full_topology.yml
deleted file mode 100644
index 060caf8..0000000
--- a/deployment/roles/metron_streaming/tasks/full_topology.yml
+++ /dev/null
@@ -1,26 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-
-- name: Submit Metron topologies
- command: storm jar {{ metron_directory }}/lib/{{ metron_jar_name }} org.apache.storm.flux.Flux --remote {{ item }} --filter {{ metron_properties_config_path }}
- with_items:
- - "{{ metron_directory }}/config/topologies/bro/remote.yaml"
- - "{{ metron_directory }}/config/topologies/snort/remote.yaml"
- - "{{ metron_directory }}/config/topologies/yaf/remote.yaml"
- - "{{ metron_directory }}/config/topologies/pcap/parse.yaml"
- - "{{ metron_directory }}/config/topologies/enrichment/remote.yaml"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/560f7abe/deployment/roles/metron_streaming/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/tasks/main.yml b/deployment/roles/metron_streaming/tasks/main.yml
index c1e1642..ad1e081 100644
--- a/deployment/roles/metron_streaming/tasks/main.yml
+++ b/deployment/roles/metron_streaming/tasks/main.yml
@@ -92,5 +92,5 @@
run_once: true
when: threat_intel_bulk_load == True
-- include: "{{ topology_name }}"
+- include: metron_topology.yml
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/560f7abe/deployment/roles/metron_streaming/tasks/metron_topology.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/tasks/metron_topology.yml b/deployment/roles/metron_streaming/tasks/metron_topology.yml
new file mode 100644
index 0000000..1da24f7
--- /dev/null
+++ b/deployment/roles/metron_streaming/tasks/metron_topology.yml
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+
+- name: Submit Metron topologies
+ command: storm jar {{ metron_directory }}/lib/{{ metron_jar_name }} org.apache.storm.flux.Flux --filter {{ metron_properties_config_path }} --remote {{ item }}
+ with_items:
+ "{{ storm_topologies }}"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/560f7abe/deployment/roles/metron_streaming/tasks/small_topology.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/tasks/small_topology.yml b/deployment/roles/metron_streaming/tasks/small_topology.yml
deleted file mode 100644
index 6707210..0000000
--- a/deployment/roles/metron_streaming/tasks/small_topology.yml
+++ /dev/null
@@ -1,26 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-
-- name: Submit Metron topologies
- command: storm jar {{ metron_directory }}/lib/{{ metron_jar_name }} org.apache.storm.flux.Flux --remote {{ item }} --filter {{ metron_properties_config_path }}
- with_items:
- - "{{ metron_directory }}/config/topologies/bro/remote.yaml"
- - "{{ metron_directory }}/config/topologies/pcap/parse.yaml"
- - "{{ metron_directory }}/config/topologies/yaf/remote.yaml"
- - "{{ metron_directory }}/config/topologies/enrichment/remote.yaml"
-