You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ni...@apache.org on 2016/05/13 21:29:19 UTC
[1/2] incubator-metron git commit: METRON-132 Monit integration +
METRON-85 Removed fixed wait from EC2 deployment (nickwallen) closes
apache/incubator-metron#118
Repository: incubator-metron
Updated Branches:
refs/heads/master 9a4481cab -> 9898b4c6a
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/monit/templates/monit/yaf.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/yaf.monit b/metron-deployment/roles/monit/templates/monit/yaf.monit
new file mode 100644
index 0000000..1f7b4d2
--- /dev/null
+++ b/metron-deployment/roles/monit/templates/monit/yaf.monit
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+check process yaf
+ with pidfile /var/run/yaf.pid
+ start program = "/etc/init.d/yaf start"
+ stop program = "/etc/init.d/yaf stop"
+ if does not exist then restart
+ group yaf
+ group sensors
+ group metron
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/monit/templates/scripts/start_enrichment_topology.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/scripts/start_enrichment_topology.sh b/metron-deployment/roles/monit/templates/scripts/start_enrichment_topology.sh
new file mode 100644
index 0000000..e170460
--- /dev/null
+++ b/metron-deployment/roles/monit/templates/scripts/start_enrichment_topology.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# start the enrichment topology
+#
+export METRON_VERSION={{ metron_version }}
+export METRON_HOME={{ metron_directory }}
+{{ item }}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/monit/templates/scripts/start_topology.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/scripts/start_topology.sh b/metron-deployment/roles/monit/templates/scripts/start_topology.sh
new file mode 100644
index 0000000..a8a60fa
--- /dev/null
+++ b/metron-deployment/roles/monit/templates/scripts/start_topology.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# start a storm topology
+#
+export METRON_VERSION={{ metron_version }}
+export METRON_HOME={{ metron_directory }}
+$METRON_HOME/bin/start_parser_topology.sh {{ item }}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/monit/templates/scripts/status_topology.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/scripts/status_topology.sh b/metron-deployment/roles/monit/templates/scripts/status_topology.sh
new file mode 100644
index 0000000..67e9373
--- /dev/null
+++ b/metron-deployment/roles/monit/templates/scripts/status_topology.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# is a given storm topology running?
+#
+
+TOPOLOGY={{ item }}
+STATUS=`storm list | grep $TOPOLOGY | awk '{print $2}'`
+
+if [ "ACTIVE" = "$STATUS" ]; then
+ echo "Running: $TOPOLOGY"
+ exit 0
+else
+ echo "Stopped: $TOPOLOGY"
+ exit 1
+fi
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/monit/templates/scripts/stop_topology.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/scripts/stop_topology.sh b/metron-deployment/roles/monit/templates/scripts/stop_topology.sh
new file mode 100644
index 0000000..d864621
--- /dev/null
+++ b/metron-deployment/roles/monit/templates/scripts/stop_topology.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# stop a storm topology
+#
+storm kill {{ item }}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/mysql_server/tasks/geoip.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/mysql_server/tasks/geoip.yml b/metron-deployment/roles/mysql_server/tasks/geoip.yml
new file mode 100644
index 0000000..87b5721
--- /dev/null
+++ b/metron-deployment/roles/mysql_server/tasks/geoip.yml
@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Download GeoIP databases
+ unarchive:
+ src: http://geolite.maxmind.com/download/geoip/database/GeoLiteCity_CSV/GeoLiteCity-latest.tar.xz
+ dest: /tmp/geoip
+ copy: no
+ creates: /tmp/geopip/*/GeoLiteCity-Blocks.csv
+
+- name: Copy to MySQL import directory
+ shell: "cp /tmp/geoip/*/*.csv /var/lib/mysql-files/"
+
+- name: Copy DDL
+ copy:
+ src: geoip_ddl.sql
+ dest: /tmp/geoip_ddl.sql
+
+- name: Import GeoIP DDL
+ mysql_db:
+ name: all
+ state: import
+ target: /tmp/geoip_ddl.sql
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/mysql_server/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/mysql_server/tasks/main.yml b/metron-deployment/roles/mysql_server/tasks/main.yml
index a484ed0..159e5a6 100644
--- a/metron-deployment/roles/mysql_server/tasks/main.yml
+++ b/metron-deployment/roles/mysql_server/tasks/main.yml
@@ -15,79 +15,6 @@
# limitations under the License.
#
---
-- name: Create temporary directories
- file:
- path: "/tmp/geoip"
- state: directory
- mode: 0755
-
-- name: Install Mysql Community Release Repo Def
- get_url:
- dest: /tmp/{{ mysql_rpm_version }}.rpm
- url: "{{ mysql_yum_repo_url }}"
-
-- name: Install Mysql Community Release Repo
- yum:
- pkg: /tmp/{{ mysql_rpm_version }}.rpm
- state: installed
- register: result
- until: result.rc == 0
- retries: 5
- delay: 10
-
-- name: Install MySQL
- yum:
- name: "{{ item }}"
- state: latest
- with_items:
- - "mysql-community-server"
- - "MySQL-python"
- register: result
- until: result.rc == 0
- retries: 5
- delay: 10
-
-- name: Start MySQL
- service:
- name: mysqld
- state: started
- enabled: yes
-
-- name: Retrieve temporary root password
- shell: "grep 'temporary password' /var/log/mysqld.log | sed 's/.*root@localhost: //'"
- args:
- creates: ~/.my.cnf
- register: temp_root_password
-
-- name: Update mysql root password
- command: "mysqladmin --user=root --password='{{ temp_root_password.stdout }}' password '{{ mysql_root_password }}'"
- ignore_errors: yes
- args:
- creates: ~/.my.cnf
-
-- name: Create .my.cnf
- template:
- src: "../roles/mysql_server/templates/.my.cnf"
- dest: ~/.my.cnf
-
-
-- name: Download GeoIP databases
- unarchive:
- src: http://geolite.maxmind.com/download/geoip/database/GeoLiteCity_CSV/GeoLiteCity-latest.tar.xz
- dest: /tmp/geoip
- copy: no
- creates: /tmp/geopip/*/GeoLiteCity-Blocks.csv
-
-- name: Copy to MySQL import directory
- shell: "cp /tmp/geoip/*/*.csv /var/lib/mysql-files/"
-
-- name: Copy DDL
- copy:
- src: geoip_ddl.sql
- dest: /tmp/geoip_ddl.sql
-
-- name: Import GeoIP DDL
- mysql_db:
- name: all
- state: import
- target: /tmp/geoip_ddl.sql
+- include: mysql.yml
+- include: mysql-password.yml
+- include: geoip.yml
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/mysql_server/tasks/mysql-password.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/mysql_server/tasks/mysql-password.yml b/metron-deployment/roles/mysql_server/tasks/mysql-password.yml
new file mode 100644
index 0000000..a4f0d0b
--- /dev/null
+++ b/metron-deployment/roles/mysql_server/tasks/mysql-password.yml
@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Retrieve temporary root password
+ shell: "grep 'temporary password' /var/log/mysqld.log | sed 's/.*root@localhost: //'"
+ args:
+ creates: ~/.my.cnf
+ register: temp_root_password
+
+- name: Update mysql root password
+ command: "mysqladmin --user=root --password='{{ temp_root_password.stdout }}' password '{{ mysql_root_password }}'"
+ ignore_errors: yes
+ args:
+ creates: ~/.my.cnf
+
+- name: Create .my.cnf
+ template:
+ src: "../roles/mysql_server/templates/.my.cnf"
+ dest: ~/.my.cnf
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/mysql_server/tasks/mysql.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/mysql_server/tasks/mysql.yml b/metron-deployment/roles/mysql_server/tasks/mysql.yml
new file mode 100644
index 0000000..e48ad5c
--- /dev/null
+++ b/metron-deployment/roles/mysql_server/tasks/mysql.yml
@@ -0,0 +1,60 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Create temporary directories
+ file:
+ path: "/tmp/geoip"
+ state: directory
+ mode: 0755
+
+- name: Install Mysql Community Release Repo Def
+ get_url:
+ dest: /tmp/{{ mysql_rpm_version }}.rpm
+ url: "{{ mysql_yum_repo_url }}"
+
+- name: Install Mysql Community Release Repo
+ yum:
+ pkg: /tmp/{{ mysql_rpm_version }}.rpm
+ state: installed
+ register: result
+ until: result.rc == 0
+ retries: 5
+ delay: 10
+
+- name: Install MySQL
+ yum:
+ name: "{{ item }}"
+ state: latest
+ with_items:
+ - "mysql-community-server"
+ - "MySQL-python"
+ register: result
+ until: result.rc == 0
+ retries: 5
+ delay: 10
+
+#
+# mysql must be running for the 'mysql_client' role to function properly. that
+# role grants access to mysql clients connecting from external hosts (think
+# storm topologies doing geoip enrichment).
+#
+# ideally the logic in 'mysql_client' would be moved to 'mysql_server' so that
+# mysql could be started, access granted, and then shutdown after this is
+# complete.
+#
+- name: Start MySQL
+ service: name=mysqld state=started
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/pcap_replay/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/pcap_replay/defaults/main.yml b/metron-deployment/roles/pcap_replay/defaults/main.yml
index b1fae1e..9b948a3 100644
--- a/metron-deployment/roles/pcap_replay/defaults/main.yml
+++ b/metron-deployment/roles/pcap_replay/defaults/main.yml
@@ -16,6 +16,6 @@
#
---
pcap_replay_interface: eth0
-pcap_path: /opt/pcap-replay
+pcap_replay_home: /opt/pcap-replay
tcpreplay_version: 4.1.1
tcpreplay_prefix: /opt
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/pcap_replay/tasks/service.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/pcap_replay/tasks/service.yml b/metron-deployment/roles/pcap_replay/tasks/service.yml
index 9e13e7f..46b00c5 100644
--- a/metron-deployment/roles/pcap_replay/tasks/service.yml
+++ b/metron-deployment/roles/pcap_replay/tasks/service.yml
@@ -16,7 +16,7 @@
#
---
- name: Create pcap directory
- file: path={{ pcap_path }} state=directory mode=0755
+ file: path={{ pcap_replay_home }} state=directory mode=0755
- name: Install init.d service script
template: src=pcap-replay dest=/etc/init.d/pcap-replay mode=0755
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/pcap_replay/templates/pcap-replay
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/pcap_replay/templates/pcap-replay b/metron-deployment/roles/pcap_replay/templates/pcap-replay
index b9ae0c3..1df7337 100644
--- a/metron-deployment/roles/pcap_replay/templates/pcap-replay
+++ b/metron-deployment/roles/pcap_replay/templates/pcap-replay
@@ -22,17 +22,15 @@
# processname: pcap-replay
#
-DAEMON_PATH="{{ pcap_path }}"
+DAEMON_PATH="{{ pcap_replay_home }}"
PCAPIN=`ls $DAEMON_PATH/*.pcap 2> /dev/null`
IFACE="{{ pcap_replay_interface }}"
EXTRA_ARGS="${@:2}"
-DAEMON="{{ tcpreplay_prefix }}/bin/tcpreplay"
-DAEMONOPTS="--intf1=$IFACE --loop=0 $EXTRA_ARGS $PCAPIN"
-
NAME=pcap-replay
DESC="Replay packet capture data"
PIDFILE=/var/run/$NAME.pid
SCRIPTNAME=/etc/init.d/$NAME
+LOGFILE=/var/log/$NAME.log
case "$1" in
start)
@@ -44,7 +42,9 @@ case "$1" in
else
# kick-off the daemon
cd $DAEMON_PATH
- PID=`$DAEMON $DAEMONOPTS > /dev/null 2>&1 & echo $!`
+ DAEMON="{{ tcpreplay_prefix }}/bin/tcpreplay"
+ DAEMONOPTS="--intf1=$IFACE --loop=0 $EXTRA_ARGS $PCAPIN"
+ PID=`$DAEMON $DAEMONOPTS > $LOGFILE 2>&1 & echo $!`
if [ -z $PID ]; then
printf "%s\n" "Fail"
else
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/pycapa/tasks/pycapa-service.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/pycapa/tasks/pycapa-service.yml b/metron-deployment/roles/pycapa/tasks/pycapa-service.yml
index 5342939..017cc66 100644
--- a/metron-deployment/roles/pycapa/tasks/pycapa-service.yml
+++ b/metron-deployment/roles/pycapa/tasks/pycapa-service.yml
@@ -20,6 +20,3 @@
- name: Install service script
template: src=pycapa dest=/etc/init.d/pycapa mode=0755
-
-- name: Start pycapa
- service: name=pycapa state=restarted
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/sensor-test-mode/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/sensor-test-mode/tasks/main.yml b/metron-deployment/roles/sensor-test-mode/tasks/main.yml
index 26b72c9..24ca87e 100644
--- a/metron-deployment/roles/sensor-test-mode/tasks/main.yml
+++ b/metron-deployment/roles/sensor-test-mode/tasks/main.yml
@@ -16,8 +16,10 @@
#
---
- include: pcap.yml
- when: pcap_replay
+ when: install_pcap_replay
+
- include: yaf.yml
when: install_yaf
+
- include: snort.yml
when: install_snort
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/sensor-test-mode/tasks/pcap.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/sensor-test-mode/tasks/pcap.yml b/metron-deployment/roles/sensor-test-mode/tasks/pcap.yml
index b088d5a..dda1bae 100644
--- a/metron-deployment/roles/sensor-test-mode/tasks/pcap.yml
+++ b/metron-deployment/roles/sensor-test-mode/tasks/pcap.yml
@@ -19,7 +19,4 @@
# load example pcap data to replay
#
- name: Install example pcap file
- copy: src=example.pcap dest={{ pcap_path }}/
-
-- name: Start the pcap-replay service
- service: name=pcap-replay state=restarted
+ copy: src=example.pcap dest={{ pcap_replay_home }}/
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/sensor-test-mode/tasks/snort.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/sensor-test-mode/tasks/snort.yml b/metron-deployment/roles/sensor-test-mode/tasks/snort.yml
index 5fd57ff..4deae23 100644
--- a/metron-deployment/roles/sensor-test-mode/tasks/snort.yml
+++ b/metron-deployment/roles/sensor-test-mode/tasks/snort.yml
@@ -30,7 +30,3 @@
dest: /etc/snort/rules/test.rules
line: "alert tcp any any -> any any (msg:'snort test alert'; sid:999158; )"
create: yes
-
-- name: Restart snort
- become: True
- service: name=snortd state=restarted
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/sensor-test-mode/tasks/yaf.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/sensor-test-mode/tasks/yaf.yml b/metron-deployment/roles/sensor-test-mode/tasks/yaf.yml
index 5fa29ce..1206e23 100644
--- a/metron-deployment/roles/sensor-test-mode/tasks/yaf.yml
+++ b/metron-deployment/roles/sensor-test-mode/tasks/yaf.yml
@@ -23,5 +23,8 @@
service: name=yaf state=stopped
- name: Configure yaf to generate a flow record for every network packet
- become: True
- service: name=yaf state=started args="--idle-timeout 0"
+ lineinfile:
+ dest: /etc/init.d/yaf
+ regexp: "^DAEMONOPTS=\"${@:2}\"$"
+ line: "DAEMONOPTS=\"${@:2} --idle-timeout 0\""
+ backup: yes
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/snort/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/snort/tasks/main.yml b/metron-deployment/roles/snort/tasks/main.yml
index 80755be..b2ed836 100644
--- a/metron-deployment/roles/snort/tasks/main.yml
+++ b/metron-deployment/roles/snort/tasks/main.yml
@@ -16,16 +16,6 @@
#
---
- include: daq.yml
-
- include: snort.yml
-
- include: flume.yml
-
-- name: Turn on promiscuous mode for {{ sniff_interface }}
- shell: "ip link set {{ sniff_interface }} promisc on"
-
-- name: Start snort
- service: name=snortd state=restarted
-
-- name: Start flume service to consume snort alerts
- service: name=flume-agent state=restarted args=snort
+- include: nic.yml
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/snort/tasks/nic.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/snort/tasks/nic.yml b/metron-deployment/roles/snort/tasks/nic.yml
new file mode 100644
index 0000000..6053618
--- /dev/null
+++ b/metron-deployment/roles/snort/tasks/nic.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Turn on promiscuous mode for {{ sniff_interface }}
+ shell: "ip link set {{ sniff_interface }} promisc on"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/yaf/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/yaf/tasks/main.yml b/metron-deployment/roles/yaf/tasks/main.yml
index 15f67f6..7d21348 100644
--- a/metron-deployment/roles/yaf/tasks/main.yml
+++ b/metron-deployment/roles/yaf/tasks/main.yml
@@ -17,3 +17,4 @@
---
- include: fixbuf.yml
- include: yaf.yml
+- include: nic.yml
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/yaf/tasks/nic.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/yaf/tasks/nic.yml b/metron-deployment/roles/yaf/tasks/nic.yml
new file mode 100644
index 0000000..6053618
--- /dev/null
+++ b/metron-deployment/roles/yaf/tasks/nic.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Turn on promiscuous mode for {{ sniff_interface }}
+ shell: "ip link set {{ sniff_interface }} promisc on"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/yaf/tasks/yaf.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/yaf/tasks/yaf.yml b/metron-deployment/roles/yaf/tasks/yaf.yml
index 10d3205..9a9787b 100644
--- a/metron-deployment/roles/yaf/tasks/yaf.yml
+++ b/metron-deployment/roles/yaf/tasks/yaf.yml
@@ -52,9 +52,3 @@
- name: Register the service with systemd
shell: systemctl enable pcap-replay
when: ansible_distribution == "CentOS" and ansible_distribution_major_version == "7"
-
-- name: Turn on promiscuous mode for {{ sniff_interface }}
- shell: "ip link set {{ sniff_interface }} promisc on"
-
-- name: Start yaf
- service: name=yaf state=restarted args="{{ yaf_args }}"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/vagrant/fastcapa-test-platform/playbook.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/fastcapa-test-platform/playbook.yml b/metron-deployment/vagrant/fastcapa-test-platform/playbook.yml
index 61509c4..cbc54d5 100644
--- a/metron-deployment/vagrant/fastcapa-test-platform/playbook.yml
+++ b/metron-deployment/vagrant/fastcapa-test-platform/playbook.yml
@@ -27,6 +27,8 @@
- role: pcap_replay
- { role: pycapa, install_pycapa_service: False }
- { role: sensor-test-mode, pcap_replay: True, install_yaf: False, install_snort: False }
+ - service: name=pcap-replay state=started
+ - service: name=pycapa state=started
#
# the 'sink' consumes network traffic
@@ -38,6 +40,7 @@
roles:
- role: librdkafka
- role: fastcapa
+ - service: name=fastcapa state=started
#
# validate the environment - needs to run on `source` node
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/vagrant/full-dev-platform/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/full-dev-platform/README.md b/metron-deployment/vagrant/full-dev-platform/README.md
index 4242be2..1139f0d 100644
--- a/metron-deployment/vagrant/full-dev-platform/README.md
+++ b/metron-deployment/vagrant/full-dev-platform/README.md
@@ -61,9 +61,9 @@ Now that the hard part is done, start the Metron deployment process.
```
vagrant provision
```
-
+
In addition to re-running the entire provisioning play book, you may now re-run an individual Ansible tag or a collection of tags in the following ways.
-
+
```
./run_ansible_role.sh web
```
@@ -72,22 +72,22 @@ Now that the hard part is done, start the Metron deployment process.
vagrant --ansible-tags="web" provision
```
Will re-run the web role on the Vagrant image. This will re-install (if necessary) and start the UI.
-
+
A collection of tags is specified as a comma separated list.
-
+
```
./run_ansbile_role.sh "sensors,enrichment"
-
+
```
-
+
Tags are listed in the playbooks, some frequently used tags:
+ hdp-install - Install HDP
+ hdp-deploy - Deploy and Start HDP Services (will start all Hadoop Services)
+ sensors - Deploy and Start Sensors.
+ enrichment - Deploy and Start Enrichment Topology.
-
+
Note: there is a convienence script, ```./run_enrichment_role.sh```, which runs the enrichment tag.
-
+
### Explore Metron
Navigate to the following resources to explore your newly minted Apache Metron environment.
[2/2] incubator-metron git commit: METRON-132 Monit integration +
METRON-85 Removed fixed wait from EC2 deployment (nickwallen) closes
apache/incubator-metron#118
Posted by ni...@apache.org.
METRON-132 Monit integration + METRON-85 Removed fixed wait from EC2 deployment (nickwallen) closes apache/incubator-metron#118
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/9898b4c6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/9898b4c6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/9898b4c6
Branch: refs/heads/master
Commit: 9898b4c6a3fb06a29ef28dbc75b074b28a89c980
Parents: 9a4481c
Author: nickwallen <ni...@nickallen.org>
Authored: Fri May 13 17:28:25 2016 -0400
Committer: Nick Allen <ni...@nickallen.org>
Committed: Fri May 13 17:28:25 2016 -0400
----------------------------------------------------------------------
metron-deployment/amazon-ec2/conf/defaults.yml | 32 ++++-
metron-deployment/amazon-ec2/run.sh | 8 +-
.../amazon-ec2/tasks/check-hosts.yml | 2 +-
.../amazon-ec2/tasks/provisioning-report.yml | 7 +-
.../inventory/devimage-vagrant/group_vars/all | 87 -------------
.../inventory/full-dev-platform/group_vars/all | 35 +++--
.../inventory/metron_example/group_vars/all | 4 +-
metron-deployment/playbooks/metron_install.yml | 14 +-
metron-deployment/roles/bro/tasks/bro.yml | 12 ++
metron-deployment/roles/bro/tasks/main.yml | 2 +-
metron-deployment/roles/bro/tasks/nic.yml | 19 +++
metron-deployment/roles/bro/tasks/start-bro.yml | 31 -----
.../elasticsearch/tasks/configure_index.yml | 44 -------
.../elasticsearch/tasks/create-indices.yml | 48 +++++++
.../roles/elasticsearch/tasks/elasticsearch.yml | 61 +++++++++
.../roles/elasticsearch/tasks/main.yml | 52 +-------
.../roles/fastcapa/tasks/fastcapa.yml | 3 -
.../metron_pcapservice/tasks/pcapservice.yml | 3 -
.../roles/metron_streaming/defaults/main.yml | 4 -
.../metron_streaming/tasks/copy_bundles.yml | 80 ++++++++++++
.../roles/metron_streaming/tasks/main.yml | 127 +------------------
.../metron_streaming/tasks/metron_topology.yml | 29 -----
.../roles/metron_streaming/tasks/topologies.yml | 87 +++++++++++++
.../roles/metron_ui/tasks/metron-ui.yml | 6 +-
.../roles/monit-start/defaults/main.yml | 23 ++++
.../roles/monit-start/tasks/main.yml | 50 ++++++++
metron-deployment/roles/monit/defaults/main.yml | 26 ++++
metron-deployment/roles/monit/tasks/main.yml | 21 +++
.../roles/monit/tasks/monit-definitions.yml | 61 +++++++++
metron-deployment/roles/monit/tasks/monit.yml | 27 ++++
metron-deployment/roles/monit/tasks/scripts.yml | 53 ++++++++
.../roles/monit/templates/monit/bro.monit | 25 ++++
.../monit/templates/monit/elasticsearch.monit | 23 ++++
.../monit/enrichment-elasticsearch.monit | 26 ++++
.../monit/templates/monit/enrichment-solr.monit | 26 ++++
.../roles/monit/templates/monit/metron-ui.monit | 22 ++++
.../roles/monit/templates/monit/monit.conf | 29 +++++
.../roles/monit/templates/monit/mysql.monit | 21 +++
.../roles/monit/templates/monit/parsers.monit | 49 +++++++
.../monit/templates/monit/pcap-replay.monit | 24 ++++
.../monit/templates/monit/pcap-service.monit | 22 ++++
.../roles/monit/templates/monit/pycapa.monit | 24 ++++
.../roles/monit/templates/monit/snort.monit | 33 +++++
.../roles/monit/templates/monit/yaf.monit | 24 ++++
.../scripts/start_enrichment_topology.sh | 25 ++++
.../monit/templates/scripts/start_topology.sh | 25 ++++
.../monit/templates/scripts/status_topology.sh | 33 +++++
.../monit/templates/scripts/stop_topology.sh | 23 ++++
.../roles/mysql_server/tasks/geoip.yml | 37 ++++++
.../roles/mysql_server/tasks/main.yml | 79 +-----------
.../roles/mysql_server/tasks/mysql-password.yml | 33 +++++
.../roles/mysql_server/tasks/mysql.yml | 60 +++++++++
.../roles/pcap_replay/defaults/main.yml | 2 +-
.../roles/pcap_replay/tasks/service.yml | 2 +-
.../roles/pcap_replay/templates/pcap-replay | 10 +-
.../roles/pycapa/tasks/pycapa-service.yml | 3 -
.../roles/sensor-test-mode/tasks/main.yml | 4 +-
.../roles/sensor-test-mode/tasks/pcap.yml | 5 +-
.../roles/sensor-test-mode/tasks/snort.yml | 4 -
.../roles/sensor-test-mode/tasks/yaf.yml | 7 +-
metron-deployment/roles/snort/tasks/main.yml | 12 +-
metron-deployment/roles/snort/tasks/nic.yml | 19 +++
metron-deployment/roles/yaf/tasks/main.yml | 1 +
metron-deployment/roles/yaf/tasks/nic.yml | 19 +++
metron-deployment/roles/yaf/tasks/yaf.yml | 6 -
.../vagrant/fastcapa-test-platform/playbook.yml | 3 +
.../vagrant/full-dev-platform/README.md | 16 +--
67 files changed, 1301 insertions(+), 533 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/amazon-ec2/conf/defaults.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/amazon-ec2/conf/defaults.yml b/metron-deployment/amazon-ec2/conf/defaults.yml
index 76c4b98..a505588 100644
--- a/metron-deployment/amazon-ec2/conf/defaults.yml
+++ b/metron-deployment/amazon-ec2/conf/defaults.yml
@@ -15,6 +15,22 @@
# limitations under the License.
#
---
+# which services should be started?
+services_to_start:
+ - mysql
+ - elasticsearch
+ - pcap-service
+ - metron-ui
+ - yaf
+ - snort
+ - snort-logs
+ - bro
+ - pcap-replay
+ - yaf-parser
+ - bro-parser
+ - snort-parser
+ - enrichment
+
# ec2
env: metron-test
region: us-west-2
@@ -45,17 +61,21 @@ retention_in_gb: 25
# metron variables
metron_version: 0.1BETA
+metron_directory: /usr/metron/{{ metron_version }}
java_home: /usr/jdk64/jdk1.8.0_40
pcapservice_port: 8081
# sensors
+install_snort: True
+install_pycapa: True
+install_bro: True
+install_yaf: True
+install_pcap_replay: True
+install_tap: True
sensor_test_mode: True
-sniff_interface: tap0
-snort_alert_csv_path: "/var/log/snort/alert.csv"
-pcap_replay: True
pcap_replay_interface: tap0
-install_tap: True
-pcap_path: /opt/pcap-replay
+pcap_replay_home: /opt/pcap-replay
+sniff_interface: tap0
# data directories
zookeeper_data_dir: "/data1/hadoop/zookeeper"
@@ -72,7 +92,7 @@ storm_local_dir: "/data1/hadoop/storm"
kafka_log_dirs: "/data2/kafka-log"
elasticsearch_data_dir: "/data1/elasticsearch,/data2/elasticsearch"
-#Search
+# search
install_elasticsearch: True
install_solr: False
elasticsearch_transport_port: 9300
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/amazon-ec2/run.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/amazon-ec2/run.sh b/metron-deployment/amazon-ec2/run.sh
index dd7f86a..41c5b52 100755
--- a/metron-deployment/amazon-ec2/run.sh
+++ b/metron-deployment/amazon-ec2/run.sh
@@ -20,10 +20,9 @@
# Builds Metron platform jars, instantiates hosts, and deploys Metron to those
# hosts on Amazon EC2
#
-
LOGFILE="./ansible.log"
DEPLOYDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-EXTRA_VARS="-v"
+EXTRA_ARGS="-v ${@:1}"
NOW=`date`
DEFAULT_ENV="metron-test"
DEFAULT_ENV_FILE="./.metron-env"
@@ -56,8 +55,7 @@ echo "$ENV" > $DEFAULT_ENV_FILE
# log information about the host platform
echo "=============================================================" >> $LOGFILE
-echo "Launching Metron @ $NOW"... >> $LOGFILE
-echo "Metron Environment: $ENV" >> $LOGFILE
+echo "Launching Metron[$ENV] @ $NOW"... >> $LOGFILE
$DEPLOYDIR/../scripts/platform-info.sh >> $LOGFILE
# build metron
@@ -67,4 +65,4 @@ mvn package -DskipTests
# deploy metron
cd $DEPLOYDIR
export EC2_INI_PATH=conf/ec2.ini
-ansible-playbook -i ec2.py playbook.yml --extra-vars="env=$ENV" $EXTRA_VARS
+ansible-playbook -i ec2.py playbook.yml --extra-vars="env=$ENV" $EXTRA_ARGS
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/amazon-ec2/tasks/check-hosts.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/amazon-ec2/tasks/check-hosts.yml b/metron-deployment/amazon-ec2/tasks/check-hosts.yml
index 1a4b2c7..a20a319 100644
--- a/metron-deployment/amazon-ec2/tasks/check-hosts.yml
+++ b/metron-deployment/amazon-ec2/tasks/check-hosts.yml
@@ -16,5 +16,5 @@
#
---
- name: Wait for connectivity to host(s)
- local_action: wait_for host={{ inventory_hostname }} state=started timeout=300 delay=10
+ local_action: wait_for host={{ inventory_hostname }} state=started port=22 timeout=300 delay=10
become: False
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/amazon-ec2/tasks/provisioning-report.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/amazon-ec2/tasks/provisioning-report.yml b/metron-deployment/amazon-ec2/tasks/provisioning-report.yml
index d2abec0..f22d0a9 100644
--- a/metron-deployment/amazon-ec2/tasks/provisioning-report.yml
+++ b/metron-deployment/amazon-ec2/tasks/provisioning-report.yml
@@ -27,9 +27,10 @@
- set_fact:
Success:
- "Apache Metron deployed successfully"
- - " Metron @ http://{{ groups.web[0] }}:5000"
- - " Ambari @ http://{{ groups.ambari_master[0] }}:{{ ambari_port }}"
- - " Sensors @ {{ groups.sensors[0] }} on {{ sniff_interface }}"
+ - " Metron @ http://{{ groups.web[0] }}:5000"
+ - " Ambari @ http://{{ groups.ambari_master[0] }}:{{ ambari_port }}"
+ - " Sensor Status @ http://{{ groups.sensors[0] }}:2812"
+ - " Topology Status @ http://{{ groups.enrichment[0] }}:2812"
- For additional information, see https://metron.incubator.apache.org/'
- debug: var=Success
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/inventory/devimage-vagrant/group_vars/all
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/devimage-vagrant/group_vars/all b/metron-deployment/inventory/devimage-vagrant/group_vars/all
deleted file mode 100644
index ac83692..0000000
--- a/metron-deployment/inventory/devimage-vagrant/group_vars/all
+++ /dev/null
@@ -1,87 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-#Ambari variables
-ambari_host: "{{ groups.ambari_master[0] }}"
-hdp_host_group: "{{ groups.ambari_slave }}"
-ambari_port: 8080
-ambari_user: admin
-ambari_password: admin
-cluster_type: single_node_vm
-
-# hbase
-pcap_hbase_table: pcap
-tracker_hbase_table: access_tracker
-threatintel_hbase_table: threatintel
-enrichment_hbase_table: enrichment
-
-# metron variables
-metron_version: 0.1BETA
-metron_directory: /usr/metron/{{ metron_version }}
-java_home: /usr/jdk64/jdk1.8.0_40
-bro_version: "2.4.1"
-fixbuf_version: "1.7.1"
-yaf_version: "2.8.0"
-daq_version: "2.0.6-1"
-pycapa_repo: "https://github.com/OpenSOC/pycapa.git"
-pycapa_home: "/opt/pycapa"
-snort_version: "2.9.8.0-1"
-snort_alert_csv_path: "/var/log/snort/alert.csv"
-
-#data directories - only required to override defaults
-zookeeper_data_dir: "/data1/hadoop/zookeeper"
-namenode_checkpoint_dir: "/data1/hadoop/hdfs/namesecondary"
-namenode_name_dir: "/data1/hadoop/hdfs/namenode"
-datanode_data_dir: "/data1/hadoop/hdfs/data,/data2/hadoop/hdfs/data"
-journalnode_edits_dir: "/data1/hadoop/hdfs/journalnode"
-nodemanager_local_dirs: "/data1/hadoop/yarn/local"
-timeline_ldb_store_path: "/data1/hadoop/yarn/timeline"
-timeline_ldb_state_path: "/data1/hadoop/yarn/timeline"
-nodemanager_log_dirs: "/data1/hadoop/yarn/log"
-jhs_recovery_store_ldb_path: "/data1/hadoop/mapreduce/jhs"
-storm_local_dir: "/data1/hadoop/storm"
-kafka_log_dirs: "/data1/kafka-log"
-elasticsearch_data_dir: "/data1/elasticsearch,/data2/elasticsearch"
-
-ambari_server_mem: 512
-threat_intel_bulk_load: False
-
-#Sensors
-install_pycapa: False
-install_bro: False
-install_snort: False
-install_yaf: False
-pcap_replay: True
-sniff_interface: eth1
-pcap_replay_interface: "{{ sniff_interface }}"
-storm_topologies:
- - "{{ metron_directory }}/config/topologies/bro/remote.yaml"
- - "{{ metron_directory }}/config/topologies/snort/remote.yaml"
- - "{{ metron_directory }}/config/topologies/yaf/remote.yaml"
- - "{{ metron_directory }}/config/topologies/enrichment/remote.yaml"
-pcapservice_port: 8081
-
-#Search
-install_elasticsearch: True
-install_solr: False
-solr_collection_name: Metron
-solr_number_shards: 1
-solr_replication_factor: 1
-elasticsearch_transport_port: 9300
-elasticsearch_network_interface: eth1
-elasticsearch_web_port: 9200
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/inventory/full-dev-platform/group_vars/all
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/full-dev-platform/group_vars/all b/metron-deployment/inventory/full-dev-platform/group_vars/all
index decd7f3..2d4dc7e 100644
--- a/metron-deployment/inventory/full-dev-platform/group_vars/all
+++ b/metron-deployment/inventory/full-dev-platform/group_vars/all
@@ -15,13 +15,30 @@
# limitations under the License.
#
-#Ambari variables
+# which services should be started?
+services_to_start:
+ - mysql
+ - elasticsearch
+ - pcap-service
+ - metron-ui
+ - yaf
+ - snort
+ - snort-logs
+ - bro
+ - pcap-replay
+ - yaf-parser
+ - bro-parser
+ - snort-parser
+ - enrichment
+
+# ambari
ambari_host: "{{ groups.ambari_master[0] }}"
hdp_host_group: "{{ groups.ambari_slave }}"
ambari_port: 8080
ambari_user: admin
ambari_password: admin
cluster_type: single_node_vm
+ambari_server_mem: 512
# hbase
pcap_hbase_table: pcap
@@ -29,7 +46,7 @@ tracker_hbase_table: access_tracker
threatintel_hbase_table: threatintel
enrichment_hbase_table: enrichment
-# metron variables
+# metron
metron_version: 0.1BETA
metron_directory: /usr/metron/{{ metron_version }}
java_home: /usr/jdk64/jdk1.8.0_40
@@ -41,8 +58,9 @@ pycapa_repo: "https://github.com/OpenSOC/pycapa.git"
pycapa_home: "/opt/pycapa"
snort_version: "2.9.8.0-1"
snort_alert_csv_path: "/var/log/snort/alert.csv"
+threat_intel_bulk_load: False
-#data directories - only required to override defaults
+# data directories - only required to override defaults
zookeeper_data_dir: "/data1/hadoop/zookeeper"
namenode_checkpoint_dir: "/data1/hadoop/hdfs/namesecondary"
namenode_name_dir: "/data1/hadoop/hdfs/namenode"
@@ -57,25 +75,18 @@ storm_local_dir: "/data1/hadoop/storm"
kafka_log_dirs: "/data1/kafka-log"
elasticsearch_data_dir: "/data1/elasticsearch,/data2/elasticsearch"
-ambari_server_mem: 512
-threat_intel_bulk_load: False
-
# sensors
sensor_test_mode: True
install_pycapa: False
install_bro: True
install_snort: True
install_yaf: True
-pcap_replay: True
+install_pcap_replay: True
sniff_interface: eth1
pcap_replay_interface: "{{ sniff_interface }}"
-storm_parser_topologies:
- - "bro"
- - "snort"
- - "yaf"
pcapservice_port: 8081
-#Search
+# search
install_elasticsearch: True
install_solr: False
solr_collection_name: Metron
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/inventory/metron_example/group_vars/all
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/metron_example/group_vars/all b/metron-deployment/inventory/metron_example/group_vars/all
index 097516d..efa0a72 100644
--- a/metron-deployment/inventory/metron_example/group_vars/all
+++ b/metron-deployment/inventory/metron_example/group_vars/all
@@ -50,8 +50,8 @@ pycapa_home: "/opt/pycapa"
snort_version: "2.9.8.0-1"
snort_alert_csv_path: "/var/log/snort/alert.csv"
-#PCAP Replay
-pcap_replay: True
+# pcap-replay
+install_pcap_replay: True
pcap_replay_interface: eth1
#data directories - only required to override defaults
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/playbooks/metron_install.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/playbooks/metron_install.yml b/metron-deployment/playbooks/metron_install.yml
index 877f9b8..0118213 100644
--- a/metron-deployment/playbooks/metron_install.yml
+++ b/metron-deployment/playbooks/metron_install.yml
@@ -26,6 +26,7 @@
become: true
roles:
- role: metron_common
+ - role: monit
tags:
- metron-prereqs
@@ -73,10 +74,10 @@
- { role: tap_interface, when: install_tap | default(False) == True }
- { role: pycapa, when: install_pycapa | default(True) == True }
- { role: bro, when: install_bro | default(True) == True }
- - { role: flume, when: install_snort | default(True) == True }
- - { role: snort , when: install_snort | default(True) == True }
+ - { role: flume, when: install_snort | default(True) == True }
+ - { role: snort, when: install_snort | default(True) == True }
- { role: yaf, when: install_yaf | default(True) == True }
- - { role: pcap_replay , when: (pcap_replay | default(False)) or (sensor_test_mode | default(False)) == True }
+ - { role: pcap_replay, when: install_pcap_replay | default(False) == True }
- { role: sensor-test-mode, when: sensor_test_mode | default(False) == True }
tags:
- sensors
@@ -101,3 +102,10 @@
- { role: metron_ui, when: install_elasticsearch | default(True) == True }
tags:
- web
+
+- hosts: metron
+ become: true
+ roles:
+ - monit-start
+ tags:
+ - start
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/bro/tasks/bro.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/bro/tasks/bro.yml b/metron-deployment/roles/bro/tasks/bro.yml
index fb27ef9..f6ef837 100644
--- a/metron-deployment/roles/bro/tasks/bro.yml
+++ b/metron-deployment/roles/bro/tasks/bro.yml
@@ -42,3 +42,15 @@
dest: "{{ bro_home }}/etc/node.cfg"
regexp: '^interface=.*$'
line: 'interface={{ sniff_interface }}'
+
+- name: Install bro configuration
+ shell: "{{ bro_home }}/bin/broctl install"
+
+- name: Bro Cronjob
+ cron:
+ name: Bro Cron
+ minute: "{{ bro_crontab_minutes }}"
+ job: "{{ item }}"
+ with_items:
+ - "{{ bro_crontab_job }}"
+ - "{{ bro_clean_job }}"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/bro/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/bro/tasks/main.yml b/metron-deployment/roles/bro/tasks/main.yml
index 14426d6..0e67063 100644
--- a/metron-deployment/roles/bro/tasks/main.yml
+++ b/metron-deployment/roles/bro/tasks/main.yml
@@ -19,4 +19,4 @@
- include: librdkafka.yml
- include: bro.yml
- include: bro-plugin-kafka.yml
-- include: start-bro.yml
+- include: nic.yml
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/bro/tasks/nic.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/bro/tasks/nic.yml b/metron-deployment/roles/bro/tasks/nic.yml
new file mode 100644
index 0000000..6053618
--- /dev/null
+++ b/metron-deployment/roles/bro/tasks/nic.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Turn on promiscuous mode for {{ sniff_interface }}
+ shell: "ip link set {{ sniff_interface }} promisc on"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/bro/tasks/start-bro.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/bro/tasks/start-bro.yml b/metron-deployment/roles/bro/tasks/start-bro.yml
deleted file mode 100644
index 1a0b938..0000000
--- a/metron-deployment/roles/bro/tasks/start-bro.yml
+++ /dev/null
@@ -1,31 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Turn on promiscuous mode for {{ sniff_interface }}
- shell: "ip link set {{ sniff_interface }} promisc on"
-
-- name: Start bro
- shell: "{{ bro_home }}/bin/broctl deploy"
-
-- name: Bro Cronjob
- cron:
- name: Bro Cron
- minute: "{{ bro_crontab_minutes }}"
- job: "{{ item }}"
- with_items:
- - "{{ bro_crontab_job }}"
- - "{{ bro_clean_job }}"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/elasticsearch/tasks/configure_index.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/elasticsearch/tasks/configure_index.yml b/metron-deployment/roles/elasticsearch/tasks/configure_index.yml
deleted file mode 100644
index 09739be..0000000
--- a/metron-deployment/roles/elasticsearch/tasks/configure_index.yml
+++ /dev/null
@@ -1,44 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name : Wait for Elasticsearch Host to Start
- wait_for:
- host: "{{ groups.search[0] }}"
- port: "{{ elasticsearch_web_port }}"
- delay: 10
- timeout: 300
-
-- name: Wait for Green Index Status
- uri:
- url: "http://{{ groups.search[0] }}:{{ elasticsearch_web_port }}/_cat/health"
- method: GET
- status_code: 200
- return_content: yes
- register: result
- until: result.content.find("green") != -1
- retries: 10
- delay: 60
- run_once: yes
-
-- name: Add Elasticsearch templates for topologies
- uri:
- url: "http://{{ groups.search[0] }}:{{ elasticsearch_web_port }}/_template/template_yaf"
- method: POST
- body: "{{ lookup('file','yaf_index.template') }}"
- status_code: 200
- body_format: json
- run_once: yes
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/elasticsearch/tasks/create-indices.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/elasticsearch/tasks/create-indices.yml b/metron-deployment/roles/elasticsearch/tasks/create-indices.yml
new file mode 100644
index 0000000..d91d220
--- /dev/null
+++ b/metron-deployment/roles/elasticsearch/tasks/create-indices.yml
@@ -0,0 +1,48 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Start Elasticsearch
+ service: name=elasticsearch state=started
+
+- name : Wait for Elasticsearch Host to Start
+ wait_for:
+ host: "{{ groups.search[0] }}"
+ port: "{{ elasticsearch_web_port }}"
+ delay: 10
+ timeout: 300
+
+- name: Wait for Green Index Status
+ uri:
+ url: "http://{{ groups.search[0] }}:{{ elasticsearch_web_port }}/_cat/health"
+ method: GET
+ status_code: 200
+ return_content: yes
+ register: result
+ until: result.content.find("green") != -1
+ retries: 10
+ delay: 60
+
+- name: Add Elasticsearch templates for topologies
+ uri:
+ url: "http://{{ groups.search[0] }}:{{ elasticsearch_web_port }}/_template/template_yaf"
+ method: POST
+ body: "{{ lookup('file','yaf_index.template') }}"
+ status_code: 200
+ body_format: json
+
+- name: Stop Elasticsearch
+ service: name=elasticsearch state=stopped
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/elasticsearch/tasks/elasticsearch.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/elasticsearch/tasks/elasticsearch.yml b/metron-deployment/roles/elasticsearch/tasks/elasticsearch.yml
new file mode 100644
index 0000000..41b7f37
--- /dev/null
+++ b/metron-deployment/roles/elasticsearch/tasks/elasticsearch.yml
@@ -0,0 +1,61 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Add Elasticsearch GPG key
+ rpm_key:
+ key: https://packages.elastic.co/GPG-KEY-elasticsearch
+ state: present
+
+- name: Add Elasticsearch repository
+ copy:
+ src: elasticsearch.repo
+ dest: /etc/yum.repos.d/elasticsearch.repo
+ mode: 0644
+
+- name: Install Elasticsearch
+ yum:
+ name: elasticsearch
+ state: installed
+ register: result
+ until: result.rc == 0
+ retries: 5
+ delay: 10
+
+- name: Create Data Directories
+ file:
+ path: "{{ item }}"
+ state: directory
+ mode: 0755
+ owner: elasticsearch
+ group: elasticsearch
+ when: elasticsearch_data_dir is defined
+ with_items:
+ - '{{ elasticsearch_data_dir.split(",") }}'
+
+- name: Configure Elasticsearch
+ lineinfile: >
+ dest=/etc/elasticsearch/elasticsearch.yml
+ regexp="{{ item.regexp }}"
+ line="{{ item.line }}"
+ state=present
+ with_items:
+ - { regexp: '#cluster\.name', line: 'cluster.name: metron' }
+ - { regexp: '#network\.host:', line: 'network.host: _{{
+ elasticsearch_network_interface }}:ipv4_' }
+ - { regexp: '#discovery\.zen\.ping\.unicast\.hosts',
+ line: 'discovery.zen.ping.unicast.hosts: [ {{ es_hosts }} ]'}
+ - { regexp: '#path\.data', line: 'path.data: {{ elasticsearch_data_dir }}' }
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/elasticsearch/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/elasticsearch/tasks/main.yml b/metron-deployment/roles/elasticsearch/tasks/main.yml
index 26554aa..71fb0b5 100644
--- a/metron-deployment/roles/elasticsearch/tasks/main.yml
+++ b/metron-deployment/roles/elasticsearch/tasks/main.yml
@@ -15,59 +15,13 @@
# limitations under the License.
#
---
-- name: Add Elasticsearch GPG key.
- rpm_key:
- key: https://packages.elastic.co/GPG-KEY-elasticsearch
- state: present
+- include: elasticsearch.yml
-- name: Add Elasticsearch repository.
- copy:
- src: elasticsearch.repo
- dest: /etc/yum.repos.d/elasticsearch.repo
- mode: 0644
-
-- name: Install Elasticsearch.
- yum:
- name: elasticsearch
- state: installed
- register: result
- until: result.rc == 0
- retries: 5
- delay: 10
-
-- name: Create Data Directories
- file:
- path: "{{ item }}"
- state: directory
- mode: 0755
- owner: elasticsearch
- group: elasticsearch
- when: elasticsearch_data_dir is defined
- with_items:
- - '{{ elasticsearch_data_dir.split(",") }}'
-
-- name: Configure Elasticsearch.
- lineinfile: >
- dest=/etc/elasticsearch/elasticsearch.yml
- regexp="{{ item.regexp }}"
- line="{{ item.line }}"
- state=present
- with_items:
- - { regexp: '#cluster\.name', line: 'cluster.name: metron' }
- - { regexp: '#network\.host:', line: 'network.host: _{{
- elasticsearch_network_interface }}:ipv4_' }
- - { regexp: '#discovery\.zen\.ping\.unicast\.hosts',
- line: 'discovery.zen.ping.unicast.hosts: [ {{ es_hosts }} ]'}
- - { regexp: '#path\.data', line: 'path.data: {{ elasticsearch_data_dir }}' }
-
-- name: Start Elasticsearch.
- service: name=elasticsearch state=started enabled=yes
-
-- include: configure_index.yml
+- include: create-indices.yml
+ run_once: yes
- name: Create Logrotate Script for Elasticsearch
template:
src: "metron-elasticsearch-logrotate.yml"
dest: "/etc/logrotate.d/metron-elasticsearch"
mode: 0644
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/fastcapa/tasks/fastcapa.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/fastcapa/tasks/fastcapa.yml b/metron-deployment/roles/fastcapa/tasks/fastcapa.yml
index 4e366c4..cad5b68 100644
--- a/metron-deployment/roles/fastcapa/tasks/fastcapa.yml
+++ b/metron-deployment/roles/fastcapa/tasks/fastcapa.yml
@@ -44,6 +44,3 @@
- name: Register the service with systemd
shell: systemctl enable fastcapa
when: ansible_distribution == "CentOS" and ansible_distribution_major_version == "7"
-
-- name: Run fastcapa
- service: name=fastcapa state=restarted
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/metron_pcapservice/tasks/pcapservice.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_pcapservice/tasks/pcapservice.yml b/metron-deployment/roles/metron_pcapservice/tasks/pcapservice.yml
index 50a9e51..d0ac411 100644
--- a/metron-deployment/roles/metron_pcapservice/tasks/pcapservice.yml
+++ b/metron-deployment/roles/metron_pcapservice/tasks/pcapservice.yml
@@ -28,6 +28,3 @@
- name: Install service script
template: src=pcapservice dest=/etc/init.d/pcapservice mode=0755
-
-- name: Start pcapservice
- service: name=pcapservice state=restarted
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/metron_streaming/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/defaults/main.yml b/metron-deployment/roles/metron_streaming/defaults/main.yml
index b82c3a6..1af0939 100644
--- a/metron-deployment/roles/metron_streaming/defaults/main.yml
+++ b/metron-deployment/roles/metron_streaming/defaults/main.yml
@@ -59,10 +59,6 @@ bro_topic: bro
yaf_topic: yaf
snort_topic: snort
enrichments_topic: enrichments
-storm_parser_topologies:
- - "bro"
- - "snort"
- - "yaf"
hdfs_retention_days: 30
hdfs_bro_purge_cronjob: "{{ metron_directory }}/bin/prune_hdfs_files.sh -f {{ hdfs_url }} -g '/apps/metron/enrichment/indexed/bro_doc/*enrichment-*' -s $(date -d '{{ hdfs_retention_days }} days ago' +%m/%d/%Y) -n 1 >> /var/log/bro-purge/cron-hdfs-bro-purge.log 2>&1"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/metron_streaming/tasks/copy_bundles.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/tasks/copy_bundles.yml b/metron-deployment/roles/metron_streaming/tasks/copy_bundles.yml
new file mode 100644
index 0000000..97fb259
--- /dev/null
+++ b/metron-deployment/roles/metron_streaming/tasks/copy_bundles.yml
@@ -0,0 +1,80 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Copy Metron Solr bundle
+ copy:
+ src: "{{ metron_solr_bundle_path }}"
+ dest: "{{ metron_directory }}"
+
+- name: Copy Metron Elasticsearch bundle
+ copy:
+ src: "{{ metron_elasticsearch_bundle_path }}"
+ dest: "{{ metron_directory }}"
+
+- name: Copy Metron Enrichment bundle
+ copy:
+ src: "{{ metron_enrichment_bundle_path }}"
+ dest: "{{ metron_directory }}"
+
+- name: Copy Metron Parsers bundle
+ copy:
+ src: "{{ metron_parsers_bundle_path }}"
+ dest: "{{ metron_directory }}"
+
+- name: Copy Metron DataLoads bundle
+ copy:
+ src: "{{ metron_data_management_bundle_path }}"
+ dest: "{{ metron_directory }}"
+
+- name: Copy Metron Common bundle
+ copy:
+ src: "{{ metron_common_bundle_path }}"
+ dest: "{{ metron_directory }}"
+
+- name: Copy Metron Pcap bundle
+ copy:
+ src: "{{ metron_pcap_bundle_path }}"
+ dest: "{{ metron_directory }}"
+
+- name: Unbundle Metron bundles
+ shell: "{{ item }}"
+ args:
+ chdir: "{{ metron_directory }}"
+ with_items:
+ - tar xzvf metron-solr*.tar.gz
+ - tar xzvf metron-elasticsearch*.tar.gz
+ - tar xzvf metron-enrichment*.tar.gz
+ - tar xzvf metron-parsers*.tar.gz
+ - tar xzvf metron-data-management*.tar.gz
+ - tar xzvf metron-common*.tar.gz
+ - tar xzvf metron-pcap-backend*.tar.gz
+ - rm *.tar.gz
+
+- name: Add *-site.xml files to topology jars
+ shell: "cd {{ item.config_path }} && jar -uf {{ metron_directory }}/lib/{{ item.jar_name }} {{ item.file_name }}"
+ with_items:
+ - { config_path: "{{ hbase_config_path }}", jar_name: "{{ metron_solr_jar_name }}", file_name: "hbase-site.xml" }
+ - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_solr_jar_name }}", file_name: "core-site.xml" }
+ - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_solr_jar_name }}", file_name: "hdfs-site.xml" }
+ - { config_path: "{{ hbase_config_path }}", jar_name: "{{ metron_elasticsearch_jar_name }}", file_name: "hbase-site.xml" }
+ - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_elasticsearch_jar_name }}", file_name: "core-site.xml" }
+ - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_elasticsearch_jar_name }}", file_name: "hdfs-site.xml" }
+ - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_pcap_jar_name }}", file_name: "core-site.xml" }
+ - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_pcap_jar_name }}", file_name: "hdfs-site.xml" }
+ - { config_path: "{{ hbase_config_path }}", jar_name: "{{ metron_parsers_jar_name }}", file_name: "hbase-site.xml" }
+ - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_parsers_jar_name }}", file_name: "core-site.xml" }
+ - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_parsers_jar_name }}", file_name: "hdfs-site.xml" }
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/metron_streaming/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/tasks/main.yml b/metron-deployment/roles/metron_streaming/tasks/main.yml
index 6d5a9fb..d44f2ef 100644
--- a/metron-deployment/roles/metron_streaming/tasks/main.yml
+++ b/metron-deployment/roles/metron_streaming/tasks/main.yml
@@ -22,59 +22,7 @@
- { name: 'bin'}
- { name: 'config'}
-
-- name: Copy Metron Solr bundle
- copy:
- src: "{{ metron_solr_bundle_path }}"
- dest: "{{ metron_directory }}"
-
-- name: Copy Metron Elasticsearch bundle
- copy:
- src: "{{ metron_elasticsearch_bundle_path }}"
- dest: "{{ metron_directory }}"
-
-- name: Copy Metron Enrichment bundle
- copy:
- src: "{{ metron_enrichment_bundle_path }}"
- dest: "{{ metron_directory }}"
-
-- name: Copy Metron Parsers bundle
- copy:
- src: "{{ metron_parsers_bundle_path }}"
- dest: "{{ metron_directory }}"
-
-- name: Copy Metron DataLoads bundle
- copy:
- src: "{{ metron_data_management_bundle_path }}"
- dest: "{{ metron_directory }}"
-
-- name: Copy Metron Common bundle
- copy:
- src: "{{ metron_common_bundle_path }}"
- dest: "{{ metron_directory }}"
-
-- name: Copy Metron Pcap bundle
- copy:
- src: "{{ metron_pcap_bundle_path }}"
- dest: "{{ metron_directory }}"
-
-- name: Unbundle Metron bundles
- shell: cd {{ metron_directory }} && tar xzvf metron-solr*.tar.gz && tar xzvf metron-elasticsearch*.tar.gz && tar xzvf metron-enrichment*.tar.gz && tar xzvf metron-parsers*.tar.gz && tar xzvf metron-data-management*.tar.gz && tar xzvf metron-common*.tar.gz && tar xzvf metron-pcap-backend*.tar.gz && rm *.tar.gz
-
-- name: Add *-site.xml files to topology jars
- shell: cd {{ item.config_path }} && jar -uf {{ metron_directory }}/lib/{{ item.jar_name }} {{ item.file_name }}
- with_items:
- - { config_path: "{{ hbase_config_path }}", jar_name: "{{ metron_solr_jar_name }}", file_name: "hbase-site.xml" }
- - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_solr_jar_name }}", file_name: "core-site.xml" }
- - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_solr_jar_name }}", file_name: "hdfs-site.xml" }
- - { config_path: "{{ hbase_config_path }}", jar_name: "{{ metron_elasticsearch_jar_name }}", file_name: "hbase-site.xml" }
- - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_elasticsearch_jar_name }}", file_name: "core-site.xml" }
- - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_elasticsearch_jar_name }}", file_name: "hdfs-site.xml" }
- - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_pcap_jar_name }}", file_name: "core-site.xml" }
- - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_pcap_jar_name }}", file_name: "hdfs-site.xml" }
- - { config_path: "{{ hbase_config_path }}", jar_name: "{{ metron_parsers_jar_name }}", file_name: "hbase-site.xml" }
- - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_parsers_jar_name }}", file_name: "core-site.xml" }
- - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_parsers_jar_name }}", file_name: "hdfs-site.xml" }
+- include: copy_bundles.yml
- name: Get Default mysql passowrd
include_vars: "../roles/mysql_server/defaults/main.yml"
@@ -86,76 +34,7 @@
- include: grok_upload.yml
run_once: true
-- name: Configure Metron Pcap Topology
- lineinfile: >
- dest={{ metron_pcap_properties_config_path }}
- regexp="{{ item.regexp }}"
- line="{{ item.line }}"
- with_items:
- - { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" }
- - { regexp: "kafka.pcap.out=", line: "kafka.pcap.out={{ pcap_hdfs_path }}" }
- - { regexp: "spout.kafka.topic.pcap=", line: "spout.kafka.topic.pcap={{ pycapa_topic }}" }
-
-- name: Configure Metron Parser Topologies
- lineinfile:
- dest: "{{ metron_parsers_properties_config_path }}"
- regexp: "{{ item.regexp }}"
- line: "{{ item.line }}"
- with_items:
- - { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" }
- - { regexp: "kafka.broker=", line: "kafka.broker={{ kafka_broker_url }}" }
-
-- name: Configure Metron Solr topology
- lineinfile: >
- dest={{ metron_solr_properties_config_path }}
- regexp="{{ item.regexp }}"
- line="{{ item.line }}"
- with_items:
- - { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" }
- - { regexp: "kafka.broker=", line: "kafka.broker={{ kafka_broker_url }}" }
- - { regexp: "es.ip=", line: "es.ip={{ groups.search[0] }}" }
- - { regexp: "es.port=", line: "es.port={{ elasticsearch_transport_port }}" }
- - { regexp: "es.clustername=", line: "es.clustername={{ elasticsearch_cluster_name }}" }
- - { regexp: "bolt.hdfs.file.system.url=", line: "bolt.hdfs.file.system.url={{ hdfs_url }}" }
- - { regexp: "spout.kafka.topic.bro=", line: "spout.kafka.topic.bro={{ bro_topic }}" }
- - { regexp: "threat.intel.tracker.table=", line: "threat.intel.tracker.table={{ tracker_hbase_table }}" }
- - { regexp: "threat.intel.tracker.cf=", line: "threat.intel.tracker.cf=t" }
- - { regexp: "threat.intel.simple.hbase.table=", line: "threat.intel.simple.hbase.table={{ threatintel_hbase_table }}" }
- - { regexp: "threat.intel.simple.hbase.cf=", line: "threat.intel.simple.hbase.cf=t" }
- - { regexp: "enrichment.simple.hbase.table=", line: "enrichment.simple.hbase.table={{ enrichment_hbase_table }}" }
- - { regexp: "enrichment.simple.hbase.cf=", line: "enrichment.simple.hbase.cf=t" }
- - { regexp: "mysql.ip=", line: "mysql.ip={{ groups.mysql[0] }}" }
- - { regexp: "mysql.password=", line: "mysql.password={{ mysql_root_password }}" }
- - { regexp: "index.hdfs.output=", line: "index.hdfs.output={{ metron_hdfs_output_dir }}/enrichment/indexed" }
- - { regexp: "bolt.hdfs.rotation.policy=", line: "bolt.hdfs.rotation.policy={{ metron_hdfs_rotation_policy }}" }
- - { regexp: "bolt.hdfs.rotation.policy.count=", line: "bolt.hdfs.rotation.policy.count={{ metron_hdfs_rotation_policy_count}}" }
- - { regexp: "bolt.hdfs.rotation.policy.units=", line: "bolt.hdfs.rotation.policy.units={{ metron_hdfs_rotation_policy_units }}" }
-
-- name: Configure Metron Elasticsearch topology
- lineinfile: >
- dest={{ metron_elasticsearch_properties_config_path }}
- regexp="{{ item.regexp }}"
- line="{{ item.line }}"
- with_items:
- - { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" }
- - { regexp: "kafka.broker=", line: "kafka.broker={{ kafka_broker_url }}" }
- - { regexp: "es.ip=", line: "es.ip={{ groups.search[0] }}" }
- - { regexp: "es.port=", line: "es.port={{ elasticsearch_transport_port }}" }
- - { regexp: "es.clustername=", line: "es.clustername={{ elasticsearch_cluster_name }}" }
- - { regexp: "bolt.hdfs.file.system.url=", line: "bolt.hdfs.file.system.url={{ hdfs_url }}" }
- - { regexp: "spout.kafka.topic.bro=", line: "spout.kafka.topic.bro={{ bro_topic }}" }
- - { regexp: "threat.intel.tracker.table=", line: "threat.intel.tracker.table={{ tracker_hbase_table }}" }
- - { regexp: "threat.intel.tracker.cf=", line: "threat.intel.tracker.cf=t" }
- - { regexp: "threat.intel.simple.hbase.table=", line: "threat.intel.simple.hbase.table={{ threatintel_hbase_table }}" }
- - { regexp: "threat.intel.simple.hbase.cf=", line: "threat.intel.simple.hbase.cf=t" }
- - { regexp: "enrichment.simple.hbase.table=", line: "enrichment.simple.hbase.table={{ enrichment_hbase_table }}" }
- - { regexp: "enrichment.simple.hbase.cf=", line: "enrichment.simple.hbase.cf=t" }
- - { regexp: "mysql.ip=", line: "mysql.ip={{ groups.mysql[0] }}" }
- - { regexp: "mysql.password=", line: "mysql.password={{ mysql_root_password }}" }
- - { regexp: "index.hdfs.output=", line: "index.hdfs.output={{ metron_hdfs_output_dir }}/enrichment/indexed" }
- - { regexp: "bolt.hdfs.rotation.policy=", line: "bolt.hdfs.rotation.policy={{ metron_hdfs_rotation_policy }}" }
- - { regexp: "bolt.hdfs.rotation.policy.count=", line: "bolt.hdfs.rotation.policy.count={{ metron_hdfs_rotation_policy_count}}" }
- - { regexp: "bolt.hdfs.rotation.policy.units=", line: "bolt.hdfs.rotation.policy.units={{ metron_hdfs_rotation_policy_units }}" }
+- include: topologies.yml
- include: source_config.yml
run_once: true
@@ -164,8 +43,6 @@
run_once: true
when: threat_intel_bulk_load == True
-- include: metron_topology.yml
-
- include: hdfs_purge.yml
- include: es_purge.yml
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/metron_streaming/tasks/metron_topology.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/tasks/metron_topology.yml b/metron-deployment/roles/metron_streaming/tasks/metron_topology.yml
deleted file mode 100644
index a4354d8..0000000
--- a/metron-deployment/roles/metron_streaming/tasks/metron_topology.yml
+++ /dev/null
@@ -1,29 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Submit Metron Parser topologies
- command: "{{ metron_directory }}/bin/start_parser_topology.sh {{ item }}"
- with_items:
- - "{{ storm_parser_topologies }}"
-
-- name: Submit Solr Metron Enrichment topology
- command: "{{ metron_directory }}/bin/start_solr_topology.sh"
- when: install_solr | default(False) == True
-
-- name: Submit Elasticsearch Metron Enrichment topology
- command: "{{ metron_directory }}/bin/start_elasticsearch_topology.sh"
- when: install_elasticsearch | default(False) == True
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/metron_streaming/tasks/topologies.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/tasks/topologies.yml b/metron-deployment/roles/metron_streaming/tasks/topologies.yml
new file mode 100644
index 0000000..d7fdb0c
--- /dev/null
+++ b/metron-deployment/roles/metron_streaming/tasks/topologies.yml
@@ -0,0 +1,87 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Configure Metron Pcap Topology
+ lineinfile: >
+ dest={{ metron_pcap_properties_config_path }}
+ regexp="{{ item.regexp }}"
+ line="{{ item.line }}"
+ with_items:
+ - { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" }
+ - { regexp: "kafka.pcap.out=", line: "kafka.pcap.out={{ pcap_hdfs_path }}" }
+ - { regexp: "spout.kafka.topic.pcap=", line: "spout.kafka.topic.pcap={{ pycapa_topic }}" }
+
+- name: Configure Metron Parser Topologies
+ lineinfile:
+ dest: "{{ metron_parsers_properties_config_path }}"
+ regexp: "{{ item.regexp }}"
+ line: "{{ item.line }}"
+ with_items:
+ - { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" }
+ - { regexp: "kafka.broker=", line: "kafka.broker={{ kafka_broker_url }}" }
+
+- name: Configure Metron Solr topology
+ lineinfile: >
+ dest={{ metron_solr_properties_config_path }}
+ regexp="{{ item.regexp }}"
+ line="{{ item.line }}"
+ with_items:
+ - { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" }
+ - { regexp: "kafka.broker=", line: "kafka.broker={{ kafka_broker_url }}" }
+ - { regexp: "es.ip=", line: "es.ip={{ groups.search[0] }}" }
+ - { regexp: "es.port=", line: "es.port={{ elasticsearch_transport_port }}" }
+ - { regexp: "es.clustername=", line: "es.clustername={{ elasticsearch_cluster_name }}" }
+ - { regexp: "bolt.hdfs.file.system.url=", line: "bolt.hdfs.file.system.url={{ hdfs_url }}" }
+ - { regexp: "spout.kafka.topic.bro=", line: "spout.kafka.topic.bro={{ bro_topic }}" }
+ - { regexp: "threat.intel.tracker.table=", line: "threat.intel.tracker.table={{ tracker_hbase_table }}" }
+ - { regexp: "threat.intel.tracker.cf=", line: "threat.intel.tracker.cf=t" }
+ - { regexp: "threat.intel.simple.hbase.table=", line: "threat.intel.simple.hbase.table={{ threatintel_hbase_table }}" }
+ - { regexp: "threat.intel.simple.hbase.cf=", line: "threat.intel.simple.hbase.cf=t" }
+ - { regexp: "enrichment.simple.hbase.table=", line: "enrichment.simple.hbase.table={{ enrichment_hbase_table }}" }
+ - { regexp: "enrichment.simple.hbase.cf=", line: "enrichment.simple.hbase.cf=t" }
+ - { regexp: "mysql.ip=", line: "mysql.ip={{ groups.mysql[0] }}" }
+ - { regexp: "mysql.password=", line: "mysql.password={{ mysql_root_password }}" }
+ - { regexp: "index.hdfs.output=", line: "index.hdfs.output={{ metron_hdfs_output_dir }}/enrichment/indexed" }
+ - { regexp: "bolt.hdfs.rotation.policy=", line: "bolt.hdfs.rotation.policy={{ metron_hdfs_rotation_policy }}" }
+ - { regexp: "bolt.hdfs.rotation.policy.count=", line: "bolt.hdfs.rotation.policy.count={{ metron_hdfs_rotation_policy_count}}" }
+ - { regexp: "bolt.hdfs.rotation.policy.units=", line: "bolt.hdfs.rotation.policy.units={{ metron_hdfs_rotation_policy_units }}" }
+
+- name: Configure Metron Elasticsearch topology
+ lineinfile: >
+ dest={{ metron_elasticsearch_properties_config_path }}
+ regexp="{{ item.regexp }}"
+ line="{{ item.line }}"
+ with_items:
+ - { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" }
+ - { regexp: "kafka.broker=", line: "kafka.broker={{ kafka_broker_url }}" }
+ - { regexp: "es.ip=", line: "es.ip={{ groups.search[0] }}" }
+ - { regexp: "es.port=", line: "es.port={{ elasticsearch_transport_port }}" }
+ - { regexp: "es.clustername=", line: "es.clustername={{ elasticsearch_cluster_name }}" }
+ - { regexp: "bolt.hdfs.file.system.url=", line: "bolt.hdfs.file.system.url={{ hdfs_url }}" }
+ - { regexp: "spout.kafka.topic.bro=", line: "spout.kafka.topic.bro={{ bro_topic }}" }
+ - { regexp: "threat.intel.tracker.table=", line: "threat.intel.tracker.table={{ tracker_hbase_table }}" }
+ - { regexp: "threat.intel.tracker.cf=", line: "threat.intel.tracker.cf=t" }
+ - { regexp: "threat.intel.simple.hbase.table=", line: "threat.intel.simple.hbase.table={{ threatintel_hbase_table }}" }
+ - { regexp: "threat.intel.simple.hbase.cf=", line: "threat.intel.simple.hbase.cf=t" }
+ - { regexp: "enrichment.simple.hbase.table=", line: "enrichment.simple.hbase.table={{ enrichment_hbase_table }}" }
+ - { regexp: "enrichment.simple.hbase.cf=", line: "enrichment.simple.hbase.cf=t" }
+ - { regexp: "mysql.ip=", line: "mysql.ip={{ groups.mysql[0] }}" }
+ - { regexp: "mysql.password=", line: "mysql.password={{ mysql_root_password }}" }
+ - { regexp: "index.hdfs.output=", line: "index.hdfs.output={{ metron_hdfs_output_dir }}/enrichment/indexed" }
+ - { regexp: "bolt.hdfs.rotation.policy=", line: "bolt.hdfs.rotation.policy={{ metron_hdfs_rotation_policy }}" }
+ - { regexp: "bolt.hdfs.rotation.policy.count=", line: "bolt.hdfs.rotation.policy.count={{ metron_hdfs_rotation_policy_count}}" }
+ - { regexp: "bolt.hdfs.rotation.policy.units=", line: "bolt.hdfs.rotation.policy.units={{ metron_hdfs_rotation_policy_units }}" }
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/metron_ui/tasks/metron-ui.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_ui/tasks/metron-ui.yml b/metron-deployment/roles/metron_ui/tasks/metron-ui.yml
index a1ac454..1e4d30a 100644
--- a/metron-deployment/roles/metron_ui/tasks/metron-ui.yml
+++ b/metron-deployment/roles/metron_ui/tasks/metron-ui.yml
@@ -51,7 +51,7 @@
- "pm2 start {{ metron_ui_directory }}/lib/metron-ui.js --name metron"
- pm2 save
- pm2 startup centos
- - su -c 'chmod +x /etc/init.d/pm2-init.sh; chkconfig --add pm2-init.sh'
+ - su -c 'chmod +x /etc/init.d/pm2-init.sh'
-- name: Start Metron UI
- service: name=pm2-init.sh state=restarted
+- name: Stop Metron UI
+ service: name=pm2-init.sh state=stopped
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/monit-start/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit-start/defaults/main.yml b/metron-deployment/roles/monit-start/defaults/main.yml
new file mode 100644
index 0000000..26a05b3
--- /dev/null
+++ b/metron-deployment/roles/monit-start/defaults/main.yml
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+
+#
+# defines which services will be started. by default, no services
+# are started
+#
+services_to_start: []
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/monit-start/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit-start/tasks/main.yml b/metron-deployment/roles/monit-start/tasks/main.yml
new file mode 100644
index 0000000..6bdae9f
--- /dev/null
+++ b/metron-deployment/roles/monit-start/tasks/main.yml
@@ -0,0 +1,50 @@
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Start monit
+ service: name=monit state=restarted
+
+- name: Reload monit definitions
+ shell: monit reload
+
+- name: Stop all services
+ shell: monit stop all
+ register: result
+ until: result.rc == 0
+ retries: 20
+ delay: 10
+
+- name: Find the services installed on the host
+ shell: monit summary | tail -n +3 | awk -F"'" '{print $2}'
+ register: installed_services
+
+# which services that need started are actually installed on this host?
+- set_fact:
+ installed_services_to_start: "{{ services_to_start | intersect(installed_services.stdout_lines) }}"
+
+- name: Start Metron services
+ debug:
+ msg: "Attemping to start: {{ installed_services_to_start }}"
+
+- name: Wait for metron services to start
+ shell: "monit start {{ item }}"
+ with_items: "{{ installed_services_to_start }}"
+ register: result
+ until: result.rc == 0
+ retries: 20
+ delay: 10
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/monit/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/defaults/main.yml b/metron-deployment/roles/monit/defaults/main.yml
new file mode 100644
index 0000000..ac86108
--- /dev/null
+++ b/metron-deployment/roles/monit/defaults/main.yml
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+monit_home: /usr/local/monit
+monit_config_home: /etc/monit.d
+monit_user: admin
+monit_pass: monit
+
+bro_pid_file: /usr/local/bro/spool/bro/.pid
+elasticsearch_pid_file: /var/run/elasticsearch/elasticsearch.pid
+mysql_pid_file: /var/run/mysqld/mysqld.pid
+snort_alert_csv_path: /var/log/snort/alert.csv
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/monit/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/tasks/main.yml b/metron-deployment/roles/monit/tasks/main.yml
new file mode 100644
index 0000000..ecae787
--- /dev/null
+++ b/metron-deployment/roles/monit/tasks/main.yml
@@ -0,0 +1,21 @@
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- include: monit.yml
+- include: scripts.yml
+- include: monit-definitions.yml
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/monit/tasks/monit-definitions.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/tasks/monit-definitions.yml b/metron-deployment/roles/monit/tasks/monit-definitions.yml
new file mode 100644
index 0000000..18ea0b0
--- /dev/null
+++ b/metron-deployment/roles/monit/tasks/monit-definitions.yml
@@ -0,0 +1,61 @@
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Create monit definition for bro
+ template: src=monit/bro.monit dest={{ monit_config_home }}/bro.monit
+ when: ("sensors" in group_names) and (install_bro | default(True))
+
+- name: Create monit definition for elasticsearch
+ template: src=monit/elasticsearch.monit dest={{ monit_config_home }}/elasticsearch.monit
+ when: ("search" in group_names) and (install_elasticsearch | default(True))
+
+- name: Create monit definition for enrichment with elasticsearch
+ template: src=monit/enrichment-elasticsearch.monit dest={{ monit_config_home }}/enrichment-elasticsearch.monit
+ when: ("enrichment" in group_names) and (install_elasticsearch | default(True))
+
+- name: Create monit definition for metron-ui
+ template: src=monit/metron-ui.monit dest={{ monit_config_home }}/metron-ui.monit
+ when: ("web" in group_names) and (install_elasticsearch | default(True))
+
+- name: Create monit definition for mysql
+ template: src=monit/mysql.monit dest={{ monit_config_home }}/mysql.monit
+ when: ("mysql" in group_names)
+
+- name: Create monit definition for parsers
+ template: src=monit/parsers.monit dest={{ monit_config_home }}/parsers.monit
+ when: ("enrichment" in group_names)
+
+- name: Create monit definition for pcap-replay
+ template: src=monit/pcap-replay.monit dest={{ monit_config_home }}/pcap-replay.monit
+ when: ("sensors" in group_names) and (install_pcap_replay | default(False))
+
+- name: Create monit definition for pcap-service
+ template: src=monit/pcap-service.monit dest={{ monit_config_home }}/pcap-service.monit
+ when: ("pcap_server" in group_names)
+
+- name: Create monit definition for pycapa
+ template: src=monit/pycapa.monit dest={{ monit_config_home }}/pycapa.monit
+ when: ("sensors" in group_names) and (install_pycapa | default(True))
+
+- name: Create monit definition for snort
+ template: src=monit/snort.monit dest={{ monit_config_home }}/snort.monit
+ when: ("sensors" in group_names) and (install_snort | default(True))
+
+- name: Create monit definition for yaf
+ template: src=monit/yaf.monit dest={{ monit_config_home }}/yaf.monit
+ when: ("sensors" in group_names) and (install_yaf | default(True))
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/monit/tasks/monit.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/tasks/monit.yml b/metron-deployment/roles/monit/tasks/monit.yml
new file mode 100644
index 0000000..bf300f4
--- /dev/null
+++ b/metron-deployment/roles/monit/tasks/monit.yml
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Install monit
+ yum:
+ name: monit
+ register: result
+ until: result.rc == 0
+ retries: 5
+ delay: 10
+
+- name: Deploy monit configuration
+ template: src=monit/monit.conf dest=/etc/monit.conf
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/monit/tasks/scripts.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/tasks/scripts.yml b/metron-deployment/roles/monit/tasks/scripts.yml
new file mode 100644
index 0000000..df1a443
--- /dev/null
+++ b/metron-deployment/roles/monit/tasks/scripts.yml
@@ -0,0 +1,53 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Create monit home
+ file: path={{ monit_home }} state=directory mode=0755
+
+- name: Deploy parser topology start scripts
+ template:
+ src: scripts/start_topology.sh
+ dest: "{{ monit_home }}/start_{{ item }}_topology.sh"
+ mode: 0755
+ with_items:
+ - yaf
+ - snort
+ - bro
+
+- name: Deploy topology stop scripts
+ template:
+ src: scripts/stop_topology.sh
+ dest: "{{ monit_home }}/stop_{{ item }}_topology.sh"
+ mode: 0755
+ with_items:
+ - yaf
+ - snort
+ - bro
+ - pcap
+ - enrichment
+
+- name: Deploy topology status scripts
+ template:
+ src: scripts/status_topology.sh
+ dest: "{{ monit_home }}/status_{{ item }}_topology.sh"
+ mode: 0755
+ with_items:
+ - yaf
+ - snort
+ - bro
+ - pcap
+ - enrichment
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/monit/templates/monit/bro.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/bro.monit b/metron-deployment/roles/monit/templates/monit/bro.monit
new file mode 100644
index 0000000..db6a318
--- /dev/null
+++ b/metron-deployment/roles/monit/templates/monit/bro.monit
@@ -0,0 +1,25 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+check process bro
+ with pidfile {{ bro_pid_file }}
+ start program = "/usr/local/bro/bin/broctl deploy"
+ stop program = "/usr/local/bro/bin/broctl stop"
+ restart program = "/usr/local/bro/bin/broctl restart"
+ if does not exist then restart
+ group bro
+ group sensors
+ group metron
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/monit/templates/monit/elasticsearch.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/elasticsearch.monit b/metron-deployment/roles/monit/templates/monit/elasticsearch.monit
new file mode 100644
index 0000000..805c3cb
--- /dev/null
+++ b/metron-deployment/roles/monit/templates/monit/elasticsearch.monit
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+check process elasticsearch with pidfile {{ elasticsearch_pid_file }}
+ start program = "/etc/init.d/elasticsearch start"
+ stop program = "/etc/init.d/elasticsearch stop"
+ if does not exist then restart
+ group enrichments
+ group search
+ group metron
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/monit/templates/monit/enrichment-elasticsearch.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/enrichment-elasticsearch.monit b/metron-deployment/roles/monit/templates/monit/enrichment-elasticsearch.monit
new file mode 100644
index 0000000..ceae475
--- /dev/null
+++ b/metron-deployment/roles/monit/templates/monit/enrichment-elasticsearch.monit
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+check program enrichment with path "{{ monit_home }}/status_enrichment_topology.sh"
+ start program "{{ metron_directory }}/bin/start_elasticsearch_topology.sh"
+ stop program "{{ monit_home }}/stop_enrichment_topology.sh"
+ if status != 0 then restart
+ group yaf
+ group bro
+ group snort
+ group enrichments
+ group metron
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/monit/templates/monit/enrichment-solr.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/enrichment-solr.monit b/metron-deployment/roles/monit/templates/monit/enrichment-solr.monit
new file mode 100644
index 0000000..94f4858
--- /dev/null
+++ b/metron-deployment/roles/monit/templates/monit/enrichment-solr.monit
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+check program enrichment with path "{{ monit_home }}/status_enrichment_topology.sh"
+ start program "{{ metron_directory }}/bin/start_solr_topology.sh"
+ stop program "{{ monit_home }}/stop_enrichment_topology.sh"
+ if status != 0 then restart
+ group yaf
+ group bro
+ group snort
+ group enrichments
+ group metron
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/monit/templates/monit/metron-ui.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/metron-ui.monit b/metron-deployment/roles/monit/templates/monit/metron-ui.monit
new file mode 100644
index 0000000..1b7e9cc
--- /dev/null
+++ b/metron-deployment/roles/monit/templates/monit/metron-ui.monit
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+check process metron-ui matching "node /usr/metron/{{ metron_version }}/metron-ui/lib/metron-ui.js"
+ start program = "/etc/init.d/pm2-init.sh start"
+ stop program = "/etc/init.d/pm2-init.sh stop"
+ if does not exist then restart
+ group web
+ group metron
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/monit/templates/monit/monit.conf
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/monit.conf b/metron-deployment/roles/monit/templates/monit/monit.conf
new file mode 100644
index 0000000..660c421
--- /dev/null
+++ b/metron-deployment/roles/monit/templates/monit/monit.conf
@@ -0,0 +1,29 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set logfile syslog
+set pidfile /var/run/monit.pid
+
+# check services at X second intervals
+set daemon 30
+include /etc/monit.d/*
+
+# allow http access
+set httpd port 2812 and
+ use address {{ inventory_hostname }}
+ allow 0.0.0.0/0
+ allow {{ monit_user }}:{{ monit_pass }}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/monit/templates/monit/mysql.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/mysql.monit b/metron-deployment/roles/monit/templates/monit/mysql.monit
new file mode 100644
index 0000000..555b444
--- /dev/null
+++ b/metron-deployment/roles/monit/templates/monit/mysql.monit
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+check process mysql with pidfile {{ mysql_pid_file }}
+ start program = "/etc/init.d/mysqld start"
+ stop program = "/etc/init.d/mysqld stop"
+ if does not exist then restart
+ group metron
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/monit/templates/monit/parsers.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/parsers.monit b/metron-deployment/roles/monit/templates/monit/parsers.monit
new file mode 100644
index 0000000..6d2fff3
--- /dev/null
+++ b/metron-deployment/roles/monit/templates/monit/parsers.monit
@@ -0,0 +1,49 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+check program pcap-parser with path "{{ monit_home }}/status_pcap_topology.sh"
+ start program "{{ metron_directory }}/bin/start_pcap_topology.sh"
+ stop program "{{ monit_home }}/stop_pcap_topology.sh"
+ if status != 0 then restart
+ group pcap
+ group parsers
+ group metron
+
+check program yaf-parser with path "{{ monit_home }}/status_yaf_topology.sh"
+ start program "{{ monit_home }}/start_yaf_topology.sh"
+ stop program "{{ monit_home }}/stop_yaf_topology.sh"
+ if status != 0 then restart
+ group yaf
+ group parsers
+ group metron
+
+check program bro-parser with path "{{ monit_home }}/status_bro_topology.sh"
+ start program "{{ monit_home }}/start_bro_topology.sh"
+ stop program "{{ monit_home }}/stop_bro_topology.sh"
+ if status != 0 then restart
+ group bro
+ group parsers
+ group metron
+
+check program snort-parser with path "{{ monit_home }}/status_snort_topology.sh"
+ start program "{{ monit_home }}/start_snort_topology.sh"
+ stop program "{{ monit_home }}/stop_snort_topology.sh"
+ if status != 0 then restart
+ group snort
+ group parsers
+ group metron
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/monit/templates/monit/pcap-replay.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/pcap-replay.monit b/metron-deployment/roles/monit/templates/monit/pcap-replay.monit
new file mode 100644
index 0000000..da0c006
--- /dev/null
+++ b/metron-deployment/roles/monit/templates/monit/pcap-replay.monit
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+check process pcap-replay
+ with pidfile /var/run/pcap-replay.pid
+ start program = "/etc/init.d/pcap-replay start"
+ stop program = "/etc/init.d/pcap-replay stop"
+ if does not exist then restart
+ group pcap
+ group sensors
+ group metron
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/monit/templates/monit/pcap-service.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/pcap-service.monit b/metron-deployment/roles/monit/templates/monit/pcap-service.monit
new file mode 100644
index 0000000..3f39c99
--- /dev/null
+++ b/metron-deployment/roles/monit/templates/monit/pcap-service.monit
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+check process pcap-service matching "org.apache.metron.pcapservice.rest.PcapService"
+ start program = "/etc/init.d/pcapservice start"
+ stop program = "/etc/init.d/pcapservice stop"
+ if does not exist then restart
+ group web
+ group metron
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/monit/templates/monit/pycapa.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/pycapa.monit b/metron-deployment/roles/monit/templates/monit/pycapa.monit
new file mode 100644
index 0000000..1123d87
--- /dev/null
+++ b/metron-deployment/roles/monit/templates/monit/pycapa.monit
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+check process pycapa
+ with pidfile /var/run/pycapa.pid
+ start program = "/etc/init.d/pycapa start"
+ stop program = "/etc/init.d/pycapa stop"
+ if does not exist then restart
+ group pcap
+ group sensors
+ group metron
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9898b4c6/metron-deployment/roles/monit/templates/monit/snort.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/snort.monit b/metron-deployment/roles/monit/templates/monit/snort.monit
new file mode 100644
index 0000000..93d2278
--- /dev/null
+++ b/metron-deployment/roles/monit/templates/monit/snort.monit
@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+check process snort matching "/usr/sbin/snort"
+ start program = "/etc/init.d/snortd start"
+ stop program = "/etc/init.d/snortd stop"
+ if does not exist then restart
+ group snort
+ group sensors
+ group metron
+
+check process snort-logs
+ matching "tail -F {{ snort_alert_csv_path }}"
+ start program = "/etc/init.d/flume-agent start snort"
+ stop program = "/etc/init.d/flume-agent stop snort"
+ if does not exist then restart
+ depends on snort
+ group snort
+ group sensors
+ group metron