You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by rm...@apache.org on 2016/11/03 16:18:52 UTC
incubator-metron git commit: METRON-514 Snort kafka topic fails to
get any data on a setup running for several days (merrimanr) closes
apache/incubator-metron#323
Repository: incubator-metron
Updated Branches:
refs/heads/master 05d29e7cb -> aed9b6a57
METRON-514 Snort kafka topic fails to get any data on a setup running for several days (merrimanr) closes apache/incubator-metron#323
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/aed9b6a5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/aed9b6a5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/aed9b6a5
Branch: refs/heads/master
Commit: aed9b6a57bd0bdf306eae0431dabf76641a0ef46
Parents: 05d29e7
Author: merrimanr <me...@gmail.com>
Authored: Thu Nov 3 11:17:57 2016 -0500
Committer: rmerriman <rm...@hortonworks.com>
Committed: Thu Nov 3 11:17:57 2016 -0500
----------------------------------------------------------------------
.../playbooks/docker_probe_install.yml | 1 -
metron-deployment/playbooks/metron_install.yml | 1 -
metron-deployment/roles/flume/defaults/main.yml | 19 -----
metron-deployment/roles/flume/meta/main.yml | 20 -----
metron-deployment/roles/flume/tasks/main.yml | 52 ------------
.../roles/monit/templates/monit/snort.monit | 8 +-
metron-deployment/roles/snort/defaults/main.yml | 2 +
.../roles/snort/files/flume-snort.conf | 44 -----------
metron-deployment/roles/snort/tasks/flume.yml | 31 --------
metron-deployment/roles/snort/tasks/main.yml | 2 +-
.../roles/snort/tasks/producer.yml | 28 +++++++
.../roles/snort/templates/snort-producer | 83 ++++++++++++++++++++
.../snort/templates/start-snort-producer.sh | 24 ++++++
metron-deployment/roles/yaf/tasks/yaf.yml | 4 -
14 files changed, 142 insertions(+), 177 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aed9b6a5/metron-deployment/playbooks/docker_probe_install.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/playbooks/docker_probe_install.yml b/metron-deployment/playbooks/docker_probe_install.yml
index 2ec087f..cc4511e 100644
--- a/metron-deployment/playbooks/docker_probe_install.yml
+++ b/metron-deployment/playbooks/docker_probe_install.yml
@@ -54,7 +54,6 @@
connection: docker
roles:
- { role: bro, tags: ['bro'] }
- - { role: flume, tags: ['snort','flume'] }
- { role: snort, tags: ['snort'] }
- { role: yaf, tags: ['yaf'] }
- { role: pcap_replay, tags: ['pcap-replay'] }
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aed9b6a5/metron-deployment/playbooks/metron_install.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/playbooks/metron_install.yml b/metron-deployment/playbooks/metron_install.yml
index 21ecebd..a18fb30 100644
--- a/metron-deployment/playbooks/metron_install.yml
+++ b/metron-deployment/playbooks/metron_install.yml
@@ -129,7 +129,6 @@
- { role: tap_interface, tags: ['tap'] }
- { role: pycapa, tags: ['pycapa'] }
- { role: bro, tags: ['bro'] }
- - { role: flume, tags: ['snort','flume'] }
- { role: snort, tags: ['snort'] }
- { role: yaf, tags: ['yaf'] }
- { role: pcap_replay, tags: ['pcap-replay'] }
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aed9b6a5/metron-deployment/roles/flume/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/flume/defaults/main.yml b/metron-deployment/roles/flume/defaults/main.yml
deleted file mode 100644
index d96c844..0000000
--- a/metron-deployment/roles/flume/defaults/main.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-hdp_repo_def: "http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.5.0.0/hdp.repo"
-java_home: /usr/jdk64/jdk1.8.0_60
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aed9b6a5/metron-deployment/roles/flume/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/flume/meta/main.yml b/metron-deployment/roles/flume/meta/main.yml
deleted file mode 100644
index ff35a5a..0000000
--- a/metron-deployment/roles/flume/meta/main.yml
+++ /dev/null
@@ -1,20 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-dependencies:
- - java_jdk
- - libselinux-python
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aed9b6a5/metron-deployment/roles/flume/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/flume/tasks/main.yml b/metron-deployment/roles/flume/tasks/main.yml
deleted file mode 100644
index 8576c3c..0000000
--- a/metron-deployment/roles/flume/tasks/main.yml
+++ /dev/null
@@ -1,52 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Retrieve HDP repository definition
- get_url:
- url: "{{ hdp_repo_def }}"
- dest: /etc/yum.repos.d/hdp.repo
- mode: 0644
-
-- name: Install flume
- yum: name={{item}}
- with_items:
- - flume
- - flume-agent
- register: result
- until: result.rc == 0
- retries: 5
- delay: 10
-
-- name: Create flume-env.sh
- shell: cp /etc/flume/conf/flume-env.sh.template /etc/flume/conf/flume-env.sh
-
-- name: Configure flume-env.sh
- lineinfile: >
- dest=/etc/flume/conf/flume-env.sh
- regexp="{{ item.regexp }}"
- line="{{ item.line }}"
- state=present
- with_items:
- - { regexp: '^.*export JAVA_HOME=.*$', line: 'export JAVA_HOME={{ java_home }}' }
-
-- name: Create flume service
- shell: "{{item}}"
- with_items:
- - cp /usr/hdp/current/flume-server/etc/rc.d/init.d/flume-agent /etc/init.d/
-
-- name: Remove default flume configuration
- file: path=/etc/flume/conf/flume.conf state=absent
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aed9b6a5/metron-deployment/roles/monit/templates/monit/snort.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/snort.monit b/metron-deployment/roles/monit/templates/monit/snort.monit
index 93d2278..6fb429b 100644
--- a/metron-deployment/roles/monit/templates/monit/snort.monit
+++ b/metron-deployment/roles/monit/templates/monit/snort.monit
@@ -22,10 +22,10 @@ check process snort matching "/usr/sbin/snort"
group sensors
group metron
-check process snort-logs
- matching "tail -F {{ snort_alert_csv_path }}"
- start program = "/etc/init.d/flume-agent start snort"
- stop program = "/etc/init.d/flume-agent stop snort"
+check process snort-producer
+ with pidfile /var/run/snort-producer.pid
+ start program = "/etc/init.d/snort-producer start"
+ stop program = "/etc/init.d/snort-producer stop"
if does not exist then restart
depends on snort
group snort
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aed9b6a5/metron-deployment/roles/snort/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/snort/defaults/main.yml b/metron-deployment/roles/snort/defaults/main.yml
index 2d8004d..c8b126b 100644
--- a/metron-deployment/roles/snort/defaults/main.yml
+++ b/metron-deployment/roles/snort/defaults/main.yml
@@ -24,3 +24,5 @@ snort_community_rules_url: "https://www.snort.org/downloads/community/community-
dag_src_url: "https://snort.org/downloads/snort/daq-{{ daq_version }}.src.rpm"
sniff_interface: eth0
snort_home_net: any
+snort_producer_home: /opt/snort-producer
+snort_producer_start: /opt/snort-producer/start-snort-producer.sh
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aed9b6a5/metron-deployment/roles/snort/files/flume-snort.conf
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/snort/files/flume-snort.conf b/metron-deployment/roles/snort/files/flume-snort.conf
deleted file mode 100644
index 7dea516..0000000
--- a/metron-deployment/roles/snort/files/flume-snort.conf
+++ /dev/null
@@ -1,44 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-snort.sources = exec-source
-snort.channels = memory-channel
-snort.sinks = kafka-sink logger-sink
-
-# snort alerts are logged to a file
-snort.sources.exec-source.type = exec
-snort.sources.exec-source.command = tail -F /var/log/snort/alert
-snort.sources.exec-source.restart = true
-snort.sources.exec-source.logStdErr = true
-
-# snort alerts are sent to kafka
-snort.sinks.kafka-sink.type = org.apache.flume.sink.kafka.KafkaSink
-snort.sinks.kafka-sink.brokerList = localhost:9092
-snort.sinks.kafka-sink.topic = snort
-
-# also log events
-snort.sinks.logger-sink.type = logger
-
-# buffer events in memory
-snort.channels.memory-channel.type = memory
-snort.channels.memory-channel.capacity = 1000
-snort.channels.memory-channel.transactionCapacity = 100
-
-# bind the source and sink to the channel
-snort.sources.exec-source.channels = memory-channel
-snort.sinks.kafka-sink.channel = memory-channel
-snort.sinks.logger-sink.channel = memory-channel
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aed9b6a5/metron-deployment/roles/snort/tasks/flume.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/snort/tasks/flume.yml b/metron-deployment/roles/snort/tasks/flume.yml
deleted file mode 100644
index e5bd593..0000000
--- a/metron-deployment/roles/snort/tasks/flume.yml
+++ /dev/null
@@ -1,31 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Install flume configurations
- copy: src={{ item.src }} dest={{ item.dest }}
- with_items:
- - { src: flume-snort.conf, dest: /etc/flume/conf/flume-snort.conf }
-
-- name: Configure destination for snort alerts
- lineinfile: dest=/etc/flume/conf/flume-snort.conf regexp={{ item.regexp }} line={{ item.line }}
- with_items:
- - { regexp: '^snort\.sinks\.kafka-sink\.brokerList.*$',
- line: 'snort.sinks.kafka-sink.brokerList = {{ kafka_broker_url }}' }
- - { regexp: '^snort\.sinks\.kafka-sink\.topic.*$',
- line: 'snort.sinks.kafka-sink.topic = {{ snort_topic }}'}
- - { regexp: '^snort.sources.exec-source.command.*$',
- line: 'snort.sources.exec-source.command = tail -F {{ snort_alert_csv_path }}' }
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aed9b6a5/metron-deployment/roles/snort/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/snort/tasks/main.yml b/metron-deployment/roles/snort/tasks/main.yml
index b2ed836..4736bee 100644
--- a/metron-deployment/roles/snort/tasks/main.yml
+++ b/metron-deployment/roles/snort/tasks/main.yml
@@ -17,5 +17,5 @@
---
- include: daq.yml
- include: snort.yml
-- include: flume.yml
+- include: producer.yml
- include: nic.yml
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aed9b6a5/metron-deployment/roles/snort/tasks/producer.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/snort/tasks/producer.yml b/metron-deployment/roles/snort/tasks/producer.yml
new file mode 100644
index 0000000..d6e9c3a
--- /dev/null
+++ b/metron-deployment/roles/snort/tasks/producer.yml
@@ -0,0 +1,28 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Create Snort producer home directory
+ file:
+ path: "{{ snort_producer_home }}"
+ state: directory
+ mode: 0755
+
+- name: Install Snort producer start script
+ template: src=start-snort-producer.sh dest={{ snort_producer_home }}/start-snort-producer.sh mode=0755
+
+- name: Install init.d service script
+ template: src=snort-producer dest=/etc/init.d/snort-producer mode=0755
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aed9b6a5/metron-deployment/roles/snort/templates/snort-producer
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/snort/templates/snort-producer b/metron-deployment/roles/snort/templates/snort-producer
new file mode 100644
index 0000000..1cb68ca
--- /dev/null
+++ b/metron-deployment/roles/snort/templates/snort-producer
@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Snort Kafka producer daemon
+# chkconfig: 345 20 80
+# description: Runs Snort Kafka producer
+# processname: snort-producer
+#
+NAME=snort-producer
+DESC="Executes Snort Kafka producer"
+PIDFILE=/var/run/$NAME.pid
+SCRIPTNAME=/etc/init.d/$NAME
+DAEMON_PATH="{{ snort_producer_home }}"
+DAEMON="{{ snort_producer_start }}"
+DAEMONOPTS="${@:2}"
+
+case "$1" in
+ start)
+ printf "%-50s" "Starting $NAME..."
+
+ # kick-off the daemon
+ cd $DAEMON_PATH
+ PID=`$DAEMON $DAEMONOPTS > /dev/null 2>&1 & echo $!`
+ if [ -z $PID ]; then
+ printf "%s\n" "Fail"
+ else
+ echo $PID > $PIDFILE
+ printf "%s\n" "Ok"
+ fi
+ ;;
+
+ status)
+ printf "%-50s" "Checking $NAME..."
+ if [ -f $PIDFILE ]; then
+ PID=`cat $PIDFILE`
+ if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then
+ printf "%s\n" "Process dead but pidfile exists"
+ else
+ echo "Running"
+ fi
+ else
+ printf "%s\n" "Service not running"
+ fi
+ ;;
+
+ stop)
+ printf "%-50s" "Stopping $NAME"
+ PID=`cat $PIDFILE`
+ PGID=`ps -o pgid= $PID | xargs`
+ cd $DAEMON_PATH
+ if [ -f $PIDFILE ]; then
+ kill -- -$PGID
+ printf "%s\n" "Ok"
+ rm -f $PIDFILE
+ else
+ printf "%s\n" "pidfile not found"
+ fi
+ ;;
+
+ restart)
+ $0 stop
+ $0 start
+ ;;
+
+ *)
+ echo "Usage: $0 {status|start|stop|restart}"
+ exit 1
+esac
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aed9b6a5/metron-deployment/roles/snort/templates/start-snort-producer.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/snort/templates/start-snort-producer.sh b/metron-deployment/roles/snort/templates/start-snort-producer.sh
new file mode 100644
index 0000000..5234aae
--- /dev/null
+++ b/metron-deployment/roles/snort/templates/start-snort-producer.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# a simple script that tails the Snort alert log and pipes alerts into the
+# snort Kafka topic via the Kafka console producer
+#
+tail -F {{ snort_alert_csv_path }} | {{ kafka_prod }} --broker-list {{ kafka_broker_url }} --topic {{ snort_topic }}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aed9b6a5/metron-deployment/roles/yaf/tasks/yaf.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/yaf/tasks/yaf.yml b/metron-deployment/roles/yaf/tasks/yaf.yml
index 7ee361e..21445b2 100644
--- a/metron-deployment/roles/yaf/tasks/yaf.yml
+++ b/metron-deployment/roles/yaf/tasks/yaf.yml
@@ -48,7 +48,3 @@
- name: Install init.d service script
template: src=yaf dest=/etc/init.d/yaf mode=0755
-
-- name: Register the service with systemd
- shell: systemctl enable yaf
- when: ansible_distribution == "CentOS" and ansible_distribution_major_version == "7"