You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2016/03/21 18:06:24 UTC
[42/43] incubator-metron git commit: METRON-73 Sensor Service
Wrappers and Deprecation Fixes (nickwallen via cestella) closes
apache/incubator-metron#49
METRON-73 Sensor Service Wrappers and Deprecation Fixes (nickwallen via cestella) closes apache/incubator-metron#49
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/a44cc016
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/a44cc016
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/a44cc016
Branch: refs/heads/Metron_0.1BETA
Commit: a44cc01607560ae7d9349843587ad482c8e1be34
Parents: a85d32b
Author: nickwallen <ni...@nickallen.org>
Authored: Mon Mar 21 12:47:06 2016 -0400
Committer: cstella <ce...@gmail.com>
Committed: Mon Mar 21 12:47:06 2016 -0400
----------------------------------------------------------------------
.travis.yml | 2 +
deployment/amazon-ec2/.gitignore | 1 +
deployment/amazon-ec2/tasks/create-hosts.yml | 3 +-
.../amazon-ec2/tasks/provisioning-report.yml | 9 +++
.../ambari_common/tasks/passwd_less_ssh.yml | 3 +-
deployment/roles/bro/defaults/main.yml | 20 -----
deployment/roles/bro/tasks/bro-plugin-kafka.yml | 6 +-
deployment/roles/bro/tasks/bro.yml | 6 +-
deployment/roles/bro/tasks/librdkafka.yml | 4 +-
deployment/roles/bro/tasks/start-bro.yml | 5 +-
deployment/roles/bro/vars/main.yml | 10 +++
deployment/roles/pycapa/tasks/dependencies.yml | 34 ++++++++
deployment/roles/pycapa/tasks/main.yml | 46 +----------
deployment/roles/pycapa/tasks/pycapa.yml | 37 +++++++++
deployment/roles/pycapa/templates/pycapa | 84 ++++++++++++++++++++
deployment/roles/pycapa/vars/main.yml | 5 +-
16 files changed, 194 insertions(+), 81 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a44cc016/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..d86ee11
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,2 @@
+language: java
+script: mvn apache-rat:check && cd metron-streaming && mvn -q integration-test package | grep -v "DEBUG\|INFO\|WARN"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a44cc016/deployment/amazon-ec2/.gitignore
----------------------------------------------------------------------
diff --git a/deployment/amazon-ec2/.gitignore b/deployment/amazon-ec2/.gitignore
index 38b03a9..9c214d2 100644
--- a/deployment/amazon-ec2/.gitignore
+++ b/deployment/amazon-ec2/.gitignore
@@ -1,3 +1,4 @@
*.pem
*.secret
*.log
+*.retry
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a44cc016/deployment/amazon-ec2/tasks/create-hosts.yml
----------------------------------------------------------------------
diff --git a/deployment/amazon-ec2/tasks/create-hosts.yml b/deployment/amazon-ec2/tasks/create-hosts.yml
index e1ff3e9..39bae3a 100644
--- a/deployment/amazon-ec2/tasks/create-hosts.yml
+++ b/deployment/amazon-ec2/tasks/create-hosts.yml
@@ -50,6 +50,5 @@
- name: Add host(s) to a hostgroup
add_host: hostname={{ item.public_dns_name }} groups={{ host_type }}
- with_items: ec2.tagged_instances
+ with_items: "{{ ec2.tagged_instances }}"
when: item.public_dns_name is defined
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a44cc016/deployment/amazon-ec2/tasks/provisioning-report.yml
----------------------------------------------------------------------
diff --git a/deployment/amazon-ec2/tasks/provisioning-report.yml b/deployment/amazon-ec2/tasks/provisioning-report.yml
index 23ee9b1..d2abec0 100644
--- a/deployment/amazon-ec2/tasks/provisioning-report.yml
+++ b/deployment/amazon-ec2/tasks/provisioning-report.yml
@@ -15,6 +15,15 @@
# limitations under the License.
#
---
+- name: Known hosts groups
+ debug: var=groups
+
+- name: Sanity check Metron web
+ local_action: wait_for host="{{ groups.web[0] }}" port=5000 timeout=20
+
+- name: Sanity check Ambari web
+ local_action: wait_for host="{{ groups.ambari_master[0] }}" port="{{ ambari_port }}" timeout=20
+
- set_fact:
Success:
- "Apache Metron deployed successfully"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a44cc016/deployment/roles/ambari_common/tasks/passwd_less_ssh.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_common/tasks/passwd_less_ssh.yml b/deployment/roles/ambari_common/tasks/passwd_less_ssh.yml
index 6f613d2..0928e34 100644
--- a/deployment/roles/ambari_common/tasks/passwd_less_ssh.yml
+++ b/deployment/roles/ambari_common/tasks/passwd_less_ssh.yml
@@ -24,9 +24,8 @@
- name: Add key pairs to existing authorized_keys
authorized_key: user={{ ambari_user }} key="{{ lookup('file', '/tmp/keys/{{ item }}.pub') }}"
with_items:
- play_hosts
+ - "{{ play_hosts }}"
-#ambari_common.hosts
- name: Remove local copy of ssh keys
local_action: file path=/tmp/keys/{{ inventory_hostname }}.pub state=absent
become: False
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a44cc016/deployment/roles/bro/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/bro/defaults/main.yml b/deployment/roles/bro/defaults/main.yml
deleted file mode 100644
index 11a39ed..0000000
--- a/deployment/roles/bro/defaults/main.yml
+++ /dev/null
@@ -1,20 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-bro_crontab_minutes: 0-59/5
-bro_crontab_job: /usr/local/bro/bin/broctl cron
-bro_clean_job: rm -rf /usr/local/bro/spool/tmp/*
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a44cc016/deployment/roles/bro/tasks/bro-plugin-kafka.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/bro/tasks/bro-plugin-kafka.yml b/deployment/roles/bro/tasks/bro-plugin-kafka.yml
index 91e5faf..d920688 100644
--- a/deployment/roles/bro/tasks/bro-plugin-kafka.yml
+++ b/deployment/roles/bro/tasks/bro-plugin-kafka.yml
@@ -22,16 +22,16 @@
shell: "{{ item }}"
args:
chdir: "/tmp/bro-plugin-kafka"
- creates: /usr/local/bro/lib/bro/plugins/BRO_KAFKA
+ creates: "{{ bro_home }}/lib/bro/plugins/BRO_KAFKA"
with_items:
- rm -rf build/
- - "./configure --bro-dist=/tmp/bro-{{ bro_version }}"
+ - "./configure --bro-dist=/tmp/bro-{{ bro_version }} --install-root={{ bro_home }}/lib/bro/plugins/ --with-librdkafka={{ librdkafka_home }}"
- make
- make install
- name: Configure bro-kafka plugin
lineinfile:
- dest: /usr/local/bro/share/bro/site/local.bro
+ dest: "{{ bro_home }}/share/bro/site/local.bro"
line: "{{ item }}"
with_items:
- "@load Bro/Kafka/logs-to-kafka.bro"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a44cc016/deployment/roles/bro/tasks/bro.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/bro/tasks/bro.yml b/deployment/roles/bro/tasks/bro.yml
index 55c86d6..fb27ef9 100644
--- a/deployment/roles/bro/tasks/bro.yml
+++ b/deployment/roles/bro/tasks/bro.yml
@@ -31,14 +31,14 @@
shell: "{{ item }}"
args:
chdir: "/tmp/bro-{{ bro_version }}"
- creates: /usr/local/bro/bin/bro
+ creates: "{{ bro_home }}/bin/bro"
with_items:
- - ./configure
+ - "./configure --prefix={{ bro_home }}"
- make
- make install
- name: Configure bro
lineinfile:
- dest: /usr/local/bro/etc/node.cfg
+ dest: "{{ bro_home }}/etc/node.cfg"
regexp: '^interface=.*$'
line: 'interface={{ sniff_interface }}'
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a44cc016/deployment/roles/bro/tasks/librdkafka.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/bro/tasks/librdkafka.yml b/deployment/roles/bro/tasks/librdkafka.yml
index 925c18f..652d319 100644
--- a/deployment/roles/bro/tasks/librdkafka.yml
+++ b/deployment/roles/bro/tasks/librdkafka.yml
@@ -31,9 +31,9 @@
shell: "{{ item }}"
args:
chdir: "/tmp/librdkafka-{{ librdkafka_version }}"
- creates: /usr/local/lib/librdkafka.so
+ creates: "{{ librdkafka_home }}/lib/librdkafka.so"
with_items:
- rm -rf build/
- - ./configure
+ - "./configure --prefix={{ librdkafka_home }}"
- make
- make install
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a44cc016/deployment/roles/bro/tasks/start-bro.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/bro/tasks/start-bro.yml b/deployment/roles/bro/tasks/start-bro.yml
index 14d5ce6..1a0b938 100644
--- a/deployment/roles/bro/tasks/start-bro.yml
+++ b/deployment/roles/bro/tasks/start-bro.yml
@@ -18,11 +18,8 @@
- name: Turn on promiscuous mode for {{ sniff_interface }}
shell: "ip link set {{ sniff_interface }} promisc on"
-- name: Update bro configuration
- shell: /usr/local/bro/bin/broctl install
-
- name: Start bro
- shell: /usr/local/bro/bin/broctl start
+ shell: "{{ bro_home }}/bin/broctl deploy"
- name: Bro Cronjob
cron:
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a44cc016/deployment/roles/bro/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/bro/vars/main.yml b/deployment/roles/bro/vars/main.yml
index 8141253..1e1d1ab 100644
--- a/deployment/roles/bro/vars/main.yml
+++ b/deployment/roles/bro/vars/main.yml
@@ -15,7 +15,17 @@
# limitations under the License.
#
---
+bro_home: /usr/local/bro
bro_version: 2.4.1
+bro_daemon_log: /var/log/bro.log
bro_topic: bro
+
+# bro cronjob
+bro_crontab_minutes: 0-59/5
+bro_crontab_job: "{{ bro_home }}/bin/broctl cron"
+bro_clean_job: "rm -rf {{ bro_home }}/spool/tmp/*"
+
+# librdkafka
librdkafka_version: 0.8.6
librdkafka_url: https://github.com/edenhill/librdkafka/archive/0.8.6.tar.gz
+librdkafka_home: /usr/local
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a44cc016/deployment/roles/pycapa/tasks/dependencies.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/pycapa/tasks/dependencies.yml b/deployment/roles/pycapa/tasks/dependencies.yml
new file mode 100644
index 0000000..19efdcd
--- /dev/null
+++ b/deployment/roles/pycapa/tasks/dependencies.yml
@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Install epel-release repository
+ yum: name=epel-release
+
+- name: Install python and tcpdump
+ yum:
+ name: "{{item}}"
+ with_items:
+ - python
+ - tcpdump
+ - git
+ register: result
+ until: result.rc == 0
+ retries: 5
+ delay: 10
+
+- name: Install pip
+ easy_install: name=pip
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a44cc016/deployment/roles/pycapa/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/pycapa/tasks/main.yml b/deployment/roles/pycapa/tasks/main.yml
index 8649f3a..76bdc1c 100644
--- a/deployment/roles/pycapa/tasks/main.yml
+++ b/deployment/roles/pycapa/tasks/main.yml
@@ -15,47 +15,5 @@
# limitations under the License.
#
---
-- name: Install python and tcpdump
- yum:
- name: "{{item}}"
- with_items:
- - python
- - tcpdump
- - git
- register: result
- until: result.rc == 0
- retries: 5
- delay: 10
-
-- name: Clone pycapa repo
- git: repo={{ pycapa_repo }} dest={{ pycapa_home }}
-
-- name: Build pycapa
- shell: "{{ item }}"
- args:
- chdir: "{{ pycapa_home }}"
- with_items:
- - pip install -r requirements.txt
- - pip install argparse
- - python setup.py install
-
-- name: Turn on promiscuous mode for {{ sniff_interface }}
- shell: "ip link set {{ sniff_interface }} promisc on"
-
-- name: Install Daemonize
- yum:
- name: daemonize
- register: result
- until: result.rc == 0
- retries: 5
- delay: 10
-
-- set_fact:
- pycapa: /opt/pycapa/pycapa/pycapa_cli.py
- pycapa_log: /var/log/pycapa.log
- pycapa_lock: /var/lock/subsys/pycapa
-
-- name: Start pycapa
- shell: "daemonize -c {{ pycapa_home }} -e {{ pycapa_log }} -o {{ pycapa_log }} -l {{ pycapa_lock }} /usr/bin/python {{ pycapa }} -z {{ zookeeper_url }} -t {{ pycapa_topic }} -i {{ sniff_interface }}"
- args:
- creates: "{{ pycapa_lock }}"
+- include: dependencies.yml
+- include: pycapa.yml
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a44cc016/deployment/roles/pycapa/tasks/pycapa.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/pycapa/tasks/pycapa.yml b/deployment/roles/pycapa/tasks/pycapa.yml
new file mode 100644
index 0000000..4d1a64e
--- /dev/null
+++ b/deployment/roles/pycapa/tasks/pycapa.yml
@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Clone pycapa repo
+ git: repo={{ pycapa_repo }} dest={{ pycapa_home }}
+
+- name: Build pycapa
+ shell: "{{ item }}"
+ args:
+ chdir: "{{ pycapa_home }}"
+ with_items:
+ - pip install -r requirements.txt
+ - pip install argparse
+ - python setup.py install
+
+- name: Turn on promiscuous mode for {{ pycapa_sniff_interface }}
+ shell: "ip link set {{ pycapa_sniff_interface }} promisc on"
+
+- name: Install service script
+ template: src=pycapa dest=/etc/init.d/pycapa mode=0755
+
+- name: Start pycapa
+ service: name=pycapa state=restarted
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a44cc016/deployment/roles/pycapa/templates/pycapa
----------------------------------------------------------------------
diff --git a/deployment/roles/pycapa/templates/pycapa b/deployment/roles/pycapa/templates/pycapa
new file mode 100644
index 0000000..fffa13f
--- /dev/null
+++ b/deployment/roles/pycapa/templates/pycapa
@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# metron pycapa service
+# chkconfig: 345 20 80
+# description: Metron Pycapa Packet Capture Daemon
+# processname: pycapa
+#
+NAME=pycapa
+DESC="Metron Pycapa Packet Capture"
+PIDFILE=/var/run/$NAME.pid
+SCRIPTNAME=/etc/init.d/$NAME
+LOGFILE="{{ pycapa_log }}"
+EXTRA_ARGS="${@:2}"
+DAEMON_PATH="{{ pycapa_home }}"
+DAEMON="/usr/bin/python"
+DAEMONOPTS="{{ pycapa_main }} -z {{ zookeeper_url }} -t {{ pycapa_topic }} -i {{ pycapa_sniff_interface }}"
+
+case "$1" in
+ start)
+ printf "%-50s" "Starting $NAME..."
+
+ # kick-off the daemon
+ cd $DAEMON_PATH
+ PID=`$DAEMON $DAEMONOPTS >> $LOGFILE 2>&1 & echo $!`
+ if [ -z $PID ]; then
+ printf "%s\n" "Fail"
+ else
+ echo $PID > $PIDFILE
+ printf "%s\n" "Ok"
+ fi
+ ;;
+
+ status)
+ printf "%-50s" "Checking $NAME..."
+ if [ -f $PIDFILE ]; then
+ PID=`cat $PIDFILE`
+ if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then
+ printf "%s\n" "Process dead but pidfile exists"
+ else
+ echo "Running"
+ fi
+ else
+ printf "%s\n" "Service not running"
+ fi
+ ;;
+
+ stop)
+ printf "%-50s" "Stopping $NAME"
+ PID=`cat $PIDFILE`
+ cd $DAEMON_PATH
+ if [ -f $PIDFILE ]; then
+ kill -HUP $PID
+ printf "%s\n" "Ok"
+ rm -f $PIDFILE
+ else
+ printf "%s\n" "pidfile not found"
+ fi
+ ;;
+
+ restart)
+ $0 stop
+ $0 start
+ ;;
+
+ *)
+ echo "Usage: $0 {status|start|stop|restart}"
+ exit 1
+esac
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a44cc016/deployment/roles/pycapa/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/pycapa/vars/main.yml b/deployment/roles/pycapa/vars/main.yml
index d47fa65..5618a8e 100644
--- a/deployment/roles/pycapa/vars/main.yml
+++ b/deployment/roles/pycapa/vars/main.yml
@@ -16,5 +16,8 @@
#
---
pycapa_repo: https://github.com/OpenSOC/pycapa.git
-pycapa_home: /opt/pycapa
+pycapa_home: /usr/local/pycapa
+pycapa_main: "{{ pycapa_home }}/pycapa/pycapa_cli.py"
+pycapa_log: /var/log/pycapa.log
pycapa_topic: pcap
+pycapa_sniff_interface: "{{ sniff_interface }}"