You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ni...@apache.org on 2016/04/27 19:06:26 UTC
[2/2] incubator-metron git commit: METRON-116 Added automated testing
and documentation of renamed packet capture process;
fastcapa (nickwallen) closes apache/incubator-metron#90
METRON-116 Added automated testing and documentation of renamed packet capture process; fastcapa (nickwallen) closes apache/incubator-metron#90
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/61aced79
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/61aced79
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/61aced79
Branch: refs/heads/master
Commit: 61aced7924ed87e45ec5904c936bac964cabf914
Parents: 64f772c
Author: nickwallen <ni...@nickallen.org>
Authored: Wed Apr 27 13:05:59 2016 -0400
Committer: Nick Allen <ni...@nickallen.org>
Committed: Wed Apr 27 13:05:59 2016 -0400
----------------------------------------------------------------------
.../roles/fastcapa/defaults/main.yml | 32 ++
metron-deployment/roles/fastcapa/meta/main.yml | 19 +
.../roles/fastcapa/tasks/debug.yml | 26 ++
.../roles/fastcapa/tasks/dependencies.yml | 38 ++
metron-deployment/roles/fastcapa/tasks/dpdk.yml | 59 ++++
.../roles/fastcapa/tasks/fastcapa.yml | 49 +++
.../roles/fastcapa/tasks/kernel.yml | 51 +++
metron-deployment/roles/fastcapa/tasks/main.yml | 22 ++
.../roles/fastcapa/templates/fastcapa | 94 +++++
.../roles/fastcapa/templates/fastcapa.conf | 67 ++++
.../roles/pcap_replay/meta/main.yml | 4 -
.../roles/pcap_replay/tasks/dependencies.yml | 32 ++
.../roles/pcap_replay/tasks/main.yml | 1 +
.../roles/sensor-test-mode/defaults/main.yml | 20 ++
.../roles/sensor-test-mode/tasks/main.yml | 45 +--
.../roles/sensor-test-mode/tasks/pcap.yml | 25 ++
.../roles/sensor-test-mode/tasks/snort.yml | 36 ++
.../roles/sensor-test-mode/tasks/yaf.yml | 27 ++
.../vagrant/fastcapa-vagrant/.gitignore | 1 +
.../vagrant/fastcapa-vagrant/README.md | 73 ++++
.../vagrant/fastcapa-vagrant/Vagrantfile | 71 ++++
.../vagrant/fastcapa-vagrant/ansible.cfg | 22 ++
.../vagrant/fastcapa-vagrant/playbook.yml | 50 +++
.../tasks/validate-messages-received.yml | 38 ++
.../tasks/validate-packets-sent.yml | 41 +++
.../vagrant/fastcapa-vagrant/vars/main.yml | 26 ++
.../vagrant/packet-capture/Vagrantfile | 69 ----
.../vagrant/packet-capture/ansible.cfg | 22 --
.../vagrant/packet-capture/playbook.yml | 43 ---
metron-sensors/fastcapa/.gitignore | 3 +
metron-sensors/fastcapa/Makefile | 26 ++
metron-sensors/fastcapa/README.md | 20 ++
metron-sensors/fastcapa/conf/localhost.kafka | 67 ++++
metron-sensors/fastcapa/src/.gitignore | 1 +
metron-sensors/fastcapa/src/Makefile | 52 +++
metron-sensors/fastcapa/src/args.c | 145 ++++++++
metron-sensors/fastcapa/src/args.h | 107 ++++++
metron-sensors/fastcapa/src/kafka.c | 224 ++++++++++++
metron-sensors/fastcapa/src/kafka.h | 45 +++
metron-sensors/fastcapa/src/main.c | 349 +++++++++++++++++++
metron-sensors/fastcapa/src/main.h | 122 +++++++
metron-sensors/packet-capture/.gitignore | 3 -
metron-sensors/packet-capture/Makefile | 26 --
metron-sensors/packet-capture/README.md | 12 -
.../packet-capture/conf/localhost.kafka | 67 ----
metron-sensors/packet-capture/src/.gitignore | 1 -
metron-sensors/packet-capture/src/Makefile | 52 ---
metron-sensors/packet-capture/src/args.c | 145 --------
metron-sensors/packet-capture/src/args.h | 107 ------
metron-sensors/packet-capture/src/kafka.c | 224 ------------
metron-sensors/packet-capture/src/kafka.h | 45 ---
metron-sensors/packet-capture/src/main.c | 349 -------------------
metron-sensors/packet-capture/src/main.h | 122 -------
53 files changed, 2087 insertions(+), 1330 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-deployment/roles/fastcapa/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/fastcapa/defaults/main.yml b/metron-deployment/roles/fastcapa/defaults/main.yml
new file mode 100644
index 0000000..2c00790
--- /dev/null
+++ b/metron-deployment/roles/fastcapa/defaults/main.yml
@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+# dpdk
+dpdk_home: "/usr/local/dpdk"
+dpdk_version: "2.2.0"
+dpdk_sdk: "/root/dpdk-{{ dpdk_version }}"
+dpdk_target: "x86_64-native-linuxapp-gcc"
+num_huge_pages: 512
+extra_cflags: -g
+
+# fastcapa
+fastcapa_work_dir: /root/fastcapa
+fastcapa_prefix: /usr/local/bin
+fastcapa_ld_library_path: /usr/local/lib
+fastcapa_portmask: 0x01
+fastcapa_kafka_config: /etc/fastcapa.conf
+fastcapa_bin: fastcapa
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-deployment/roles/fastcapa/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/fastcapa/meta/main.yml b/metron-deployment/roles/fastcapa/meta/main.yml
new file mode 100644
index 0000000..d253e88
--- /dev/null
+++ b/metron-deployment/roles/fastcapa/meta/main.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+dependencies:
+ - librdkafka
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-deployment/roles/fastcapa/tasks/debug.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/fastcapa/tasks/debug.yml b/metron-deployment/roles/fastcapa/tasks/debug.yml
new file mode 100644
index 0000000..06f1526
--- /dev/null
+++ b/metron-deployment/roles/fastcapa/tasks/debug.yml
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+ - name: Install debug utilities
+ yum: name=yum-utils
+ tags:
+ - debug
+
+ - name: Install debug symbols
+ shell: debuginfo-install -y glibc glib2 zlib
+ tags:
+ - debug
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-deployment/roles/fastcapa/tasks/dependencies.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/fastcapa/tasks/dependencies.yml b/metron-deployment/roles/fastcapa/tasks/dependencies.yml
new file mode 100644
index 0000000..4d6edc4
--- /dev/null
+++ b/metron-deployment/roles/fastcapa/tasks/dependencies.yml
@@ -0,0 +1,38 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+ - name: Install dependencies
+ yum: name={{ item }}
+ with_items:
+ - "@Development tools"
+ - pciutils
+ - net-tools
+ - glib2
+ - glib2-devel
+ - git
+
+ #
+ # install prerequisite packages and the latest kernel headers. need to
+ # ensure that the kernel headers match the current running kernel version.
+ # if this is not the case, the DPDK build process will fail
+ #
+ - name: Install latest kernel headers and source
+ yum: name={{ item }} state=latest
+ with_items:
+ - kernel
+ - kernel-devel
+ - kernel-headers
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-deployment/roles/fastcapa/tasks/dpdk.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/fastcapa/tasks/dpdk.yml b/metron-deployment/roles/fastcapa/tasks/dpdk.yml
new file mode 100644
index 0000000..3780be7
--- /dev/null
+++ b/metron-deployment/roles/fastcapa/tasks/dpdk.yml
@@ -0,0 +1,59 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+ - name: "Download DPDK version {{ dpdk_version }}"
+ unarchive:
+ src: "http://dpdk.org/browse/dpdk/snapshot/dpdk-{{ dpdk_version }}.tar.gz"
+ dest: "/root"
+ creates: "{{ dpdk_sdk }}"
+ copy: no
+
+ - name: "Configure DPDK for the target environment: {{ dpdk_target }}"
+ shell: "make config T={{ dpdk_target }} DESTDIR={{ dpdk_home }}"
+ args:
+ chdir: "{{ dpdk_sdk }}"
+ creates: "{{ dpdk_home }}"
+
+ - name: "Turn on debug flags"
+ lineinfile:
+ dest: "{{ dpdk_sdk }}/config/common_linuxapp"
+ regexp: 'DEBUG=n'
+ line: 'DEBUG=y'
+ tags:
+ - debug
+
+ - name: "Build DPDK for the target environment: {{ dpdk_target }}"
+ shell: "make install T={{ dpdk_target }} DESTDIR={{ dpdk_home }} EXTRA_CFLAGS={{ extra_cflags }}"
+ args:
+ chdir: "{{ dpdk_sdk }}"
+ creates: "{{ dpdk_home }}"
+
+ - name: Load kernel modules to enable userspace IO
+ shell: "{{ item }}"
+ with_items:
+ - modprobe uio_pci_generic
+ - modprobe vfio-pci
+
+ - name: Bind the device to the loaded kernel module(s)
+ shell: "{{ dpdk_home }}/sbin/dpdk_nic_bind --force --bind=uio_pci_generic {{ item }}"
+ with_items: "{{ dpdk_device }}"
+
+ - name: Set useful environment variables
+ lineinfile: "dest=/root/.bash_profile line={{ item }}"
+ with_items:
+ - "export RTE_SDK={{ dpdk_sdk }}"
+ - "export RTE_TARGET={{ dpdk_target }}"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-deployment/roles/fastcapa/tasks/fastcapa.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/fastcapa/tasks/fastcapa.yml b/metron-deployment/roles/fastcapa/tasks/fastcapa.yml
new file mode 100644
index 0000000..4e366c4
--- /dev/null
+++ b/metron-deployment/roles/fastcapa/tasks/fastcapa.yml
@@ -0,0 +1,49 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Distribute fastcapa
+ copy: src=../../../metron-sensors/fastcapa dest={{ fastcapa_work_dir | dirname }} mode=0755
+
+- name: Build fastcapa
+ shell: "{{ item }}"
+ args:
+ chdir: "{{ fastcapa_work_dir }}"
+ with_items:
+ - make
+ environment:
+ RTE_SDK: "{{ dpdk_sdk }}"
+ RTE_TARGET: "{{ dpdk_target }}"
+ LD_LIBRARY_PATH: "{{ fastcapa_ld_library_path }}"
+
+- name: Install fastcapa
+ shell: "cp {{ fastcapa_work_dir }}/src/build/app/{{ fastcapa_bin }} {{ fastcapa_prefix }}"
+ args:
+ chdir: "{{ fastcapa_work_dir }}"
+ creates: "{{ fastcapa_prefix }}/{{ fastcapa_bin }}"
+
+- name: Deploy configuration
+ template: src=fastcapa.conf dest={{ fastcapa_kafka_config }} mode=0755
+
+- name: Deploy service
+ template: src=fastcapa dest=/etc/init.d/ mode=0755
+
+- name: Register the service with systemd
+ shell: systemctl enable fastcapa
+ when: ansible_distribution == "CentOS" and ansible_distribution_major_version == "7"
+
+- name: Run fastcapa
+ service: name=fastcapa state=restarted
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-deployment/roles/fastcapa/tasks/kernel.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/fastcapa/tasks/kernel.yml b/metron-deployment/roles/fastcapa/tasks/kernel.yml
new file mode 100644
index 0000000..cd4abe6
--- /dev/null
+++ b/metron-deployment/roles/fastcapa/tasks/kernel.yml
@@ -0,0 +1,51 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+# DPDK requires specific kernel boot parameters. set the params and reboot
+# the host, if the actual params differ from what is expected.
+#
+---
+ - set_fact:
+ expected_kernel_params: "default_hugepagesz=1G hugepagesz=1G hugepages={{ num_huge_pages }} iommu=pt intel_iommu=on"
+
+ - name: Check kernel boot parameters
+ shell: "cat /proc/cmdline"
+ register: actual_kernel_params
+
+ - name: Alter kernel boot parameters
+ lineinfile:
+ dest: /etc/default/grub
+ regexp: '^(GRUB_CMDLINE_LINUX=\"[^\"]+)\"$'
+ line: '\1 {{ expected_kernel_params }}"'
+ backrefs: yes
+ when: not expected_kernel_params in actual_kernel_params.stdout
+
+ - name: Update grub with kernel boot parameters
+ shell: /sbin/grub2-mkconfig -o /boot/grub2/grub.cfg
+ when: not expected_kernel_params in actual_kernel_params.stdout
+
+ - name: Restart for modified kernel params
+ command: shutdown -r now "modified kernel params"
+ async: 0
+ poll: 0
+ ignore_errors: true
+ when: not expected_kernel_params in actual_kernel_params.stdout
+
+ - name: Wait for reboot of '{{ inventory_hostname }}'
+ local_action: wait_for host={{ inventory_hostname }} state=started port=22 timeout=300 delay=10
+ become: false
+ when: not expected_kernel_params in actual_kernel_params.stdout
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-deployment/roles/fastcapa/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/fastcapa/tasks/main.yml b/metron-deployment/roles/fastcapa/tasks/main.yml
new file mode 100644
index 0000000..b98d557
--- /dev/null
+++ b/metron-deployment/roles/fastcapa/tasks/main.yml
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+ - include: dependencies.yml
+ - include: kernel.yml
+ - include: dpdk.yml
+ - include: fastcapa.yml
+ - include: debug.yml
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-deployment/roles/fastcapa/templates/fastcapa
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/fastcapa/templates/fastcapa b/metron-deployment/roles/fastcapa/templates/fastcapa
new file mode 100644
index 0000000..637317f
--- /dev/null
+++ b/metron-deployment/roles/fastcapa/templates/fastcapa
@@ -0,0 +1,94 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# fastcapa daemon
+# chkconfig: 345 20 80
+# description: Packet capture probe
+# processname: fastcapa
+#
+
+export RTE_SDK="{{ dpdk_sdk }}"
+export RTE_TARGET="{{ dpdk_target }}"
+export LD_LIBRARY_PATH="{{ fastcapa_ld_library_path }}"
+
+NAME="fastcapa"
+DESC="Metron network packet capture probe"
+PIDFILE=/var/run/$NAME.pid
+SCRIPTNAME=/etc/init.d/$NAME
+DAEMONLOG=/var/log/$NAME.log
+NOW=`date`
+DAEMON_PATH="{{ dpdk_sdk }}"
+
+case "$1" in
+ start)
+ printf "%-50s" "Starting $NAME..."
+ echo "$NOW: Starting $NAME..." >> $DAEMONLOG
+
+ cd $DAEMON_PATH
+ DAEMON="{{ fastcapa_prefix }}/{{ fastcapa_bin }}"
+ DAEMONOPTS+=" -- "
+ DAEMONOPTS+="-p {{ fastcapa_portmask }} "
+ DAEMONOPTS+="-t {{ fastcapa_topic }} "
+ DAEMONOPTS+="-c {{ fastcapa_kafka_config }} "
+
+ PID=`$DAEMON $DAEMONOPTS >> $DAEMONLOG 2>&1 & echo $!`
+ if [ -z $PID ]; then
+ printf "%s\n" "Fail"
+ else
+ echo $PID > $PIDFILE
+ printf "%s\n" "Ok"
+ fi
+ ;;
+
+ status)
+ printf "%-50s" "Checking $NAME..."
+ if [ -f $PIDFILE ]; then
+ PID=`cat $PIDFILE`
+ if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then
+ printf "%s\n" "Process dead but pidfile exists"
+ else
+ echo "Running"
+ fi
+ else
+ printf "%s\n" "Service not running"
+ fi
+ ;;
+
+ stop)
+ printf "%-50s" "Stopping $NAME"
+ PID=`cat $PIDFILE`
+ cd $DAEMON_PATH
+ if [ -f $PIDFILE ]; then
+ echo "$NOW: Stopping $NAME with pid=$PID" >> $DAEMONLOG
+ kill -HUP $PID
+ printf "%s\n" "Ok"
+ rm -f $PIDFILE
+ else
+ printf "%s\n" "pidfile not found"
+ fi
+ ;;
+
+ restart)
+ $0 stop
+ $0 start
+ ;;
+
+ *)
+ echo "Usage: $0 {status|start|stop|restart}"
+ exit 1
+esac
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-deployment/roles/fastcapa/templates/fastcapa.conf
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/fastcapa/templates/fastcapa.conf b/metron-deployment/roles/fastcapa/templates/fastcapa.conf
new file mode 100644
index 0000000..6f6a89f
--- /dev/null
+++ b/metron-deployment/roles/fastcapa/templates/fastcapa.conf
@@ -0,0 +1,67 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# kafka global settings
+#
+[kafka-global]
+
+# initial list of kafka brokers
+metadata.broker.list = {{ kafka_broker_url }}
+
+# identifies the client to kafka
+client.id = metron-fastcapa
+
+# max number of messages allowed on the producer queue
+queue.buffering.max.messages = 1000
+
+# maximum time, in milliseconds, for buffering data on the producer queue
+queue.buffering.max.ms = 3000
+
+# compression codec = none, gzip or snappy
+compression.codec = snappy
+
+# maximum number of messages batched in one MessageSet (increase for better compression)
+batch.num.messages = 10
+
+# max times to retry sending a failed message set
+message.send.max.retries = 5
+
+# backoff time before retrying a message send
+retry.backoff.ms = 250
+
+# how often statistics are emitted; 0 = never
+statistics.interval.ms = 0
+
+# only provide delivery reports for failed messages
+delivery.report.only.error = false
+
+#
+# kafka topic settings
+#
+[kafka-topic]
+
+# broker acks { 1 = leader ack, 0 = no acks, -1 = in sync replica ack }
+request.required.acks = 1
+
+# local message timeout. This value is only enforced locally and limits the time a
+# produced message waits for successful delivery. A time of 0 is infinite.
+message.timeout.ms = 10000
+
+# report offset of produced message back to application. The application must be
+# use the dr_msg_cb to retrieve the offset from rd_kafka_message_t.offset
+produce.offset.report = false
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-deployment/roles/pcap_replay/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/pcap_replay/meta/main.yml b/metron-deployment/roles/pcap_replay/meta/main.yml
index 0c47853..841d185 100644
--- a/metron-deployment/roles/pcap_replay/meta/main.yml
+++ b/metron-deployment/roles/pcap_replay/meta/main.yml
@@ -15,7 +15,3 @@
# limitations under the License.
#
---
-dependencies:
- - libselinux-python
- - build-tools
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-deployment/roles/pcap_replay/tasks/dependencies.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/pcap_replay/tasks/dependencies.yml b/metron-deployment/roles/pcap_replay/tasks/dependencies.yml
new file mode 100644
index 0000000..effe6c4
--- /dev/null
+++ b/metron-deployment/roles/pcap_replay/tasks/dependencies.yml
@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Install dependencies
+ yum: name={{ item }}
+ with_items:
+ - "@Development tools"
+ - libpcap
+ - libpcap-devel
+ - pcre
+ - pcre-devel
+ - zlib
+ - zlib-devel
+ - glib2-devel
+ register: result
+ until: result.rc == 0
+ retries: 5
+ delay: 10
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-deployment/roles/pcap_replay/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/pcap_replay/tasks/main.yml b/metron-deployment/roles/pcap_replay/tasks/main.yml
index 06919ed..bdc2a0b 100644
--- a/metron-deployment/roles/pcap_replay/tasks/main.yml
+++ b/metron-deployment/roles/pcap_replay/tasks/main.yml
@@ -15,5 +15,6 @@
# limitations under the License.
#
---
+- include: dependencies.yml
- include: tcpreplay.yml
- include: service.yml
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-deployment/roles/sensor-test-mode/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/sensor-test-mode/defaults/main.yml b/metron-deployment/roles/sensor-test-mode/defaults/main.yml
new file mode 100644
index 0000000..46c9750
--- /dev/null
+++ b/metron-deployment/roles/sensor-test-mode/defaults/main.yml
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+pcap_replay: True
+install_yaf: True
+install_snort: True
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-deployment/roles/sensor-test-mode/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/sensor-test-mode/tasks/main.yml b/metron-deployment/roles/sensor-test-mode/tasks/main.yml
index 26b4e2a..26b72c9 100644
--- a/metron-deployment/roles/sensor-test-mode/tasks/main.yml
+++ b/metron-deployment/roles/sensor-test-mode/tasks/main.yml
@@ -15,42 +15,9 @@
# limitations under the License.
#
---
-#
-# load example pcap data to replay
-#
-- name: Install example pcap file
- copy: src=example.pcap dest={{ pcap_path }}/
-
-- name: Start the pcap-replay service
- service: name=pcap-replay state=restarted
-
-#
-# configure yaf to generate a flow record for every packet
-#
-- name: Stop running instances of yaf
- become: True
- service: name=yaf state=stopped
-
-- name: Configure yaf to generate a flow record for every network packet
- become: True
- service: name=yaf state=started args="--idle-timeout 0"
-
-#
-# configure snort to alert on every packet
-#
-- name: Configure snort to use a set of test rules
- become: True
- lineinfile:
- dest: /etc/snort/snort.conf
- line: "include $RULE_PATH/test.rules"
-
-- name: Create a snort alert for testing that alerts on every packet
- become: True
- lineinfile:
- dest: /etc/snort/rules/test.rules
- line: "alert tcp any any -> any any (msg:'snort test alert'; sid:999158; )"
- create: yes
-
-- name: Restart snort
- become: True
- service: name=snortd state=restarted
+- include: pcap.yml
+ when: pcap_replay
+- include: yaf.yml
+ when: install_yaf
+- include: snort.yml
+ when: install_snort
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-deployment/roles/sensor-test-mode/tasks/pcap.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/sensor-test-mode/tasks/pcap.yml b/metron-deployment/roles/sensor-test-mode/tasks/pcap.yml
new file mode 100644
index 0000000..b088d5a
--- /dev/null
+++ b/metron-deployment/roles/sensor-test-mode/tasks/pcap.yml
@@ -0,0 +1,25 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+#
+# load example pcap data to replay
+#
+- name: Install example pcap file
+ copy: src=example.pcap dest={{ pcap_path }}/
+
+- name: Start the pcap-replay service
+ service: name=pcap-replay state=restarted
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-deployment/roles/sensor-test-mode/tasks/snort.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/sensor-test-mode/tasks/snort.yml b/metron-deployment/roles/sensor-test-mode/tasks/snort.yml
new file mode 100644
index 0000000..5fd57ff
--- /dev/null
+++ b/metron-deployment/roles/sensor-test-mode/tasks/snort.yml
@@ -0,0 +1,36 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+#
+# configure snort to alert on every packet
+#
+- name: Configure snort to use a set of test rules
+ become: True
+ lineinfile:
+ dest: /etc/snort/snort.conf
+ line: "include $RULE_PATH/test.rules"
+
+- name: Create a snort alert for testing that alerts on every packet
+ become: True
+ lineinfile:
+ dest: /etc/snort/rules/test.rules
+ line: "alert tcp any any -> any any (msg:'snort test alert'; sid:999158; )"
+ create: yes
+
+- name: Restart snort
+ become: True
+ service: name=snortd state=restarted
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-deployment/roles/sensor-test-mode/tasks/yaf.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/sensor-test-mode/tasks/yaf.yml b/metron-deployment/roles/sensor-test-mode/tasks/yaf.yml
new file mode 100644
index 0000000..5fa29ce
--- /dev/null
+++ b/metron-deployment/roles/sensor-test-mode/tasks/yaf.yml
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+#
+# configure yaf to generate a flow record for every packet
+#
+- name: Stop running instances of yaf
+ become: True
+ service: name=yaf state=stopped
+
+- name: Configure yaf to generate a flow record for every network packet
+ become: True
+ service: name=yaf state=started args="--idle-timeout 0"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-deployment/vagrant/fastcapa-vagrant/.gitignore
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/fastcapa-vagrant/.gitignore b/metron-deployment/vagrant/fastcapa-vagrant/.gitignore
new file mode 100644
index 0000000..a8b42eb
--- /dev/null
+++ b/metron-deployment/vagrant/fastcapa-vagrant/.gitignore
@@ -0,0 +1 @@
+*.retry
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-deployment/vagrant/fastcapa-vagrant/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/fastcapa-vagrant/README.md b/metron-deployment/vagrant/fastcapa-vagrant/README.md
new file mode 100644
index 0000000..9bb0a39
--- /dev/null
+++ b/metron-deployment/vagrant/fastcapa-vagrant/README.md
@@ -0,0 +1,73 @@
+Fastcapa Test Environment
+=========================
+
+Provides a test environment for the development and testing of Fastcapa. The environment is automatically validated after it is created to ensure that Fastcapa is behaving correctly.
+
+Two virtualized nodes are launched with Vagrant that can communicate with one another over a private network.
+- The `source` node uses Metron's `pcap_replay` functionality to transmit raw network packet data over a private network.
+- The `sink` node is running `fastcapa` and is capturing these network packets.
+- Fastcapa then transforms and bundles the packets into a message.
+- The message is sent to a Kafka broker running on the `source` node.
+
+Getting Started
+---------------
+
+Simply run `vagrant up` to launch the environment. Automated tests are executed after provisioning completes to ensure that Fastcapa and the rest of the environment is functioning properly.
+
+```
+$ vagrant up
+==> source: Running provisioner: ansible...
+ source: Running ansible-playbook...
+...
+TASK [debug] *******************************************************************
+ok: [source] => {
+ "msg": "Successfully received packets sent from pcap-replay!"
+}
+...
+TASK [debug] *******************************************************************
+ok: [source] => {
+ "msg": "Successfully received a Kafka message from fastcapa!"
+}
+```
+
+Going Deeper
+------------
+
+This section will outline in more detail the environment and how to interact with it.
+
+### `source`
+
+To validate that the `source` node is functioning properly, run the following commands.
+
+First, ensure that the `pcap-replay` service is running.
+
+```
+vagrant ssh source
+sudo service pcap-replay status
+```
+
+Use `tcpdump` to ensure that the raw packet data is being sent over the private network. Enter 'CTRL-C' to kill the `tcpdump` process once you are able to see that packets are being sent.
+
+```
+sudo yum -y install tcpdump
+sudo tcpdump -i enp0s8
+```
+
+### `sink`
+
+Next validate that the `sink` is functioning properly. Run the following commands starting from the host operating system.
+
+First, ensure that the `fastcapa` service is running.
+
+```
+vagrant ssh sink
+service fastcapa status
+```
+
+Ensure that the raw network packet data is being received by Kafka.
+
+```
+/usr/hdp/current/kafka-broker/bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic pcap
+```
+
+Enter 'CTRL-C' to kill the `kafka-console-consumer` process once you are able to see that packets are being sent. These packets will appear to be gibberish in the console. This is the raw binary network packet data after all.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-deployment/vagrant/fastcapa-vagrant/Vagrantfile
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/fastcapa-vagrant/Vagrantfile b/metron-deployment/vagrant/fastcapa-vagrant/Vagrantfile
new file mode 100644
index 0000000..7f58417
--- /dev/null
+++ b/metron-deployment/vagrant/fastcapa-vagrant/Vagrantfile
@@ -0,0 +1,71 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+Vagrant.configure("2") do |config|
+
+ # enable hostmanager
+ config.hostmanager.enabled = true
+ config.hostmanager.manage_host = true
+
+ #
+ # source
+ #
+ config.vm.define "source" do |node|
+
+ # host settings
+ node.vm.hostname = "source"
+ node.vm.box = "bento/centos-6.7"
+ node.ssh.insert_key = "true"
+ node.vm.network :private_network, ip: "192.168.33.10", netmask: "255.255.255.0"
+
+ # provider
+ node.vm.provider "virtualbox" do |vb|
+ vb.memory = 1024
+ vb.cpus = 1
+ end
+ end
+
+ #
+ # sink
+ #
+ config.vm.define "sink" do |node|
+
+ # host settings
+ node.vm.hostname = "sink"
+ node.vm.box = "bento/centos-7.1"
+ node.ssh.insert_key = "true"
+ node.vm.network :private_network, ip: "192.168.33.11", netmask: "255.255.255.0"
+
+ # provider
+ node.vm.provider "virtualbox" do |vb|
+ vb.memory = 4096
+ vb.cpus = 3
+
+ # network adapter settings; [Am79C970A|Am79C973|82540EM|82543GC|82545EM|virtio]
+ vb.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
+ vb.customize ["modifyvm", :id, "--nictype2","82545EM"]
+ end
+
+ # provision host
+ node.vm.provision :ansible do |ansible|
+ ansible.limit = "all"
+ ansible.playbook = "playbook.yml"
+ end
+ end
+
+
+end
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-deployment/vagrant/fastcapa-vagrant/ansible.cfg
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/fastcapa-vagrant/ansible.cfg b/metron-deployment/vagrant/fastcapa-vagrant/ansible.cfg
new file mode 100644
index 0000000..9c650c2
--- /dev/null
+++ b/metron-deployment/vagrant/fastcapa-vagrant/ansible.cfg
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+[defaults]
+host_key_checking = false
+library = ../../extra_modules
+roles_path = ../../roles
+pipelining = True
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-deployment/vagrant/fastcapa-vagrant/playbook.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/fastcapa-vagrant/playbook.yml b/metron-deployment/vagrant/fastcapa-vagrant/playbook.yml
new file mode 100644
index 0000000..bd6be7c
--- /dev/null
+++ b/metron-deployment/vagrant/fastcapa-vagrant/playbook.yml
@@ -0,0 +1,50 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+#
+# the 'source' produces network traffic
+#
+- hosts: source
+ become: yes
+ vars_files:
+ - vars/main.yml
+ roles:
+ - role: kafka-broker
+ - role: pcap_replay
+ - { role: sensor-test-mode, pcap_replay: True, install_yaf: False, install_snort: False }
+
+#
+# the 'sink' consumes network traffic
+#
+- hosts: sink
+ become: yes
+ vars_files:
+ - vars/main.yml
+ roles:
+ - role: librdkafka
+ - role: fastcapa
+
+#
+# validate the environment - needs to run on `source` node
+#
+- hosts: source
+ become: yes
+ vars_files:
+ - vars/main.yml
+ tasks:
+ - include: tasks/validate-packets-sent.yml
+ - include: tasks/validate-messages-received.yml
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-deployment/vagrant/fastcapa-vagrant/tasks/validate-messages-received.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/fastcapa-vagrant/tasks/validate-messages-received.yml b/metron-deployment/vagrant/fastcapa-vagrant/tasks/validate-messages-received.yml
new file mode 100644
index 0000000..5109a0e
--- /dev/null
+++ b/metron-deployment/vagrant/fastcapa-vagrant/tasks/validate-messages-received.yml
@@ -0,0 +1,38 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- set_fact:
+ kafka_consumer_cmd: "{{ kafka_broker_home}}/bin/kafka-console-consumer.sh --zookeeper {{ zookeeper_url }} --topic {{ fastcapa_topic }} --max-messages 10 --from-beginning"
+
+- name: Attempting to receive message(s) sent by fastcapa
+ shell: "{{ kafka_consumer_cmd }}"
+ async: 30
+ register: kafka_consumer
+
+- name: Waiting to receive message(s) sent by fastcapa
+ async_status: jid={{ kafka_consumer.ansible_job_id }}
+ register: job
+ until: job.finished
+ retries: 30
+
+- fail:
+ msg: "No messages received from fastcapa within timeout: {{ kafka_consumer_cmd }}"
+ when: job.finished != 1
+
+- debug: var=job
+
+- debug: msg="Successfully received a Kafka message from fastcapa!"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-deployment/vagrant/fastcapa-vagrant/tasks/validate-packets-sent.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/fastcapa-vagrant/tasks/validate-packets-sent.yml b/metron-deployment/vagrant/fastcapa-vagrant/tasks/validate-packets-sent.yml
new file mode 100644
index 0000000..25347dd
--- /dev/null
+++ b/metron-deployment/vagrant/fastcapa-vagrant/tasks/validate-packets-sent.yml
@@ -0,0 +1,41 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Install tcpdump
+ yum: name=tcpdump
+
+- set_fact:
+ sniff_cmd: "tcpdump -i {{ pcap_replay_interface }} -c 10"
+
+- name: "Attempting to sniff packet(s)"
+ shell: "{{ sniff_cmd }}"
+ async: 30
+ register: sniffer
+
+- name: Waiting to receive packet(s)
+ async_status: jid={{ sniffer.ansible_job_id }}
+ register: job
+ until: job.finished
+ retries: 30
+
+- fail:
+ msg: "No packets received from pcap-replay within timeout: {{ sniff_cmd }}"
+ when: job.finished != 1
+
+- debug: var=job
+
+- debug: msg="Successfully received packets sent from pcap-replay!"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-deployment/vagrant/fastcapa-vagrant/vars/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/fastcapa-vagrant/vars/main.yml b/metron-deployment/vagrant/fastcapa-vagrant/vars/main.yml
new file mode 100644
index 0000000..3054a4b
--- /dev/null
+++ b/metron-deployment/vagrant/fastcapa-vagrant/vars/main.yml
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+dpdk_device: ["00:08.0"]
+dpdk_target: "x86_64-native-linuxapp-gcc"
+num_huge_pages: 512
+fastcapa_portmask: "0x01"
+fastcapa_topic: pcap
+kafka_broker_url: source:9092
+zookeeper_url: source:2181
+pcap_replay_interface: eth1
+kafka_broker_home: /usr/hdp/current/kafka-broker/
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-deployment/vagrant/packet-capture/Vagrantfile
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/packet-capture/Vagrantfile b/metron-deployment/vagrant/packet-capture/Vagrantfile
deleted file mode 100644
index 1303712..0000000
--- a/metron-deployment/vagrant/packet-capture/Vagrantfile
+++ /dev/null
@@ -1,69 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-Vagrant.configure("2") do |config|
-
- # enable hostmanager
- config.hostmanager.enabled = true
- config.hostmanager.manage_host = true
-
- #
- # source
- #
- config.vm.define "source" do |node|
-
- # host settings
- node.vm.hostname = "source"
- node.vm.box = "bento/centos-7.1"
- node.ssh.insert_key = "true"
- node.vm.network :private_network, ip: "192.168.33.10", netmask: "255.255.255.0"
-
- # provider
- node.vm.provider "virtualbox" do |vb|
- vb.memory = 1024
- vb.cpus = 1
- end
- end
-
- #
- # sink
- #
- config.vm.define "sink" do |node|
-
- # host settings
- node.vm.hostname = "sink"
- node.vm.box = "bento/centos-7.1"
- node.ssh.insert_key = "true"
- node.vm.network "public_network"
- node.vm.network :private_network, ip: "192.168.33.11", netmask: "255.255.255.0"
-
- # provider
- node.vm.provider "virtualbox" do |vb|
- vb.memory = 4096
- vb.cpus = 3
-
- # network adapter settings; [Am79C970A|Am79C973|82540EM|82543GC|82545EM|virtio]
- vb.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
- vb.customize ["modifyvm", :id, "--nictype2","82545EM"]
- end
- end
-
- # provision hosts
- config.vm.provision :ansible do |ansible|
- ansible.playbook = "playbook.yml"
- end
-end
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-deployment/vagrant/packet-capture/ansible.cfg
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/packet-capture/ansible.cfg b/metron-deployment/vagrant/packet-capture/ansible.cfg
deleted file mode 100644
index 9c650c2..0000000
--- a/metron-deployment/vagrant/packet-capture/ansible.cfg
+++ /dev/null
@@ -1,22 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-[defaults]
-host_key_checking = false
-library = ../../extra_modules
-roles_path = ../../roles
-pipelining = True
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-deployment/vagrant/packet-capture/playbook.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/packet-capture/playbook.yml b/metron-deployment/vagrant/packet-capture/playbook.yml
deleted file mode 100644
index 7a5128c..0000000
--- a/metron-deployment/vagrant/packet-capture/playbook.yml
+++ /dev/null
@@ -1,43 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-#
-# produces network traffic
-#
-- hosts: source
- become: yes
- vars:
- pcap_replay_interface: "enp0s8"
- roles:
- - role: pcap_replay
-
-#
-# consumes network traffic
-#
-- hosts: sink
- become: yes
- vars:
- dpdk_device: ["00:08.0"]
- dpdk_target: "x86_64-native-linuxapp-gcc"
- num_huge_pages: 512
- pcapture_portmask: 0xf
- pcapture_topic: pcap
- kafka_broker_url: localhost:9092
- roles:
- - role: librdkafka
- - role: kafka-broker
- - role: packet-capture
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-sensors/fastcapa/.gitignore
----------------------------------------------------------------------
diff --git a/metron-sensors/fastcapa/.gitignore b/metron-sensors/fastcapa/.gitignore
new file mode 100644
index 0000000..2efc4c0
--- /dev/null
+++ b/metron-sensors/fastcapa/.gitignore
@@ -0,0 +1,3 @@
+roles
+.vagrant
+*.retry
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-sensors/fastcapa/Makefile
----------------------------------------------------------------------
diff --git a/metron-sensors/fastcapa/Makefile b/metron-sensors/fastcapa/Makefile
new file mode 100644
index 0000000..90ecb17
--- /dev/null
+++ b/metron-sensors/fastcapa/Makefile
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+all: src
+
+src:
+ cd src; make
+
+clean:
+ cd src; make clean
+
+.PHONY: all src clean
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-sensors/fastcapa/README.md
----------------------------------------------------------------------
diff --git a/metron-sensors/fastcapa/README.md b/metron-sensors/fastcapa/README.md
new file mode 100644
index 0000000..2278bfe
--- /dev/null
+++ b/metron-sensors/fastcapa/README.md
@@ -0,0 +1,20 @@
+Fastcapa
+========
+
+Fastcapa is an Apache Metron sensor that performs fast network packet capture by leveraging Linux kernel-bypass and user space networking technology.
+
+The sensor will bind to a network interface, capture network packets, and send the raw packet data to Kafka. This provides a scalable mechanism for ingesting high-volumes of network packet data into a Hadoop-y cluster.
+
+Fastcapa leverages the Data Plane Development Kit ([DPDK](http://dpdk.org/)). DPDK is a set of libraries and drivers to perform fast packet processing in Linux user space.
+
+Getting Started
+---------------
+
+The quickest way to get up and running is to use a Virtualbox environment on your local machine. The necessary files and instructions to do this are located at [`metron-deployment/vagrant/fastcapa-vagrant`](../../metron-deployment/vagrant/fastcapa-vagrant).
+
+Installation
+------------
+
+The process of installing Fastcapa has a fair number of steps and involves building DPDK, loading specific kernel modules, enabling huge page memory, and binding compatible network interface cards.
+
+The best documentation is code that actually does this for you. An Ansible role that performs the entire installation procedure can be found at [`metron-deployment/roles/fastcapa`](../../metron-deployment/roles/fastcapa). Use this to install Fastcapa or as a guide for manual installation.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-sensors/fastcapa/conf/localhost.kafka
----------------------------------------------------------------------
diff --git a/metron-sensors/fastcapa/conf/localhost.kafka b/metron-sensors/fastcapa/conf/localhost.kafka
new file mode 100644
index 0000000..b50c20e
--- /dev/null
+++ b/metron-sensors/fastcapa/conf/localhost.kafka
@@ -0,0 +1,67 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# kafka global settings
+#
+[kafka-global]
+
+# identifies the client to kafka
+client.id = pcap-localhost
+
+# initial list of kafka brokers
+metadata.broker.list = localhost:9092
+
+# max number of messages allowed on the producer queue
+queue.buffering.max.messages = 10000000
+
+# maximum time, in milliseconds, for buffering data on the producer queue
+queue.buffering.max.ms = 1000
+
+# compression codec = none, gzip or snappy
+compression.codec = snappy
+
+# maximum number of messages batched in one MessageSet (increase for better compression)
+batch.num.messages = 1000
+
+# max times to retry sending a failed message set
+message.send.max.retries = 10
+
+# backoff time before retrying a message send
+retry.backoff.ms = 250
+
+# how often statistics are emitted; 0 = never
+statistics.interval.ms = 0
+
+# only provide delivery reports for failed messages
+delivery.report.only.error = false
+
+#
+# kafka topic settings
+#
+[kafka-topic]
+
+# broker acks { 1 = leader ack, 0 = no acks, -1 = in sync replica ack }
+request.required.acks = 1
+
+# local message timeout. This value is only enforced locally and limits the time a
+# produced message waits for successful delivery. A time of 0 is infinite.
+message.timeout.ms = 300000
+
+# report offset of produced message back to application. The application must be
+# use the dr_msg_cb to retrieve the offset from rd_kafka_message_t.offset
+produce.offset.report = true
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-sensors/fastcapa/src/.gitignore
----------------------------------------------------------------------
diff --git a/metron-sensors/fastcapa/src/.gitignore b/metron-sensors/fastcapa/src/.gitignore
new file mode 100644
index 0000000..378eac2
--- /dev/null
+++ b/metron-sensors/fastcapa/src/.gitignore
@@ -0,0 +1 @@
+build
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-sensors/fastcapa/src/Makefile
----------------------------------------------------------------------
diff --git a/metron-sensors/fastcapa/src/Makefile b/metron-sensors/fastcapa/src/Makefile
new file mode 100644
index 0000000..9e7849d
--- /dev/null
+++ b/metron-sensors/fastcapa/src/Makefile
@@ -0,0 +1,52 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+ifeq ($(RTE_SDK),)
+$(error "Please define RTE_SDK environment variable")
+endif
+
+# Default target, can be overriden by command line or environment
+RTE_TARGET ?= x86_64-native-linuxapp-gcc
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+# binary name
+APP = fastcapa
+
+# all source are stored in SRCS-y
+SRCS-y := main.c args.c kafka.c
+
+KAFKALIB = -L/usr/local/lib -lrdkafka
+KAFKAINC = -I/usr/local/include/librdkafka/
+
+GLIB = $(shell pkg-config --libs glib-2.0) -lgthread-2.0
+GLIBINC = $(shell pkg-config --cflags glib-2.0)
+
+LDLIBS += $(KAFKALIB) $(GLIB)
+CFLAGS += $(WERROR_FLAGS) $(KAFKAINC) $(GLIBINC)
+
+# workaround for a gcc bug with noreturn attribute
+# http://gcc.gnu.org/bugzilla/show_bug.cgi?id=12603
+ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y)
+CFLAGS_main.o += -Wno-return-type
+endif
+
+#EXTRA_CFLAGS += -O3 -Wfatal-errors
+EXTRA_CFLAGS += -g -Wall
+
+include $(RTE_SDK)/mk/rte.extapp.mk
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-sensors/fastcapa/src/args.c
----------------------------------------------------------------------
diff --git a/metron-sensors/fastcapa/src/args.c b/metron-sensors/fastcapa/src/args.c
new file mode 100644
index 0000000..04fb8d0
--- /dev/null
+++ b/metron-sensors/fastcapa/src/args.c
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "args.h"
+
+typedef int bool;
+#define true 1
+#define false 0
+#define valid(s) (s == NULL ? false : strlen(s) > 1)
+
+/*
+ * Print usage information to the user.
+ */
+void print_usage(const char* prgname)
+{
+ printf("%s [EAL options] -- [APP options]\n"
+ " -p PORTMASK hex bitmask of ports to bind [0x01]\n"
+ " -t KAFKATOPIC kafka topic [pcap]\n"
+ " -c KAFKACONF kafka config file [conf/kafka.conf]\n",
+ prgname);
+}
+
+/*
+ * Parse the 'portmask' command line argument.
+ */
+int parse_portmask(const char* portmask)
+{
+ char* end = NULL;
+ unsigned long pm;
+
+ // parse hexadecimal string
+ pm = strtoul(portmask, &end, 16);
+
+ if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0')) {
+ return -1;
+ }
+ else if (pm == 0) {
+ return -1;
+ }
+ else {
+ return pm;
+ }
+}
+
+/*
+ * Check if a file exists
+ */
+static bool file_exists(const char* filepath)
+{
+ struct stat buf;
+ return (stat(filepath, &buf) == 0);
+}
+
+/**
+ * Parse the command line arguments passed to the application.
+ */
+int parse_args(int argc, char** argv)
+{
+ int opt;
+ char** argvopt;
+ int option_index;
+ char* prgname = argv[0];
+ static struct option lgopts[] = {
+ { NULL, 0, 0, 0 }
+ };
+
+ // initialize args
+ memset(&app, 0, sizeof(struct app_params));
+
+ // parse arguments to this application
+ argvopt = argv;
+ while ((opt = getopt_long(argc, argvopt, "p:b:t:c:", lgopts, &option_index)) != EOF) {
+ switch (opt) {
+
+ // portmask
+ case 'p':
+ app.enabled_port_mask = parse_portmask(optarg);
+ if (app.enabled_port_mask == 0) {
+ printf("Error: Invalid portmask: '%s'\n", optarg);
+ print_usage(prgname);
+ return -1;
+ }
+ break;
+
+ // kafka topic
+ case 't':
+ app.kafka_topic = strdup(optarg);
+ if (!valid(app.kafka_topic)) {
+ printf("Error: Invalid kafka topic: '%s'\n", optarg);
+ print_usage(prgname);
+ return -1;
+ }
+ break;
+
+ // kafka config path
+ case 'c':
+ app.kafka_config_path = strdup(optarg);
+ if (!valid(app.kafka_config_path) || !file_exists(app.kafka_config_path)) {
+ printf("Error: Invalid kafka config: '%s'\n", optarg);
+ print_usage(prgname);
+ return -1;
+ }
+ break;
+
+ default:
+ printf("Error: Invalid argument: '%s'\n", optarg);
+ print_usage(prgname);
+ return -1;
+ }
+ }
+
+ // check for required command-line arguments
+ if (app.enabled_port_mask == 0) {
+ printf("Error: Missing -p PORTMASK\n");
+ print_usage(prgname);
+ return -1;
+ }
+
+ if (!valid(app.kafka_topic)) {
+ printf("Error: Missing -t KAFKATOPIC\n");
+ print_usage(prgname);
+ return -1;
+ }
+
+ argv[optind - 1] = prgname;
+
+ // reset getopt lib
+ optind = 0;
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-sensors/fastcapa/src/args.h
----------------------------------------------------------------------
diff --git a/metron-sensors/fastcapa/src/args.h b/metron-sensors/fastcapa/src/args.h
new file mode 100644
index 0000000..36dde29
--- /dev/null
+++ b/metron-sensors/fastcapa/src/args.h
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef METRON_ARGS_H
+#define METRON_ARGS_H
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <inttypes.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/queue.h>
+#include <sys/stat.h>
+#include <stdarg.h>
+#include <errno.h>
+#include <getopt.h>
+#include <glib.h>
+#include <rte_common.h>
+#include <rte_byteorder.h>
+#include <rte_log.h>
+#include <rte_memory.h>
+#include <rte_memcpy.h>
+#include <rte_memzone.h>
+#include <rte_eal.h>
+#include <rte_per_lcore.h>
+#include <rte_launch.h>
+#include <rte_atomic.h>
+#include <rte_cycles.h>
+#include <rte_prefetch.h>
+#include <rte_lcore.h>
+#include <rte_per_lcore.h>
+#include <rte_branch_prediction.h>
+#include <rte_interrupts.h>
+#include <rte_pci.h>
+#include <rte_random.h>
+#include <rte_debug.h>
+#include <rte_ether.h>
+#include <rte_ethdev.h>
+#include <rte_ring.h>
+#include <rte_mempool.h>
+#include <rte_mbuf.h>
+#include <rte_ip.h>
+#include <rte_tcp.h>
+#include <rte_lpm.h>
+#include <rte_string_fns.h>
+
+/*
+ * Logging definitions
+ */
+#define LOG_ERROR(log_type, fmt, args...) RTE_LOG(ERR, log_type, fmt, ##args);
+#define LOG_WARN(log_type, fmt, args...) RTE_LOG(WARNING, log_type, fmt, ##args);
+#define LOG_INFO(log_type, fmt, args...) RTE_LOG(INFO, log_type, fmt, ##args);
+
+#ifdef DEBUG
+#define LOG_LEVEL RTE_LOG_DEBUG
+#define LOG_DEBUG(log_type, fmt, args...) RTE_LOG(DEBUG, log_type, fmt, ##args);
+#else
+#define LOG_LEVEL RTE_LOG_INFO
+#define LOG_DEBUG(log_type, fmt, args...) do {} while (0)
+#endif
+
+/**
+ * Application configuration parameters.
+ */
+struct app_params {
+ uint32_t enabled_port_mask;
+ char* kafka_topic;
+ char* kafka_config_path;
+} __rte_cache_aligned;
+
+/*
+ * Contains all application parameters.
+ */
+struct app_params app;
+
+/*
+ * Print usage information to the user.
+ */
+void print_usage(const char* prgname);
+
+/*
+ * Parse the 'portmask' command line argument.
+ */
+int parse_portmask(const char* portmask);
+
+/**
+ * Parse the command line arguments passed to the application.
+ */
+int parse_args(int argc, char** argv);
+
+#endif
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-sensors/fastcapa/src/kafka.c
----------------------------------------------------------------------
diff --git a/metron-sensors/fastcapa/src/kafka.c b/metron-sensors/fastcapa/src/kafka.c
new file mode 100644
index 0000000..608e308
--- /dev/null
+++ b/metron-sensors/fastcapa/src/kafka.c
@@ -0,0 +1,224 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "kafka.h"
+
+#define POLL_TIMEOUT_MS 1000
+
+/*
+ * data structures required for the kafka client
+ */
+static rd_kafka_t** kaf_h;
+static rd_kafka_topic_t** kaf_top_h;
+static int num_conns;
+
+/**
+ * A callback executed for each global Kafka option.
+ */
+static void kaf_global_option(const char* key, const char* val, void* arg)
+{
+ rd_kafka_conf_t* conf = (rd_kafka_conf_t*)arg;
+ rd_kafka_conf_res_t rc;
+ char err[512];
+
+ rc = rd_kafka_conf_set(conf, key, val, err, sizeof(err));
+ if (RD_KAFKA_CONF_OK != rc) {
+ LOG_WARN(USER1, "unable to set kafka global option: '%s' = '%s': %s\n", key, val, err);
+ }
+}
+
+/**
+ * A callback executed for topic-level Kafka option.
+ */
+static void kaf_topic_option(const char* key, const char* val, void* arg)
+{
+ rd_kafka_topic_conf_t* conf = (rd_kafka_topic_conf_t*)arg;
+ rd_kafka_conf_res_t rc;
+ char err[512];
+
+ rc = rd_kafka_topic_conf_set(conf, key, val, err, sizeof(err));
+ if (RD_KAFKA_CONF_OK != rc) {
+ LOG_WARN(USER1, "unable to set kafka topic option: '%s' = '%s': %s\n", key, val, err);
+ }
+}
+
+/**
+ * Parses the configuration values from a configuration file.
+ */
+static void parse_kafka_config(char* file_path, const char* group,
+ void (*option_cb)(const char* key, const char* val, void* arg), void* arg)
+{
+
+ gsize i;
+ gchar* value;
+ gchar** keys;
+ gsize num_keys;
+ GError* err = NULL;
+ GError** errs = NULL;
+
+ // load the configuration file
+ GKeyFile* gkf = g_key_file_new();
+ if (!g_key_file_load_from_file(gkf, file_path, G_KEY_FILE_NONE, &err)) {
+ LOG_ERROR(USER1, "bad config: %s: %s\n", file_path, err->message);
+ }
+
+ // only grab keys within the specified group
+ keys = g_key_file_get_keys(gkf, group, &num_keys, errs);
+ if (keys) {
+
+ // execute the callback for each key/value
+ for (i = 0; i < num_keys; i++) {
+ value = g_key_file_get_value(gkf, group, keys[i], errs);
+ if (value) {
+ LOG_DEBUG(USER1, "config[%s]: %s = %s\n", group, keys[i], value);
+ option_cb(keys[i], value, arg);
+ }
+ else {
+ LOG_INFO(USER1, "bad config: %s: %s = %s: %s\n", file_path, keys[i], value, errs[0]->message);
+ }
+ }
+ }
+ else {
+ LOG_WARN(USER1, "bad config: %s: %s\n", file_path, errs[0]->message);
+ }
+
+ g_strfreev(keys);
+ g_key_file_free(gkf);
+}
+
+/**
+ * Initializes a pool of Kafka connections.
+ */
+void kaf_init(int num_of_conns)
+{
+ int i;
+ char errstr[512];
+
+ // the number of connections to maintain
+ num_conns = num_of_conns;
+
+ // create kafka resources for each consumer
+ kaf_h = calloc(num_of_conns, sizeof(rd_kafka_t*));
+ kaf_top_h = calloc(num_of_conns, sizeof(rd_kafka_topic_t*));
+
+ for (i = 0; i < num_of_conns; i++) {
+
+ // configure kafka connection; values parsed from kafka config file
+ rd_kafka_conf_t* kaf_conf = rd_kafka_conf_new();
+ if (NULL != app.kafka_config_path) {
+ parse_kafka_config(app.kafka_config_path, "kafka-global", kaf_global_option, (void*)kaf_conf);
+ }
+
+ // create a new kafka connection
+ kaf_h[i] = rd_kafka_new(RD_KAFKA_PRODUCER, kaf_conf, errstr, sizeof(errstr));
+ if (!kaf_h[i]) {
+ rte_exit(EXIT_FAILURE, "Cannot init kafka connection: %s", errstr);
+ }
+
+ // configure kafka topic; values parsed from kafka config file
+ rd_kafka_topic_conf_t* topic_conf = rd_kafka_topic_conf_new();
+ if (NULL != app.kafka_config_path) {
+ parse_kafka_config(app.kafka_config_path, "kafka-topic", kaf_topic_option, (void*)topic_conf);
+ }
+
+ // connect to a kafka topic
+ kaf_top_h[i] = rd_kafka_topic_new(kaf_h[i], app.kafka_topic, topic_conf);
+ if (!kaf_top_h[i]) {
+ rte_exit(EXIT_FAILURE, "Cannot init kafka topic: %s", app.kafka_topic);
+ }
+ }
+}
+
+/**
+ * Closes the pool of Kafka connections.
+ */
+void kaf_close(void)
+{
+ int i;
+ for (i = 0; i < num_conns; i++) {
+ // wait for messages to be delivered
+ while (rd_kafka_outq_len(kaf_h[i]) > 0) {
+ LOG_INFO(USER1, "waiting for %d messages to clear on conn [%i/%i]",
+ rd_kafka_outq_len(kaf_h[i]), i + 1, num_conns);
+ rd_kafka_poll(kaf_h[i], POLL_TIMEOUT_MS);
+ }
+
+ rd_kafka_topic_destroy(kaf_top_h[i]);
+ rd_kafka_destroy(kaf_h[i]);
+ }
+}
+
+/**
+ * The current time in microseconds.
+ */
+static uint64_t current_time(void)
+{
+ struct timeval tv;
+ gettimeofday(&tv, NULL);
+ return tv.tv_sec * (uint64_t)1000000 + tv.tv_usec;
+}
+
+/**
+ * Publish a set of packets to a kafka topic.
+ */
+int kaf_send(struct rte_mbuf* data, int pkt_count, int conn_id)
+{
+ // unassigned partition
+ int partition = RD_KAFKA_PARTITION_UA;
+ int i;
+ int pkts_sent = 0;
+ int drops;
+ rd_kafka_message_t kaf_msgs[pkt_count];
+
+ // TODO: ensure that librdkafka cleans this up for us
+ uint64_t *now = malloc(sizeof(uint64_t));
+
+ // the current time in microseconds from the epoch (in big-endian aka network
+ // byte order) is added as a message key before being sent to kafka
+ *now = htobe64(current_time());
+
+ // find the topic connection based on the conn_id
+ rd_kafka_topic_t* kaf_topic = kaf_top_h[conn_id];
+
+ // create the batch message for kafka
+ for (i = 0; i < pkt_count; i++) {
+ kaf_msgs[i].err = 0;
+ kaf_msgs[i].rkt = kaf_topic;
+ kaf_msgs[i].partition = partition;
+ kaf_msgs[i].payload = rte_ctrlmbuf_data(&data[i]);
+ kaf_msgs[i].len = rte_ctrlmbuf_len(&data[i]);
+ kaf_msgs[i].key = (void*) now;
+ kaf_msgs[i].key_len = sizeof(uint64_t);
+ kaf_msgs[i].offset = 0;
+ }
+
+ // hand all of the messages off to kafka
+ pkts_sent = rd_kafka_produce_batch(kaf_topic, partition, RD_KAFKA_MSG_F_COPY, kaf_msgs, pkt_count);
+
+ // did we drop packets?
+ drops = pkt_count - pkts_sent;
+ if (drops > 0) {
+ for (i = 0; i < pkt_count; i++) {
+ if (!kaf_msgs[i].err) {
+ LOG_ERROR(USER1, "'%d' packets dropped, first error: %s", drops, (char*)kaf_msgs[i].payload);
+ }
+ }
+ }
+
+ return pkts_sent;
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/61aced79/metron-sensors/fastcapa/src/kafka.h
----------------------------------------------------------------------
diff --git a/metron-sensors/fastcapa/src/kafka.h b/metron-sensors/fastcapa/src/kafka.h
new file mode 100644
index 0000000..8b6bc78
--- /dev/null
+++ b/metron-sensors/fastcapa/src/kafka.h
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef METRON_KAFKA_H
+#define METRON_KAFKA_H
+
+#include <string.h>
+#include <sys/time.h>
+#include <endian.h>
+#include <librdkafka/rdkafka.h>
+#include <rte_common.h>
+#include <rte_mbuf.h>
+#include "args.h"
+
+/**
+ * Initializes a pool of Kafka connections.
+ */
+void kaf_init(int num_of_conns);
+
+/**
+ * Publish a set of packets to a kafka topic.
+ */
+int kaf_send(struct rte_mbuf* data, int num_to_send, int conn_id);
+
+/**
+ * Closes the pool of Kafka connections.
+ */
+void kaf_close(void);
+
+#endif