You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ni...@apache.org on 2016/04/13 16:48:28 UTC
[2/3] incubator-metron git commit: METRON-108 Create Fast Packet
Capture Process (nickwallen) closes apache/incubator-metron#73
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/packet-capture/tasks/dependencies.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/packet-capture/tasks/dependencies.yml b/deployment/roles/packet-capture/tasks/dependencies.yml
new file mode 100644
index 0000000..4d6edc4
--- /dev/null
+++ b/deployment/roles/packet-capture/tasks/dependencies.yml
@@ -0,0 +1,38 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+ - name: Install dependencies
+ yum: name={{ item }}
+ with_items:
+ - "@Development tools"
+ - pciutils
+ - net-tools
+ - glib2
+ - glib2-devel
+ - git
+
+ #
+ # install prerequisite packages and the latest kernel headers. need to
+ # ensure that the kernel headers match the current running kernel version.
+ # if this is not the case, the DPDK build process will fail
+ #
+ - name: Install latest kernel headers and source
+ yum: name={{ item }} state=latest
+ with_items:
+ - kernel
+ - kernel-devel
+ - kernel-headers
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/packet-capture/tasks/dpdk.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/packet-capture/tasks/dpdk.yml b/deployment/roles/packet-capture/tasks/dpdk.yml
new file mode 100644
index 0000000..3780be7
--- /dev/null
+++ b/deployment/roles/packet-capture/tasks/dpdk.yml
@@ -0,0 +1,59 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+ - name: "Download DPDK version {{ dpdk_version }}"
+ unarchive:
+ src: "http://dpdk.org/browse/dpdk/snapshot/dpdk-{{ dpdk_version }}.tar.gz"
+ dest: "/root"
+ creates: "{{ dpdk_sdk }}"
+ copy: no
+
+ - name: "Configure DPDK for the target environment: {{ dpdk_target }}"
+ shell: "make config T={{ dpdk_target }} DESTDIR={{ dpdk_home }}"
+ args:
+ chdir: "{{ dpdk_sdk }}"
+ creates: "{{ dpdk_home }}"
+
+ - name: "Turn on debug flags"
+ lineinfile:
+ dest: "{{ dpdk_sdk }}/config/common_linuxapp"
+ regexp: 'DEBUG=n'
+ line: 'DEBUG=y'
+ tags:
+ - debug
+
+ - name: "Build DPDK for the target environment: {{ dpdk_target }}"
+ shell: "make install T={{ dpdk_target }} DESTDIR={{ dpdk_home }} EXTRA_CFLAGS={{ extra_cflags }}"
+ args:
+ chdir: "{{ dpdk_sdk }}"
+ creates: "{{ dpdk_home }}"
+
+ - name: Load kernel modules to enable userspace IO
+ shell: "{{ item }}"
+ with_items:
+ - modprobe uio_pci_generic
+ - modprobe vfio-pci
+
+ - name: Bind the device to the loaded kernel module(s)
+ shell: "{{ dpdk_home }}/sbin/dpdk_nic_bind --force --bind=uio_pci_generic {{ item }}"
+ with_items: "{{ dpdk_device }}"
+
+ - name: Set useful environment variables
+ lineinfile: "dest=/root/.bash_profile line={{ item }}"
+ with_items:
+ - "export RTE_SDK={{ dpdk_sdk }}"
+ - "export RTE_TARGET={{ dpdk_target }}"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/packet-capture/tasks/kernel.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/packet-capture/tasks/kernel.yml b/deployment/roles/packet-capture/tasks/kernel.yml
new file mode 100644
index 0000000..cd4abe6
--- /dev/null
+++ b/deployment/roles/packet-capture/tasks/kernel.yml
@@ -0,0 +1,51 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+# DPDK requires specific kernel boot parameters. set the params and reboot
+# the host, if the actual params differ from what is expected.
+#
+---
+ - set_fact:
+ expected_kernel_params: "default_hugepagesz=1G hugepagesz=1G hugepages={{ num_huge_pages }} iommu=pt intel_iommu=on"
+
+ - name: Check kernel boot parameters
+ shell: "cat /proc/cmdline"
+ register: actual_kernel_params
+
+ - name: Alter kernel boot parameters
+ lineinfile:
+ dest: /etc/default/grub
+ regexp: '^(GRUB_CMDLINE_LINUX=\"[^\"]+)\"$'
+ line: '\1 {{ expected_kernel_params }}"'
+ backrefs: yes
+ when: not expected_kernel_params in actual_kernel_params.stdout
+
+ - name: Update grub with kernel boot parameters
+ shell: /sbin/grub2-mkconfig -o /boot/grub2/grub.cfg
+ when: not expected_kernel_params in actual_kernel_params.stdout
+
+ - name: Restart for modified kernel params
+ command: shutdown -r now "modified kernel params"
+ async: 0
+ poll: 0
+ ignore_errors: true
+ when: not expected_kernel_params in actual_kernel_params.stdout
+
+ - name: Wait for reboot of '{{ inventory_hostname }}'
+ local_action: wait_for host={{ inventory_hostname }} state=started port=22 timeout=300 delay=10
+ become: false
+ when: not expected_kernel_params in actual_kernel_params.stdout
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/packet-capture/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/packet-capture/tasks/main.yml b/deployment/roles/packet-capture/tasks/main.yml
new file mode 100644
index 0000000..f096178
--- /dev/null
+++ b/deployment/roles/packet-capture/tasks/main.yml
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+ - include: dependencies.yml
+ - include: kernel.yml
+ - include: dpdk.yml
+ - include: pcapture.yml
+ - include: debug.yml
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/packet-capture/tasks/pcapture.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/packet-capture/tasks/pcapture.yml b/deployment/roles/packet-capture/tasks/pcapture.yml
new file mode 100644
index 0000000..d00d379
--- /dev/null
+++ b/deployment/roles/packet-capture/tasks/pcapture.yml
@@ -0,0 +1,49 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Distribute pcapture
+ copy: src=../../../metron-sensors/packet-capture dest={{ pcapture_work_dir | dirname }} mode=0755
+
+- name: Build pcapture
+ shell: "{{ item }}"
+ args:
+ chdir: "{{ pcapture_work_dir }}"
+ with_items:
+ - make
+ environment:
+ RTE_SDK: "{{ dpdk_sdk }}"
+ RTE_TARGET: "{{ dpdk_target }}"
+ LD_LIBRARY_PATH: "{{ pcapture_ld_library_path }}"
+
+- name: Install pcapture
+ shell: "cp {{ pcapture_work_dir }}/src/build/app/{{ pcapture_bin }} {{ pcapture_prefix }}"
+ args:
+ chdir: "{{ pcapture_work_dir }}"
+ creates: "{{ pcapture_prefix }}/{{ pcapture_bin }}"
+
+- name: Deploy configuration
+ template: src=pcapture.conf dest={{ pcapture_kafka_config }} mode=0755
+
+- name: Deploy service
+ template: src=pcapture dest=/etc/init.d/ mode=0755
+
+- name: Register the service with systemd
+ shell: systemctl enable pcapture
+ when: ansible_distribution == "CentOS" and ansible_distribution_major_version == "7"
+
+- name: Run pcapture
+ service: name=pcapture state=restarted
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/packet-capture/templates/pcapture
----------------------------------------------------------------------
diff --git a/deployment/roles/packet-capture/templates/pcapture b/deployment/roles/packet-capture/templates/pcapture
new file mode 100644
index 0000000..8c2221a
--- /dev/null
+++ b/deployment/roles/packet-capture/templates/pcapture
@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# pcapture daemon
+# chkconfig: 345 20 80
+# description: Packet capture probe
+# processname: pcapture
+#
+
+export RTE_SDK="{{ dpdk_sdk }}"
+export RTE_TARGET="{{ dpdk_target }}"
+export LD_LIBRARY_PATH="{{ pcapture_ld_library_path }}"
+
+DAEMON_PATH="{{ dpdk_sdk }}"
+DAEMON="{{ pcapture_prefix }}/{{ pcapture_bin }}"
+DAEMONOPTS+=" -- "
+DAEMONOPTS+="-p {{ pcapture_portmask }} "
+DAEMONOPTS+="-t {{ pcapture_topic }} "
+DAEMONOPTS+="-c {{ pcapture_kafka_config }} "
+
+NAME="pcapture"
+DESC="Metron network packet capture probe"
+PIDFILE=/var/run/$NAME.pid
+SCRIPTNAME=/etc/init.d/$NAME
+DAEMONLOG=/var/log/$NAME.log
+NOW=`date`
+
+case "$1" in
+ start)
+ printf "%-50s" "Starting $NAME..."
+ echo "$NOW: Starting $NAME..." >> $DAEMONLOG
+ cd $DAEMON_PATH
+ PID=`$DAEMON $DAEMONOPTS >> $DAEMONLOG 2>&1 & echo $!`
+ if [ -z $PID ]; then
+ printf "%s\n" "Fail"
+ else
+ echo $PID > $PIDFILE
+ printf "%s\n" "Ok"
+ fi
+ ;;
+
+ status)
+ printf "%-50s" "Checking $NAME..."
+ if [ -f $PIDFILE ]; then
+ PID=`cat $PIDFILE`
+ if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then
+ printf "%s\n" "Process dead but pidfile exists"
+ else
+ echo "Running"
+ fi
+ else
+ printf "%s\n" "Service not running"
+ fi
+ ;;
+
+ stop)
+ printf "%-50s" "Stopping $NAME"
+ PID=`cat $PIDFILE`
+ cd $DAEMON_PATH
+ if [ -f $PIDFILE ]; then
+ echo "$NOW: Stopping $NAME with pid=$PID" >> $DAEMONLOG
+ kill -HUP $PID
+ printf "%s\n" "Ok"
+ rm -f $PIDFILE
+ else
+ printf "%s\n" "pidfile not found"
+ fi
+ ;;
+
+ restart)
+ $0 stop
+ $0 start
+ ;;
+
+ *)
+ echo "Usage: $0 {status|start|stop|restart}"
+ exit 1
+esac
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/packet-capture/templates/pcapture.conf
----------------------------------------------------------------------
diff --git a/deployment/roles/packet-capture/templates/pcapture.conf b/deployment/roles/packet-capture/templates/pcapture.conf
new file mode 100644
index 0000000..e404476
--- /dev/null
+++ b/deployment/roles/packet-capture/templates/pcapture.conf
@@ -0,0 +1,67 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# kafka global settings
+#
+[kafka-global]
+
+# initial list of kafka brokers
+metadata.broker.list = {{ kafka_broker_url }}
+
+# identifies the client to kafka
+client.id = metron-packet-capture
+
+# max number of messages allowed on the producer queue
+queue.buffering.max.messages = 1000
+
+# maximum time, in milliseconds, for buffering data on the producer queue
+queue.buffering.max.ms = 3000
+
+# compression codec = none, gzip or snappy
+compression.codec = snappy
+
+# maximum number of messages batched in one MessageSet (increase for better compression)
+batch.num.messages = 10
+
+# max times to retry sending a failed message set
+message.send.max.retries = 5
+
+# backoff time before retrying a message send
+retry.backoff.ms = 250
+
+# how often statistics are emitted; 0 = never
+statistics.interval.ms = 0
+
+# only provide delivery reports for failed messages
+delivery.report.only.error = false
+
+#
+# kafka topic settings
+#
+[kafka-topic]
+
+# broker acks { 1 = leader ack, 0 = no acks, -1 = in sync replica ack }
+request.required.acks = 1
+
+# local message timeout. This value is only enforced locally and limits the time a
+# produced message waits for successful delivery. A time of 0 is infinite.
+message.timeout.ms = 10000
+
+# report offset of produced message back to application. The application must be
+# use the dr_msg_cb to retrieve the offset from rd_kafka_message_t.offset
+produce.offset.report = false
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/vagrant/packet-capture/Vagrantfile
----------------------------------------------------------------------
diff --git a/deployment/vagrant/packet-capture/Vagrantfile b/deployment/vagrant/packet-capture/Vagrantfile
new file mode 100644
index 0000000..1303712
--- /dev/null
+++ b/deployment/vagrant/packet-capture/Vagrantfile
@@ -0,0 +1,69 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+Vagrant.configure("2") do |config|
+
+ # enable hostmanager
+ config.hostmanager.enabled = true
+ config.hostmanager.manage_host = true
+
+ #
+ # source
+ #
+ config.vm.define "source" do |node|
+
+ # host settings
+ node.vm.hostname = "source"
+ node.vm.box = "bento/centos-7.1"
+ node.ssh.insert_key = "true"
+ node.vm.network :private_network, ip: "192.168.33.10", netmask: "255.255.255.0"
+
+ # provider
+ node.vm.provider "virtualbox" do |vb|
+ vb.memory = 1024
+ vb.cpus = 1
+ end
+ end
+
+ #
+ # sink
+ #
+ config.vm.define "sink" do |node|
+
+ # host settings
+ node.vm.hostname = "sink"
+ node.vm.box = "bento/centos-7.1"
+ node.ssh.insert_key = "true"
+ node.vm.network "public_network"
+ node.vm.network :private_network, ip: "192.168.33.11", netmask: "255.255.255.0"
+
+ # provider
+ node.vm.provider "virtualbox" do |vb|
+ vb.memory = 4096
+ vb.cpus = 3
+
+ # network adapter settings; [Am79C970A|Am79C973|82540EM|82543GC|82545EM|virtio]
+ vb.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
+ vb.customize ["modifyvm", :id, "--nictype2","82545EM"]
+ end
+ end
+
+ # provision hosts
+ config.vm.provision :ansible do |ansible|
+ ansible.playbook = "playbook.yml"
+ end
+end
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/vagrant/packet-capture/ansible.cfg
----------------------------------------------------------------------
diff --git a/deployment/vagrant/packet-capture/ansible.cfg b/deployment/vagrant/packet-capture/ansible.cfg
new file mode 100644
index 0000000..9c650c2
--- /dev/null
+++ b/deployment/vagrant/packet-capture/ansible.cfg
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+[defaults]
+host_key_checking = false
+library = ../../extra_modules
+roles_path = ../../roles
+pipelining = True
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/vagrant/packet-capture/playbook.yml
----------------------------------------------------------------------
diff --git a/deployment/vagrant/packet-capture/playbook.yml b/deployment/vagrant/packet-capture/playbook.yml
new file mode 100644
index 0000000..7a5128c
--- /dev/null
+++ b/deployment/vagrant/packet-capture/playbook.yml
@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+#
+# produces network traffic
+#
+- hosts: source
+ become: yes
+ vars:
+ pcap_replay_interface: "enp0s8"
+ roles:
+ - role: pcap_replay
+
+#
+# consumes network traffic
+#
+- hosts: sink
+ become: yes
+ vars:
+ dpdk_device: ["00:08.0"]
+ dpdk_target: "x86_64-native-linuxapp-gcc"
+ num_huge_pages: 512
+ pcapture_portmask: 0xf
+ pcapture_topic: pcap
+ kafka_broker_url: localhost:9092
+ roles:
+ - role: librdkafka
+ - role: kafka-broker
+ - role: packet-capture
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/.gitignore
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/.gitignore b/metron-sensors/bro-plugin-kafka/.gitignore
new file mode 100644
index 0000000..28a8358
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/.gitignore
@@ -0,0 +1,31 @@
+.state
+build
+
+# Compiled Object files
+*.slo
+*.lo
+*.o
+*.obj
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
+# Fortran module files
+*.mod
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+*.lib
+
+# Executables
+*.exe
+*.out
+*.app
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/CHANGES
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/CHANGES b/metron-sensors/bro-plugin-kafka/CHANGES
new file mode 100644
index 0000000..d9e26de
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/CHANGES
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/CMakeLists.txt b/metron-sensors/bro-plugin-kafka/CMakeLists.txt
new file mode 100644
index 0000000..30bf3b5
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/CMakeLists.txt
@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+cmake_minimum_required(VERSION 2.8)
+project(Plugin)
+include(BroPlugin)
+find_package(LibRDKafka)
+find_package(OpenSSL)
+
+if (LIBRDKAFKA_FOUND AND OPENSSL_FOUND)
+ include_directories(BEFORE ${LibRDKafka_INCLUDE_DIR} ${OpenSSL_INCLUDE_DIR})
+ bro_plugin_begin(BRO KAFKA)
+ bro_plugin_cc(src/KafkaWriter.cc)
+ bro_plugin_cc(src/Plugin.cc)
+ bro_plugin_cc(src/TaggedJSON.cc)
+ bro_plugin_bif(src/kafka.bif)
+ bro_plugin_dist_files(README CHANGES COPYING VERSION)
+ bro_plugin_link_library(${LibRDKafka_LIBRARIES})
+ bro_plugin_link_library(${LibRDKafka_C_LIBRARIES})
+ bro_plugin_link_library(${OpenSSL_LIBRARIES})
+ bro_plugin_end()
+
+elseif (NOT LIBRDKAFKA_FOUND)
+ message(FATAL_ERROR "LibRDKafka not found.")
+
+elseif (NOT OPENSSL_FOUND)
+ message(FATAL_ERROR "OpenSSL not found.")
+
+endif ()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/COPYING
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/COPYING b/metron-sensors/bro-plugin-kafka/COPYING
new file mode 100644
index 0000000..5c304d1
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/COPYING
@@ -0,0 +1,201 @@
+Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "{}"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright {yyyy} {name of copyright owner}
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/MAINTAINER
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/MAINTAINER b/metron-sensors/bro-plugin-kafka/MAINTAINER
new file mode 100644
index 0000000..0ddaed7
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/MAINTAINER
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+Apache Metron <us...@metron.incubator.apache.org>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/Makefile
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/Makefile b/metron-sensors/bro-plugin-kafka/Makefile
new file mode 100644
index 0000000..50fa3ca
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/Makefile
@@ -0,0 +1,44 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Convenience Makefile providing a few common top-level targets.
+#
+
+cmake_build_dir=build
+arch=`uname -s | tr A-Z a-z`-`uname -m`
+
+all: build-it
+
+build-it:
+ @test -e $(cmake_build_dir)/config.status || ./configure
+ -@test -e $(cmake_build_dir)/CMakeCache.txt && \
+ test $(cmake_build_dir)/CMakeCache.txt -ot `cat $(cmake_build_dir)/CMakeCache.txt | grep BRO_DIST | cut -d '=' -f 2`/build/CMakeCache.txt && \
+ echo Updating stale CMake cache && \
+ touch $(cmake_build_dir)/CMakeCache.txt
+
+ ( cd $(cmake_build_dir) && make )
+
+install:
+ ( cd $(cmake_build_dir) && make install )
+
+clean:
+ ( cd $(cmake_build_dir) && make clean )
+
+distclean:
+ rm -rf $(cmake_build_dir)
+
+test:
+ make -C tests
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/README
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/README b/metron-sensors/bro-plugin-kafka/README
new file mode 100644
index 0000000..e6caa7a
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/README
@@ -0,0 +1,92 @@
+Bro Logging Output to Kafka
+===========================
+
+A Bro log writer that sends logging output to Kafka. This provides a convenient
+means for tools in the Hadoop ecosystem, such as Storm, Spark, and others, to
+process the data generated by Bro.
+
+Installation
+------------
+
+Install librdkafka (https://github.com/edenhill/librdkafka), a native client
+library for Kafka. This plugin has been tested against the latest release of
+librdkafka, which at the time of this writing is v0.8.6.
+
+ # curl -L https://github.com/edenhill/librdkafka/archive/0.8.6.tar.gz | tar xvz
+ # cd librdkafka-0.8.6/
+ # ./configure
+ # make
+ # sudo make install
+
+Then compile this Bro plugin using the following commands.
+
+ # ./configure --bro-dist=$BRO_SRC
+ # make
+ # sudo make install
+
+Run the following command to ensure that the plugin was installed successfully.
+
+ # bro -N Bro::Kafka
+ Bro::Kafka - Writes logs to Kafka (dynamic, version 0.1)
+
+Activation
+----------
+
+The easiest way to enable Kafka output is to load the plugin's
+``logs-to-kafka.bro`` script. If you are using BroControl, the following lines
+added to local.bro will activate it.
+
+```
+@load Bro/Kafka/logs-to-kafka.bro
+redef Kafka::logs_to_send = set(Conn::LOG, HTTP::LOG, DNS::LOG);
+redef Kafka::topic_name = "bro";
+redef Kafka::kafka_conf = table(
+ ["metadata.broker.list"] = "localhost:9092"
+);
+```
+
+This example will send all HTTP, DNS, and Conn logs to a Kafka broker running on
+the localhost to a topic called ``bro``. Any configuration value accepted by
+librdkafka can be added to the ``kafka_conf`` configuration table.
+
+Settings
+--------
+
+### ``kafka_conf``
+
+The global configuration settings for Kafka. These values are passed through
+directly to librdkafka. Any valid librdkafka settings can be defined in this
+table.
+
+```
+redef Kafka::kafka_conf = table(
+ ["metadata.broker.list"] = "localhost:9092",
+ ["client.id"] = "bro"
+);
+```
+
+### ``topic_name``
+
+The name of the topic in Kafka where all Bro logs will be sent to.
+
+```
+redef Kafka::topic_name = "bro";
+```
+
+### ``max_wait_on_shutdown``
+
+The maximum number of milliseconds that the plugin will wait for any backlog of
+queued messages to be sent to Kafka before forced shutdown.
+
+```
+redef Kafka::max_wait_on_shutdown = 3000;
+```
+
+### ``tag_json``
+
+If true, a log stream identifier is appended to each JSON-formatted message. For
+example, a Conn::LOG message will look like ``{ 'conn' : { ... }}``.
+
+```
+redef Kafka::tag_json = T;
+```
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/VERSION
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/VERSION b/metron-sensors/bro-plugin-kafka/VERSION
new file mode 100644
index 0000000..204e5ca
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/VERSION
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+0.1
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/cmake/FindLibRDKafka.cmake
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/cmake/FindLibRDKafka.cmake b/metron-sensors/bro-plugin-kafka/cmake/FindLibRDKafka.cmake
new file mode 100644
index 0000000..c64d8f9
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/cmake/FindLibRDKafka.cmake
@@ -0,0 +1,49 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+find_path(LibRDKafka_ROOT_DIR
+ NAMES include/librdkafka/rdkafkacpp.h
+)
+
+find_library(LibRDKafka_LIBRARIES
+ NAMES rdkafka++
+ HINTS ${LibRDKafka_ROOT_DIR}/lib
+)
+
+find_library(LibRDKafka_C_LIBRARIES
+ NAMES rdkafka
+ HINTS ${LibRDKafka_ROT_DIR}/lib
+)
+
+find_path(LibRDKafka_INCLUDE_DIR
+ NAMES librdkafka/rdkafkacpp.h
+ HINTS ${LibRDKafka_ROOT_DIR}/include
+)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(LibRDKafka DEFAULT_MSG
+ LibRDKafka_LIBRARIES
+ LibRDKafka_C_LIBRARIES
+ LibRDKafka_INCLUDE_DIR
+)
+
+mark_as_advanced(
+ LibRDKafka_ROOT_DIR
+ LibRDKafka_LIBRARIES
+ LibRDKafka_C_LIBRARIES
+ LibRDKafka_INCLUDE_DIR
+)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/cmake/FindOpenSSL.cmake
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/cmake/FindOpenSSL.cmake b/metron-sensors/bro-plugin-kafka/cmake/FindOpenSSL.cmake
new file mode 100644
index 0000000..5ed955c
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/cmake/FindOpenSSL.cmake
@@ -0,0 +1,72 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# - Try to find openssl include dirs and libraries
+#
+# Usage of this module as follows:
+#
+# find_package(OpenSSL)
+#
+# Variables used by this module, they can change the default behaviour and need
+# to be set before calling find_package:
+#
+# OpenSSL_ROOT_DIR Set this variable to the root installation of
+# openssl if the module has problems finding the
+# proper installation path.
+#
+# Variables defined by this module:
+#
+# OPENSSL_FOUND System has openssl, include and library dirs found
+# OpenSSL_INCLUDE_DIR The openssl include directories.
+# OpenSSL_LIBRARIES The openssl libraries.
+# OpenSSL_CYRPTO_LIBRARY The openssl crypto library.
+# OpenSSL_SSL_LIBRARY The openssl ssl library.
+
+find_path(OpenSSL_ROOT_DIR
+ NAMES include/openssl/ssl.h
+)
+
+find_path(OpenSSL_INCLUDE_DIR
+ NAMES openssl/ssl.h
+ HINTS ${OpenSSL_ROOT_DIR}/include
+)
+
+find_library(OpenSSL_SSL_LIBRARY
+ NAMES ssl ssleay32 ssleay32MD
+ HINTS ${OpenSSL_ROOT_DIR}/lib
+)
+
+find_library(OpenSSL_CRYPTO_LIBRARY
+ NAMES crypto
+ HINTS ${OpenSSL_ROOT_DIR}/lib
+)
+
+set(OpenSSL_LIBRARIES ${OpenSSL_SSL_LIBRARY} ${OpenSSL_CRYPTO_LIBRARY}
+ CACHE STRING "OpenSSL SSL and crypto libraries" FORCE)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(OpenSSL DEFAULT_MSG
+ OpenSSL_LIBRARIES
+ OpenSSL_INCLUDE_DIR
+)
+
+mark_as_advanced(
+ OpenSSL_ROOT_DIR
+ OpenSSL_INCLUDE_DIR
+ OpenSSL_LIBRARIES
+ OpenSSL_CRYPTO_LIBRARY
+ OpenSSL_SSL_LIBRARY
+)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/configure
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/configure b/metron-sensors/bro-plugin-kafka/configure
new file mode 100755
index 0000000..d053488
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/configure
@@ -0,0 +1,130 @@
+#!/bin/sh
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Wrapper for viewing/setting options that the plugin's CMake
+# scripts will recognize.
+#
+# Don't edit this. Edit configure.plugin to add plugin-specific options.
+#
+
+set -e
+command="$0 $*"
+
+if [ -e `dirname $0`/configure.plugin ]; then
+ # Include custom additions.
+ . `dirname $0`/configure.plugin
+fi
+
+# Check for `cmake` command.
+type cmake > /dev/null 2>&1 || {
+ echo "\
+This package requires CMake, please install it first, then you may
+use this configure script to access CMake equivalent functionality.\
+" >&2;
+ exit 1;
+}
+
+usage() {
+
+cat 1>&2 <<EOF
+Usage: $0 [OPTIONS]
+
+ Plugin Options:
+ --bro-dist=DIR Path to Bro source tree
+ --install-root=DIR Path where to install plugin into
+EOF
+
+if type plugin_usage >/dev/null 2>&1; then
+ plugin_usage 1>&2
+fi
+
+echo
+
+exit 1
+}
+
+# Function to append a CMake cache entry definition to the
+# CMakeCacheEntries variable
+# $1 is the cache entry variable name
+# $2 is the cache entry variable type
+# $3 is the cache entry variable value
+append_cache_entry () {
+ CMakeCacheEntries="$CMakeCacheEntries -D $1:$2=$3"
+}
+
+# set defaults
+builddir=build
+brodist=`cd ../../.. && pwd`
+installroot="default"
+CMakeCacheEntries=""
+
+while [ $# -ne 0 ]; do
+ case "$1" in
+ -*=*) optarg=`echo "$1" | sed 's/[-_a-zA-Z0-9]*=//'` ;;
+ *) optarg= ;;
+ esac
+
+ case "$1" in
+ --help|-h)
+ usage
+ ;;
+ --bro-dist=*)
+ brodist=`cd $optarg && pwd`
+ ;;
+ --install-root=*)
+ installroot=$optarg
+ ;;
+ --with-openssl=*)
+ append_cache_entry OpenSSL_ROOT_DIR PATH $optarg
+ ;;
+ *)
+ if type plugin_option >/dev/null 2>&1; then
+ plugin_option $1 && shift && continue;
+ fi
+
+ echo "Invalid option '$1'. Try $0 --help to see available options."
+ exit 1
+ ;;
+ esac
+ shift
+done
+
+if [ ! -e "$brodist/bro-path-dev.in" ]; then
+ echo "Cannot determine Bro source directory, use --bro-dist=DIR."
+ exit 1
+fi
+
+append_cache_entry BRO_DIST PATH $brodist
+append_cache_entry CMAKE_MODULE_PATH PATH $brodist/cmake
+
+if [ "$installroot" != "default" ]; then
+ mkdir -p $installroot
+ append_cache_entry BRO_PLUGIN_INSTALL_ROOT PATH $installroot
+fi
+
+echo "Build Directory : $builddir"
+echo "Bro Source Directory : $brodist"
+
+mkdir -p $builddir
+cd $builddir
+
+cmake $CMakeCacheEntries ..
+
+echo "# This is the command used to configure this build" > config.status
+echo $command >> config.status
+chmod u+x config.status
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/configure.plugin
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/configure.plugin b/metron-sensors/bro-plugin-kafka/configure.plugin
new file mode 100644
index 0000000..1cb2086
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/configure.plugin
@@ -0,0 +1,43 @@
+#!/bin/sh
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Hooks to add custom options to the configure script.
+#
+
+plugin_usage()
+{
+ cat <<EOF
+ --with-librdkafka=PATH path to librdkafka
+ --with-openssl=PATH path to OpenSSL install root
+EOF
+}
+
+plugin_option()
+{
+ case "$1" in
+ --with-librdkafka=*)
+ append_cache_entry LibRdKafka_ROOT_DIR PATH $optarg
+ ;;
+ --with-openssl=*)
+ append_cache_entry OpenSSL_ROOT_DIR PATH $optarg
+ ;;
+ *)
+ return 1;
+ ;;
+ esac
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/scripts/Bro/Kafka/__load__.bro
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/scripts/Bro/Kafka/__load__.bro b/metron-sensors/bro-plugin-kafka/scripts/Bro/Kafka/__load__.bro
new file mode 100644
index 0000000..12295a9
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/scripts/Bro/Kafka/__load__.bro
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This is loaded when a user activates the plugin. Include scripts here that should be
+# loaded automatically at that point.
+#
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/scripts/Bro/Kafka/logs-to-kafka.bro
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/scripts/Bro/Kafka/logs-to-kafka.bro b/metron-sensors/bro-plugin-kafka/scripts/Bro/Kafka/logs-to-kafka.bro
new file mode 100644
index 0000000..84e390c
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/scripts/Bro/Kafka/logs-to-kafka.bro
@@ -0,0 +1,44 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+##! load this script to enable log output to kafka
+
+module Kafka;
+
+export {
+ ##
+ ## which log streams should be sent to kafka?
+ ## example:
+ ## redef Kafka::logs_to_send = set(Conn::Log, HTTP::LOG, DNS::LOG);
+ ##
+ const logs_to_send: set[Log::ID] &redef;
+}
+
+event bro_init() &priority=-5
+{
+ for (stream_id in Log::active_streams)
+ {
+ if (stream_id in Kafka::logs_to_send)
+ {
+ local filter: Log::Filter = [
+ $name = fmt("kafka-%s", stream_id),
+ $writer = Log::WRITER_KAFKAWRITER
+ ];
+
+ Log::add_filter(stream_id, filter);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/scripts/__load__.bro
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/scripts/__load__.bro b/metron-sensors/bro-plugin-kafka/scripts/__load__.bro
new file mode 100644
index 0000000..fee9549
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/scripts/__load__.bro
@@ -0,0 +1,25 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This is loaded unconditionally at Bro startup. Include scripts here that should
+# always be loaded.
+#
+# Normally, that will be only code that initializes built-in elements. Load
+# your standard scripts in
+# scripts/<plugin-namespace>/<plugin-name>/__load__.bro instead.
+#
+
+@load ./init.bro
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/scripts/init.bro
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/scripts/init.bro b/metron-sensors/bro-plugin-kafka/scripts/init.bro
new file mode 100644
index 0000000..c76b2a6
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/scripts/init.bro
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Kafka;
+
+export {
+ const topic_name: string = "bro" &redef;
+ const max_wait_on_shutdown: count = 3000 &redef;
+ const tag_json: bool = F &redef;
+ const kafka_conf: table[string] of string = table(
+ ["metadata.broker.list"] = "localhost:9092"
+ ) &redef;
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/src/KafkaWriter.cc
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/src/KafkaWriter.cc b/metron-sensors/bro-plugin-kafka/src/KafkaWriter.cc
new file mode 100644
index 0000000..9019790
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/src/KafkaWriter.cc
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <Type.h>
+#include <threading/Formatter.h>
+#include <threading/formatters/JSON.h>
+#include "kafka.bif.h"
+#include "TaggedJSON.h"
+#include "KafkaWriter.h"
+
+using namespace logging;
+using namespace writer;
+
+KafkaWriter::KafkaWriter(WriterFrontend* frontend): WriterBackend(frontend), formatter(NULL), producer(NULL), topic(NULL)
+{
+ // TODO do we need this??
+ topic_name.assign((const char*)BifConst::Kafka::topic_name->Bytes(),
+ BifConst::Kafka::topic_name->Len());
+}
+
+KafkaWriter::~KafkaWriter()
+{}
+
+bool KafkaWriter::DoInit(const WriterInfo& info, int num_fields, const threading::Field* const* fields)
+{
+ // initialize the formatter
+ if(BifConst::Kafka::tag_json) {
+ formatter = new threading::formatter::TaggedJSON(info.path, this, threading::formatter::JSON::TS_EPOCH);
+ } else {
+ formatter = new threading::formatter::JSON(this, threading::formatter::JSON::TS_EPOCH);
+ }
+
+ // kafka global configuration
+ string err;
+ conf = RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL);
+
+ // apply the user-defined settings to kafka
+ Val* val = BifConst::Kafka::kafka_conf->AsTableVal();
+ IterCookie* c = val->AsTable()->InitForIteration();
+ HashKey* k;
+ TableEntryVal* v;
+ while ((v = val->AsTable()->NextEntry(k, c))) {
+
+ // fetch the key and value
+ ListVal* index = val->AsTableVal()->RecoverIndex(k);
+ string key = index->Index(0)->AsString()->CheckString();
+ string val = v->Value()->AsString()->CheckString();
+
+ // apply setting to kafka
+ if (RdKafka::Conf::CONF_OK != conf->set(key, val, err)) {
+ reporter->Error("Failed to set '%s'='%s': %s", key.c_str(), val.c_str(), err.c_str());
+ return false;
+ }
+
+ // cleanup
+ Unref(index);
+ delete k;
+ }
+
+ // create kafka producer
+ producer = RdKafka::Producer::create(conf, err);
+ if (!producer) {
+ reporter->Error("Failed to create producer: %s", err.c_str());
+ return false;
+ }
+
+ // create handle to topic
+ topic_conf = RdKafka::Conf::create(RdKafka::Conf::CONF_TOPIC);
+ topic = RdKafka::Topic::create(producer, topic_name, topic_conf, err);
+ if (!topic) {
+ reporter->Error("Failed to create topic handle: %s", err.c_str());
+ return false;
+ }
+
+ return true;
+}
+
+/**
+ * Writer-specific method called just before the threading system is
+ * going to shutdown. It is assumed that once this messages returns,
+ * the thread can be safely terminated.
+ */
+bool KafkaWriter::DoFinish(double network_time)
+{
+ bool success = false;
+ int poll_interval = 1000;
+ int waited = 0;
+ int max_wait = BifConst::Kafka::max_wait_on_shutdown;
+
+ // wait a bit for queued messages to be delivered
+ while (producer->outq_len() > 0 && waited <= max_wait) {
+ producer->poll(poll_interval);
+ waited += poll_interval;
+ }
+
+ // successful only if all messages delivered
+ if (producer->outq_len() == 0) {
+ reporter->Error("Unable to deliver %0d message(s)", producer->outq_len());
+ success = true;
+ }
+
+ delete topic;
+ delete producer;
+ delete formatter;
+
+ return success;
+}
+
+/**
+ * Writer-specific output method implementing recording of one log
+ * entry.
+ */
+bool KafkaWriter::DoWrite(int num_fields, const threading::Field* const* fields, threading::Value** vals)
+{
+ ODesc buff;
+ buff.Clear();
+
+ // format the log entry
+ formatter->Describe(&buff, num_fields, fields, vals);
+
+ // send the formatted log entry to kafka
+ const char* raw = (const char*)buff.Bytes();
+ RdKafka::ErrorCode resp = producer->produce(
+ topic, RdKafka::Topic::PARTITION_UA, RdKafka::Producer::RK_MSG_COPY,
+ const_cast<char*>(raw), strlen(raw), NULL, NULL);
+
+ if (RdKafka::ERR_NO_ERROR == resp) {
+ producer->poll(0);
+ }
+ else {
+ string err = RdKafka::err2str(resp);
+ reporter->Error("Kafka send failed: %s", err.c_str());
+ }
+
+ return true;
+}
+
+/**
+ * Writer-specific method implementing a change of fthe buffering
+ * state. If buffering is disabled, the writer should attempt to
+ * write out information as quickly as possible even if doing so may
+ * have a performance impact. If enabled (which is the default), it
+ * may buffer data as helpful and write it out later in a way
+ * optimized for performance. The current buffering state can be
+ * queried via IsBuf().
+ */
+bool KafkaWriter::DoSetBuf(bool enabled)
+{
+ // no change in behavior
+ return true;
+}
+
+/**
+ * Writer-specific method implementing flushing of its output. A writer
+ * implementation must override this method but it can just
+ * ignore calls if flushing doesn't align with its semantics.
+ */
+bool KafkaWriter::DoFlush(double network_time)
+{
+ producer->poll(0);
+ return true;
+}
+
+/**
+ * Writer-specific method implementing log rotation. Most directly
+ * this only applies to writers writing into files, which should then
+ * close the current file and open a new one. However, a writer may
+ * also trigger other apppropiate actions if semantics are similar.
+ * Once rotation has finished, the implementation *must* call
+ * FinishedRotation() to signal the log manager that potential
+ * postprocessors can now run.
+ */
+bool KafkaWriter::DoRotate(const char* rotated_path, double open, double close, bool terminating)
+{
+ // no need to perform log rotation
+ return FinishedRotation();
+}
+
+/**
+ * Triggered by regular heartbeat messages from the main thread.
+ */
+bool KafkaWriter::DoHeartbeat(double network_time, double current_time)
+{
+ producer->poll(0);
+ return true;
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/src/KafkaWriter.h
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/src/KafkaWriter.h b/metron-sensors/bro-plugin-kafka/src/KafkaWriter.h
new file mode 100644
index 0000000..2299667
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/src/KafkaWriter.h
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef BRO_PLUGIN_BRO_KAFKA_KAFKAWRITER_H
+#define BRO_PLUGIN_BRO_KAFKA_KAFKAWRITER_H
+
+#include <string>
+#include <librdkafka/rdkafkacpp.h>
+#include <logging/WriterBackend.h>
+#include <threading/formatters/JSON.h>
+#include <Type.h>
+#include "kafka.bif.h"
+
+#include "TaggedJSON.h"
+
+namespace logging { namespace writer {
+
+/**
+ * A logging writer that sends data to a Kafka broker.
+ */
+class KafkaWriter : public WriterBackend {
+
+public:
+ KafkaWriter(WriterFrontend* frontend);
+ ~KafkaWriter();
+
+ static WriterBackend* Instantiate(WriterFrontend* frontend)
+ {
+ return new KafkaWriter(frontend);
+ }
+
+protected:
+ virtual bool DoInit(const WriterBackend::WriterInfo& info, int num_fields, const threading::Field* const* fields);
+ virtual bool DoWrite(int num_fields, const threading::Field* const* fields, threading::Value** vals);
+ virtual bool DoSetBuf(bool enabled);
+ virtual bool DoRotate(const char* rotated_path, double open, double close, bool terminating);
+ virtual bool DoFlush(double network_time);
+ virtual bool DoFinish(double network_time);
+ virtual bool DoHeartbeat(double network_time, double current_time);
+
+private:
+ string topic_name;
+ threading::formatter::Formatter *formatter;
+ RdKafka::Producer* producer;
+ RdKafka::Topic* topic;
+ RdKafka::Conf* conf;
+ RdKafka::Conf* topic_conf;
+};
+
+}}
+
+#endif
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/src/Plugin.cc
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/src/Plugin.cc b/metron-sensors/bro-plugin-kafka/src/Plugin.cc
new file mode 100644
index 0000000..d523d23
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/src/Plugin.cc
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Plugin.h"
+#include "KafkaWriter.h"
+
+namespace plugin { namespace Bro_Kafka {
+ Plugin plugin;
+}}
+
+using namespace plugin::Bro_Kafka;
+
+plugin::Configuration Plugin::Configure()
+{
+ AddComponent(new ::logging::Component("KafkaWriter", ::logging::writer::KafkaWriter::Instantiate));
+
+ plugin::Configuration config;
+ config.name = "Bro::Kafka";
+ config.description = "Writes logs to Kafka";
+ config.version.major = 0;
+ config.version.minor = 1;
+ return config;
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/src/Plugin.h
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/src/Plugin.h b/metron-sensors/bro-plugin-kafka/src/Plugin.h
new file mode 100644
index 0000000..8adeb18
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/src/Plugin.h
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef BRO_PLUGIN_BRO_KAFKA
+#define BRO_PLUGIN_BRO_KAFKA
+
+#include <plugin/Plugin.h>
+
+namespace plugin { namespace Bro_Kafka {
+
+ class Plugin : public ::plugin::Plugin {
+ protected:
+ // Overridden from plugin::Plugin.
+ virtual plugin::Configuration Configure();
+ };
+
+ extern Plugin plugin;
+}}
+
+#endif
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/src/TaggedJSON.cc
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/src/TaggedJSON.cc b/metron-sensors/bro-plugin-kafka/src/TaggedJSON.cc
new file mode 100644
index 0000000..db3f305
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/src/TaggedJSON.cc
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "TaggedJSON.h"
+
+namespace threading { namespace formatter {
+
+TaggedJSON::TaggedJSON(string sn, MsgThread* t, JSON::TimeFormat tf): JSON(t, tf), stream_name(sn)
+{}
+
+TaggedJSON::~TaggedJSON()
+{}
+
+bool TaggedJSON::Describe(ODesc* desc, int num_fields, const Field* const* fields, Value** vals) const
+{
+ desc->AddRaw("{");
+
+ // 'tag' the json; aka prepend the stream name to the json-formatted log content
+ desc->AddRaw("\"");
+ desc->AddRaw(stream_name);
+ desc->AddRaw("\": ");
+
+ // append the JSON formatted log record itself
+ JSON::Describe(desc, num_fields, fields, vals);
+
+ desc->AddRaw("}");
+ return true;
+}
+}}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/src/TaggedJSON.h
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/src/TaggedJSON.h b/metron-sensors/bro-plugin-kafka/src/TaggedJSON.h
new file mode 100644
index 0000000..08a50df
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/src/TaggedJSON.h
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef BRO_PLUGIN_BRO_KAFKA_TAGGEDJSON_H
+#define BRO_PLUGIN_BRO_KAFKA_TAGGEDJSON_H
+
+#include <string>
+#include <threading/Formatter.h>
+#include <threading/formatters/JSON.h>
+
+using threading::formatter::JSON;
+using threading::MsgThread;
+using threading::Value;
+using threading::Field;
+
+namespace threading { namespace formatter {
+
+/*
+ * A JSON formatter that prepends or 'tags' the content with a log stream
+ * identifier. For example,
+ * { 'conn' : { ... }}
+ * { 'http' : { ... }}
+ */
+class TaggedJSON : public JSON {
+
+public:
+ TaggedJSON(string stream_name, MsgThread* t, JSON::TimeFormat tf);
+ virtual ~TaggedJSON();
+ virtual bool Describe(ODesc* desc, int num_fields, const Field* const* fields, Value** vals) const;
+
+private:
+ string stream_name;
+};
+
+}}
+#endif
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/src/kafka.bif
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/src/kafka.bif b/metron-sensors/bro-plugin-kafka/src/kafka.bif
new file mode 100644
index 0000000..8a8070c
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/src/kafka.bif
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Kafka;
+
+const kafka_conf: config;
+const topic_name: string;
+const max_wait_on_shutdown: count;
+const tag_json: bool;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/src/kafka_const.bif
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/src/kafka_const.bif b/metron-sensors/bro-plugin-kafka/src/kafka_const.bif
new file mode 100644
index 0000000..989c0ae
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/src/kafka_const.bif
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Kafka;
+
+type config : table[string] of string;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/tests/Makefile
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/tests/Makefile b/metron-sensors/bro-plugin-kafka/tests/Makefile
new file mode 100644
index 0000000..a637cd3
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/tests/Makefile
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+test:
+ @btest
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/tests/Scripts/get-bro-env
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/tests/Scripts/get-bro-env b/metron-sensors/bro-plugin-kafka/tests/Scripts/get-bro-env
new file mode 100755
index 0000000..8aa0ea7
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/tests/Scripts/get-bro-env
@@ -0,0 +1,36 @@
+#! /bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# BTest helper for getting values for Bro-related environment variables.
+
+base=`dirname $0`
+bro=`cat ${base}/../../build/CMakeCache.txt | grep BRO_DIST | cut -d = -f 2`
+
+if [ "$1" = "brobase" ]; then
+ echo ${bro}
+elif [ "$1" = "bropath" ]; then
+ ${bro}/build/bro-path-dev
+elif [ "$1" = "bro_plugin_path" ]; then
+ ( cd ${base}/../.. && pwd )
+elif [ "$1" = "bro_seed_file" ]; then
+ echo ${bro}/testing/btest/random.seed
+elif [ "$1" = "path" ]; then
+ echo ${bro}/build/src:${bro}/aux/btest:${base}/:${bro}/aux/bro-cut:$PATH
+else
+ echo "usage: `basename $0` <var>" >&2
+ exit 1
+fi
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/tests/btest.cfg
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/tests/btest.cfg b/metron-sensors/bro-plugin-kafka/tests/btest.cfg
new file mode 100644
index 0000000..e42fefd
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/tests/btest.cfg
@@ -0,0 +1,36 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+[btest]
+TestDirs = kafka
+TmpDir = %(testbase)s/.tmp
+BaselineDir = %(testbase)s/Baseline
+IgnoreDirs = .svn CVS .tmp
+IgnoreFiles = *.tmp *.swp #* *.trace .DS_Store
+
+[environment]
+BROBASE=`%(testbase)s/Scripts/get-bro-env brobase`
+BROPATH=`%(testbase)s/Scripts/get-bro-env bropath`
+BRO_PLUGIN_PATH=`%(testbase)s/Scripts/get-bro-env bro_plugin_path`
+BRO_SEED_FILE=`%(testbase)s/Scripts/get-bro-env bro_seed_file`
+PATH=`%(testbase)s/Scripts/get-bro-env path`
+TZ=UTC
+LC_ALL=C
+TRACES=%(testbase)s/Traces
+TMPDIR=%(testbase)s/.tmp
+BRO_TRACES=`%(testbase)s/Scripts/get-bro-env brobase`/testing/btest/Traces
+TEST_DIFF_CANONIFIER=`%(testbase)s/Scripts/get-bro-env brobase`/testing/scripts/diff-canonifier
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/tests/kafka/show-plugin.bro
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/tests/kafka/show-plugin.bro b/metron-sensors/bro-plugin-kafka/tests/kafka/show-plugin.bro
new file mode 100644
index 0000000..4e8dd6a
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/tests/kafka/show-plugin.bro
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# @TEST-EXEC: bro -NN Bro::Kafka >output
+# @TEST-EXEC: btest-diff output
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/packet-capture/.gitignore
----------------------------------------------------------------------
diff --git a/metron-sensors/packet-capture/.gitignore b/metron-sensors/packet-capture/.gitignore
new file mode 100644
index 0000000..2efc4c0
--- /dev/null
+++ b/metron-sensors/packet-capture/.gitignore
@@ -0,0 +1,3 @@
+roles
+.vagrant
+*.retry