You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ni...@apache.org on 2016/04/13 16:48:27 UTC
[1/3] incubator-metron git commit: METRON-108 Create Fast Packet
Capture Process (nickwallen) closes apache/incubator-metron#73
Repository: incubator-metron
Updated Branches:
refs/heads/master 8d5fb1bbc -> 564272e6a
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/packet-capture/Makefile
----------------------------------------------------------------------
diff --git a/metron-sensors/packet-capture/Makefile b/metron-sensors/packet-capture/Makefile
new file mode 100644
index 0000000..90ecb17
--- /dev/null
+++ b/metron-sensors/packet-capture/Makefile
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+all: src
+
+src:
+ cd src; make
+
+clean:
+ cd src; make clean
+
+.PHONY: all src clean
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/packet-capture/README.md
----------------------------------------------------------------------
diff --git a/metron-sensors/packet-capture/README.md b/metron-sensors/packet-capture/README.md
new file mode 100644
index 0000000..dc299a8
--- /dev/null
+++ b/metron-sensors/packet-capture/README.md
@@ -0,0 +1,12 @@
+Metron Packet Capture
+=====================
+
+A sensor that performs network packet capture leveraging the Data Plane Development Kit ([DPDK](http://dpdk.org/)). DPDK is a set of libraries and drivers to perform fast packet processing in Linux user space. The packet capture process will bind to a DPDK-compatible network interface, capture network packets, and send the raw packet data to a Kafka Broker.
+
+Getting Started
+---------------
+
+```
+cd deployment/vagrant/packet-capture
+vagrant up
+```
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/packet-capture/conf/localhost.kafka
----------------------------------------------------------------------
diff --git a/metron-sensors/packet-capture/conf/localhost.kafka b/metron-sensors/packet-capture/conf/localhost.kafka
new file mode 100644
index 0000000..b50c20e
--- /dev/null
+++ b/metron-sensors/packet-capture/conf/localhost.kafka
@@ -0,0 +1,67 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# kafka global settings
+#
+[kafka-global]
+
+# identifies the client to kafka
+client.id = pcap-localhost
+
+# initial list of kafka brokers
+metadata.broker.list = localhost:9092
+
+# max number of messages allowed on the producer queue
+queue.buffering.max.messages = 10000000
+
+# maximum time, in milliseconds, for buffering data on the producer queue
+queue.buffering.max.ms = 1000
+
+# compression codec = none, gzip or snappy
+compression.codec = snappy
+
+# maximum number of messages batched in one MessageSet (increase for better compression)
+batch.num.messages = 1000
+
+# max times to retry sending a failed message set
+message.send.max.retries = 10
+
+# backoff time before retrying a message send
+retry.backoff.ms = 250
+
+# how often statistics are emitted; 0 = never
+statistics.interval.ms = 0
+
+# only provide delivery reports for failed messages
+delivery.report.only.error = false
+
+#
+# kafka topic settings
+#
+[kafka-topic]
+
+# broker acks { 1 = leader ack, 0 = no acks, -1 = in sync replica ack }
+request.required.acks = 1
+
+# local message timeout. This value is only enforced locally and limits the time a
+# produced message waits for successful delivery. A time of 0 is infinite.
+message.timeout.ms = 300000
+
+# report offset of produced message back to application. The application must be
+# use the dr_msg_cb to retrieve the offset from rd_kafka_message_t.offset
+produce.offset.report = true
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/packet-capture/src/.gitignore
----------------------------------------------------------------------
diff --git a/metron-sensors/packet-capture/src/.gitignore b/metron-sensors/packet-capture/src/.gitignore
new file mode 100644
index 0000000..378eac2
--- /dev/null
+++ b/metron-sensors/packet-capture/src/.gitignore
@@ -0,0 +1 @@
+build
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/packet-capture/src/Makefile
----------------------------------------------------------------------
diff --git a/metron-sensors/packet-capture/src/Makefile b/metron-sensors/packet-capture/src/Makefile
new file mode 100644
index 0000000..c045faa
--- /dev/null
+++ b/metron-sensors/packet-capture/src/Makefile
@@ -0,0 +1,52 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+ifeq ($(RTE_SDK),)
+$(error "Please define RTE_SDK environment variable")
+endif
+
+# Default target, can be overriden by command line or environment
+RTE_TARGET ?= x86_64-native-linuxapp-gcc
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+# binary name
+APP = pcapture
+
+# all source are stored in SRCS-y
+SRCS-y := main.c args.c kafka.c
+
+KAFKALIB = -L/usr/local/lib -lrdkafka
+KAFKAINC = -I/usr/local/include/librdkafka/
+
+GLIB = $(shell pkg-config --libs glib-2.0) -lgthread-2.0
+GLIBINC = $(shell pkg-config --cflags glib-2.0)
+
+LDLIBS += $(KAFKALIB) $(GLIB)
+CFLAGS += $(WERROR_FLAGS) $(KAFKAINC) $(GLIBINC)
+
+# workaround for a gcc bug with noreturn attribute
+# http://gcc.gnu.org/bugzilla/show_bug.cgi?id=12603
+ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y)
+CFLAGS_main.o += -Wno-return-type
+endif
+
+#EXTRA_CFLAGS += -O3 -Wfatal-errors
+EXTRA_CFLAGS += -g -Wall
+
+include $(RTE_SDK)/mk/rte.extapp.mk
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/packet-capture/src/args.c
----------------------------------------------------------------------
diff --git a/metron-sensors/packet-capture/src/args.c b/metron-sensors/packet-capture/src/args.c
new file mode 100644
index 0000000..04fb8d0
--- /dev/null
+++ b/metron-sensors/packet-capture/src/args.c
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "args.h"
+
+typedef int bool;
+#define true 1
+#define false 0
+#define valid(s) (s == NULL ? false : strlen(s) > 1)
+
+/*
+ * Print usage information to the user.
+ */
+void print_usage(const char* prgname)
+{
+ printf("%s [EAL options] -- [APP options]\n"
+ " -p PORTMASK hex bitmask of ports to bind [0x01]\n"
+ " -t KAFKATOPIC kafka topic [pcap]\n"
+ " -c KAFKACONF kafka config file [conf/kafka.conf]\n",
+ prgname);
+}
+
+/*
+ * Parse the 'portmask' command line argument.
+ */
+int parse_portmask(const char* portmask)
+{
+ char* end = NULL;
+ unsigned long pm;
+
+ // parse hexadecimal string
+ pm = strtoul(portmask, &end, 16);
+
+ if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0')) {
+ return -1;
+ }
+ else if (pm == 0) {
+ return -1;
+ }
+ else {
+ return pm;
+ }
+}
+
+/*
+ * Check if a file exists
+ */
+static bool file_exists(const char* filepath)
+{
+ struct stat buf;
+ return (stat(filepath, &buf) == 0);
+}
+
+/**
+ * Parse the command line arguments passed to the application.
+ */
+int parse_args(int argc, char** argv)
+{
+ int opt;
+ char** argvopt;
+ int option_index;
+ char* prgname = argv[0];
+ static struct option lgopts[] = {
+ { NULL, 0, 0, 0 }
+ };
+
+ // initialize args
+ memset(&app, 0, sizeof(struct app_params));
+
+ // parse arguments to this application
+ argvopt = argv;
+ while ((opt = getopt_long(argc, argvopt, "p:b:t:c:", lgopts, &option_index)) != EOF) {
+ switch (opt) {
+
+ // portmask
+ case 'p':
+ app.enabled_port_mask = parse_portmask(optarg);
+ if (app.enabled_port_mask == 0) {
+ printf("Error: Invalid portmask: '%s'\n", optarg);
+ print_usage(prgname);
+ return -1;
+ }
+ break;
+
+ // kafka topic
+ case 't':
+ app.kafka_topic = strdup(optarg);
+ if (!valid(app.kafka_topic)) {
+ printf("Error: Invalid kafka topic: '%s'\n", optarg);
+ print_usage(prgname);
+ return -1;
+ }
+ break;
+
+ // kafka config path
+ case 'c':
+ app.kafka_config_path = strdup(optarg);
+ if (!valid(app.kafka_config_path) || !file_exists(app.kafka_config_path)) {
+ printf("Error: Invalid kafka config: '%s'\n", optarg);
+ print_usage(prgname);
+ return -1;
+ }
+ break;
+
+ default:
+ printf("Error: Invalid argument: '%s'\n", optarg);
+ print_usage(prgname);
+ return -1;
+ }
+ }
+
+ // check for required command-line arguments
+ if (app.enabled_port_mask == 0) {
+ printf("Error: Missing -p PORTMASK\n");
+ print_usage(prgname);
+ return -1;
+ }
+
+ if (!valid(app.kafka_topic)) {
+ printf("Error: Missing -t KAFKATOPIC\n");
+ print_usage(prgname);
+ return -1;
+ }
+
+ argv[optind - 1] = prgname;
+
+ // reset getopt lib
+ optind = 0;
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/packet-capture/src/args.h
----------------------------------------------------------------------
diff --git a/metron-sensors/packet-capture/src/args.h b/metron-sensors/packet-capture/src/args.h
new file mode 100644
index 0000000..36dde29
--- /dev/null
+++ b/metron-sensors/packet-capture/src/args.h
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef METRON_ARGS_H
+#define METRON_ARGS_H
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <inttypes.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/queue.h>
+#include <sys/stat.h>
+#include <stdarg.h>
+#include <errno.h>
+#include <getopt.h>
+#include <glib.h>
+#include <rte_common.h>
+#include <rte_byteorder.h>
+#include <rte_log.h>
+#include <rte_memory.h>
+#include <rte_memcpy.h>
+#include <rte_memzone.h>
+#include <rte_eal.h>
+#include <rte_per_lcore.h>
+#include <rte_launch.h>
+#include <rte_atomic.h>
+#include <rte_cycles.h>
+#include <rte_prefetch.h>
+#include <rte_lcore.h>
+#include <rte_per_lcore.h>
+#include <rte_branch_prediction.h>
+#include <rte_interrupts.h>
+#include <rte_pci.h>
+#include <rte_random.h>
+#include <rte_debug.h>
+#include <rte_ether.h>
+#include <rte_ethdev.h>
+#include <rte_ring.h>
+#include <rte_mempool.h>
+#include <rte_mbuf.h>
+#include <rte_ip.h>
+#include <rte_tcp.h>
+#include <rte_lpm.h>
+#include <rte_string_fns.h>
+
+/*
+ * Logging definitions
+ */
+#define LOG_ERROR(log_type, fmt, args...) RTE_LOG(ERR, log_type, fmt, ##args);
+#define LOG_WARN(log_type, fmt, args...) RTE_LOG(WARNING, log_type, fmt, ##args);
+#define LOG_INFO(log_type, fmt, args...) RTE_LOG(INFO, log_type, fmt, ##args);
+
+#ifdef DEBUG
+#define LOG_LEVEL RTE_LOG_DEBUG
+#define LOG_DEBUG(log_type, fmt, args...) RTE_LOG(DEBUG, log_type, fmt, ##args);
+#else
+#define LOG_LEVEL RTE_LOG_INFO
+#define LOG_DEBUG(log_type, fmt, args...) do {} while (0)
+#endif
+
+/**
+ * Application configuration parameters.
+ */
+struct app_params {
+ uint32_t enabled_port_mask;
+ char* kafka_topic;
+ char* kafka_config_path;
+} __rte_cache_aligned;
+
+/*
+ * Contains all application parameters.
+ */
+struct app_params app;
+
+/*
+ * Print usage information to the user.
+ */
+void print_usage(const char* prgname);
+
+/*
+ * Parse the 'portmask' command line argument.
+ */
+int parse_portmask(const char* portmask);
+
+/**
+ * Parse the command line arguments passed to the application.
+ */
+int parse_args(int argc, char** argv);
+
+#endif
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/packet-capture/src/kafka.c
----------------------------------------------------------------------
diff --git a/metron-sensors/packet-capture/src/kafka.c b/metron-sensors/packet-capture/src/kafka.c
new file mode 100644
index 0000000..608e308
--- /dev/null
+++ b/metron-sensors/packet-capture/src/kafka.c
@@ -0,0 +1,224 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "kafka.h"
+
+#define POLL_TIMEOUT_MS 1000
+
+/*
+ * data structures required for the kafka client
+ */
+static rd_kafka_t** kaf_h;
+static rd_kafka_topic_t** kaf_top_h;
+static int num_conns;
+
+/**
+ * A callback executed for each global Kafka option.
+ */
+static void kaf_global_option(const char* key, const char* val, void* arg)
+{
+ rd_kafka_conf_t* conf = (rd_kafka_conf_t*)arg;
+ rd_kafka_conf_res_t rc;
+ char err[512];
+
+ rc = rd_kafka_conf_set(conf, key, val, err, sizeof(err));
+ if (RD_KAFKA_CONF_OK != rc) {
+ LOG_WARN(USER1, "unable to set kafka global option: '%s' = '%s': %s\n", key, val, err);
+ }
+}
+
+/**
+ * A callback executed for topic-level Kafka option.
+ */
+static void kaf_topic_option(const char* key, const char* val, void* arg)
+{
+ rd_kafka_topic_conf_t* conf = (rd_kafka_topic_conf_t*)arg;
+ rd_kafka_conf_res_t rc;
+ char err[512];
+
+ rc = rd_kafka_topic_conf_set(conf, key, val, err, sizeof(err));
+ if (RD_KAFKA_CONF_OK != rc) {
+ LOG_WARN(USER1, "unable to set kafka topic option: '%s' = '%s': %s\n", key, val, err);
+ }
+}
+
+/**
+ * Parses the configuration values from a configuration file.
+ */
+static void parse_kafka_config(char* file_path, const char* group,
+ void (*option_cb)(const char* key, const char* val, void* arg), void* arg)
+{
+
+ gsize i;
+ gchar* value;
+ gchar** keys;
+ gsize num_keys;
+ GError* err = NULL;
+ GError** errs = NULL;
+
+ // load the configuration file
+ GKeyFile* gkf = g_key_file_new();
+ if (!g_key_file_load_from_file(gkf, file_path, G_KEY_FILE_NONE, &err)) {
+ LOG_ERROR(USER1, "bad config: %s: %s\n", file_path, err->message);
+ }
+
+ // only grab keys within the specified group
+ keys = g_key_file_get_keys(gkf, group, &num_keys, errs);
+ if (keys) {
+
+ // execute the callback for each key/value
+ for (i = 0; i < num_keys; i++) {
+ value = g_key_file_get_value(gkf, group, keys[i], errs);
+ if (value) {
+ LOG_DEBUG(USER1, "config[%s]: %s = %s\n", group, keys[i], value);
+ option_cb(keys[i], value, arg);
+ }
+ else {
+ LOG_INFO(USER1, "bad config: %s: %s = %s: %s\n", file_path, keys[i], value, errs[0]->message);
+ }
+ }
+ }
+ else {
+ LOG_WARN(USER1, "bad config: %s: %s\n", file_path, errs[0]->message);
+ }
+
+ g_strfreev(keys);
+ g_key_file_free(gkf);
+}
+
+/**
+ * Initializes a pool of Kafka connections.
+ */
+void kaf_init(int num_of_conns)
+{
+ int i;
+ char errstr[512];
+
+ // the number of connections to maintain
+ num_conns = num_of_conns;
+
+ // create kafka resources for each consumer
+ kaf_h = calloc(num_of_conns, sizeof(rd_kafka_t*));
+ kaf_top_h = calloc(num_of_conns, sizeof(rd_kafka_topic_t*));
+
+ for (i = 0; i < num_of_conns; i++) {
+
+ // configure kafka connection; values parsed from kafka config file
+ rd_kafka_conf_t* kaf_conf = rd_kafka_conf_new();
+ if (NULL != app.kafka_config_path) {
+ parse_kafka_config(app.kafka_config_path, "kafka-global", kaf_global_option, (void*)kaf_conf);
+ }
+
+ // create a new kafka connection
+ kaf_h[i] = rd_kafka_new(RD_KAFKA_PRODUCER, kaf_conf, errstr, sizeof(errstr));
+ if (!kaf_h[i]) {
+ rte_exit(EXIT_FAILURE, "Cannot init kafka connection: %s", errstr);
+ }
+
+ // configure kafka topic; values parsed from kafka config file
+ rd_kafka_topic_conf_t* topic_conf = rd_kafka_topic_conf_new();
+ if (NULL != app.kafka_config_path) {
+ parse_kafka_config(app.kafka_config_path, "kafka-topic", kaf_topic_option, (void*)topic_conf);
+ }
+
+ // connect to a kafka topic
+ kaf_top_h[i] = rd_kafka_topic_new(kaf_h[i], app.kafka_topic, topic_conf);
+ if (!kaf_top_h[i]) {
+ rte_exit(EXIT_FAILURE, "Cannot init kafka topic: %s", app.kafka_topic);
+ }
+ }
+}
+
+/**
+ * Closes the pool of Kafka connections.
+ */
+void kaf_close(void)
+{
+ int i;
+ for (i = 0; i < num_conns; i++) {
+ // wait for messages to be delivered
+ while (rd_kafka_outq_len(kaf_h[i]) > 0) {
+ LOG_INFO(USER1, "waiting for %d messages to clear on conn [%i/%i]",
+ rd_kafka_outq_len(kaf_h[i]), i + 1, num_conns);
+ rd_kafka_poll(kaf_h[i], POLL_TIMEOUT_MS);
+ }
+
+ rd_kafka_topic_destroy(kaf_top_h[i]);
+ rd_kafka_destroy(kaf_h[i]);
+ }
+}
+
+/**
+ * The current time in microseconds.
+ */
+static uint64_t current_time(void)
+{
+ struct timeval tv;
+ gettimeofday(&tv, NULL);
+ return tv.tv_sec * (uint64_t)1000000 + tv.tv_usec;
+}
+
+/**
+ * Publish a set of packets to a kafka topic.
+ */
+int kaf_send(struct rte_mbuf* data, int pkt_count, int conn_id)
+{
+ // unassigned partition
+ int partition = RD_KAFKA_PARTITION_UA;
+ int i;
+ int pkts_sent = 0;
+ int drops;
+ rd_kafka_message_t kaf_msgs[pkt_count];
+
+ // TODO: ensure that librdkafka cleans this up for us
+ uint64_t *now = malloc(sizeof(uint64_t));
+
+ // the current time in microseconds from the epoch (in big-endian aka network
+ // byte order) is added as a message key before being sent to kafka
+ *now = htobe64(current_time());
+
+ // find the topic connection based on the conn_id
+ rd_kafka_topic_t* kaf_topic = kaf_top_h[conn_id];
+
+ // create the batch message for kafka
+ for (i = 0; i < pkt_count; i++) {
+ kaf_msgs[i].err = 0;
+ kaf_msgs[i].rkt = kaf_topic;
+ kaf_msgs[i].partition = partition;
+ kaf_msgs[i].payload = rte_ctrlmbuf_data(&data[i]);
+ kaf_msgs[i].len = rte_ctrlmbuf_len(&data[i]);
+ kaf_msgs[i].key = (void*) now;
+ kaf_msgs[i].key_len = sizeof(uint64_t);
+ kaf_msgs[i].offset = 0;
+ }
+
+ // hand all of the messages off to kafka
+ pkts_sent = rd_kafka_produce_batch(kaf_topic, partition, RD_KAFKA_MSG_F_COPY, kaf_msgs, pkt_count);
+
+ // did we drop packets?
+ drops = pkt_count - pkts_sent;
+ if (drops > 0) {
+ for (i = 0; i < pkt_count; i++) {
+ if (!kaf_msgs[i].err) {
+ LOG_ERROR(USER1, "'%d' packets dropped, first error: %s", drops, (char*)kaf_msgs[i].payload);
+ }
+ }
+ }
+
+ return pkts_sent;
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/packet-capture/src/kafka.h
----------------------------------------------------------------------
diff --git a/metron-sensors/packet-capture/src/kafka.h b/metron-sensors/packet-capture/src/kafka.h
new file mode 100644
index 0000000..8b6bc78
--- /dev/null
+++ b/metron-sensors/packet-capture/src/kafka.h
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef METRON_KAFKA_H
+#define METRON_KAFKA_H
+
+#include <string.h>
+#include <sys/time.h>
+#include <endian.h>
+#include <librdkafka/rdkafka.h>
+#include <rte_common.h>
+#include <rte_mbuf.h>
+#include "args.h"
+
+/**
+ * Initializes a pool of Kafka connections.
+ */
+void kaf_init(int num_of_conns);
+
+/**
+ * Publish a set of packets to a kafka topic.
+ */
+int kaf_send(struct rte_mbuf* data, int num_to_send, int conn_id);
+
+/**
+ * Closes the pool of Kafka connections.
+ */
+void kaf_close(void);
+
+#endif
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/packet-capture/src/main.c
----------------------------------------------------------------------
diff --git a/metron-sensors/packet-capture/src/main.c b/metron-sensors/packet-capture/src/main.c
new file mode 100644
index 0000000..87a385e
--- /dev/null
+++ b/metron-sensors/packet-capture/src/main.c
@@ -0,0 +1,349 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "main.h"
+
+/*
+ * Initialize a port using global settings and with the rx buffers
+ * coming from the mbuf_pool passed as parameter
+ */
+static inline int init_port(uint8_t port, struct rte_mempool* mbuf_pool)
+{
+ struct rte_eth_conf port_conf = port_conf_default;
+ int retval;
+ uint16_t q;
+ const uint16_t rxRings = 1;
+ const uint16_t txRings = 1;
+ int socket = rte_eth_dev_socket_id(port);
+
+ if (port >= rte_eth_dev_count()) {
+ rte_exit(EXIT_FAILURE, "Port %" PRIu8 " does not exist; only %u known port(s)",
+ port, rte_eth_dev_count());
+ return -1;
+ }
+
+ retval = rte_eth_dev_configure(port, rxRings, txRings, &port_conf);
+ if (retval != 0) {
+ rte_exit(EXIT_FAILURE, "Unable to configure port %" PRIu8 "\n", port);
+ return retval;
+ }
+
+ // setup the receive rings
+ for (q = 0; q < rxRings; q++) {
+ retval = rte_eth_rx_queue_setup(port, q, RX_RING_SIZE, socket, NULL, mbuf_pool);
+ if (retval < 0) {
+ rte_exit(EXIT_FAILURE, "Unable to setup rx queue on port %" PRIu8 "\n", port);
+ return retval;
+ }
+ }
+
+ // setup the transmit rings
+ for (q = 0; q < txRings; q++) {
+ retval = rte_eth_tx_queue_setup(port, q, TX_RING_SIZE, socket, NULL);
+
+ if (retval < 0) {
+ rte_exit(EXIT_FAILURE, "Unable to setup rx queue on port %" PRIu8 "\n", port);
+ return retval;
+ }
+ }
+
+ // start the receive and transmit units on the device
+ retval = rte_eth_dev_start(port);
+ if (retval < 0) {
+ rte_exit(EXIT_FAILURE, "Unable to start device on port %" PRIu8 "\n", port);
+ return retval;
+ }
+
+ // retrieve information about the device
+ struct rte_eth_link link;
+ rte_eth_link_get_nowait(port, &link);
+ if (!link.link_status) {
+ sleep(1);
+ rte_eth_link_get_nowait(port, &link);
+ }
+
+ // if still no link information, must be down
+ if (!link.link_status) {
+ rte_exit(EXIT_FAILURE, "Link down on port %" PRIu8 "\n", port);
+ return 0;
+ }
+
+ // print diagnostics
+ struct ether_addr addr;
+ rte_eth_macaddr_get(port, &addr);
+ LOG_INFO(USER1, "Port %u MAC: %02" PRIx8 " %02" PRIx8 " %02" PRIx8 " %02" PRIx8 " %02" PRIx8 " %02" PRIx8 "\n",
+ (unsigned)port,
+ addr.addr_bytes[0], addr.addr_bytes[1],
+ addr.addr_bytes[2], addr.addr_bytes[3],
+ addr.addr_bytes[4], addr.addr_bytes[5]);
+
+ // enable promisc mode
+ rte_eth_promiscuous_enable(port);
+ return 0;
+}
+
+static void quit_workers(struct rte_distributor* d, struct rte_mempool* p, unsigned num_workers)
+{
+ unsigned i;
+ struct rte_mbuf* bufs[num_workers];
+ rte_mempool_get_bulk(p, (void*)bufs, num_workers);
+
+ for (i = 0; i < num_workers; i++) {
+ bufs[i]->hash.rss = i << 1;
+ }
+
+ rte_distributor_process(d, bufs, num_workers);
+ rte_mempool_put_bulk(p, (void*)bufs, num_workers);
+}
+
+/**
+ * Master distribution logic that receives a packet and distributes it to a
+ * worker.
+ */
+static int receive_packets(struct lcore_params* p)
+{
+ struct rte_distributor* d = p->d;
+ struct rte_mempool* mem_pool = p->mem_pool;
+ const uint8_t nb_ports = rte_eth_dev_count();
+ const int socket_id = rte_socket_id();
+ uint8_t port;
+
+ // check for cross-socket communication
+ for (port = 0; port < nb_ports; port++) {
+
+ // skip ports that are not enabled
+ if ((app.enabled_port_mask & (1 << port)) == 0) {
+ continue;
+ }
+
+ if (rte_eth_dev_socket_id(port) > 0 && rte_eth_dev_socket_id(port) != socket_id) {
+ LOG_WARN(USER1, "Warning: Port %u on different socket from thread; performance will suffer\n", port);
+ }
+ }
+
+ LOG_INFO(USER1, "Core %u doing packet receive and distribution.\n", rte_lcore_id());
+ port = 0;
+ while (!quit_signal_rx) {
+
+ // skip to the next enabled port
+ if ((app.enabled_port_mask & (1 << port)) == 0) {
+ if (++port == nb_ports) {
+ port = 0;
+ }
+ continue;
+ }
+
+ // receive a 'burst' of many packets
+ struct rte_mbuf* bufs[BURST_SIZE * 2];
+ const uint16_t nb_rx = rte_eth_rx_burst(port, 0, bufs, BURST_SIZE);
+ app_stats.rx.received_pkts += nb_rx;
+
+ // distribute the packets amongst all workers
+ rte_distributor_process(d, bufs, nb_rx);
+
+ // track packets completed by the workers
+ const uint16_t nb_ret = rte_distributor_returned_pkts(d, bufs, BURST_SIZE * 2);
+ app_stats.rx.enqueued_pkts += nb_ret;
+ if (unlikely(nb_ret == 0)) {
+ continue;
+ }
+
+ // wrap-around to the first port
+ if (++port == nb_ports) {
+ port = 0;
+ }
+ }
+
+ // flush distributor process
+ rte_distributor_process(d, NULL, 0);
+ rte_distributor_flush(d);
+
+ // notify workers that it is quitting time
+ quit_signal = 1;
+ quit_workers(d, mem_pool, p->num_workers);
+
+ return 0;
+}
+
+/*
+ * Send packets to a Kafka broker.
+ */
+static int send_packets(struct lcore_params* p)
+{
+ struct rte_distributor* d = p->d;
+ const unsigned id = p->worker_id;
+ struct rte_mbuf* buf = NULL;
+
+ LOG_INFO(USER1, "Core %u is a worker core.\n", rte_lcore_id());
+ while (!quit_signal) {
+ buf = rte_distributor_get_pkt(d, id, buf);
+
+ LOG_DEBUG(USER1, "packet received; core = %u, pkt_len = %u, data_len = %u \n",
+ rte_lcore_id(), buf->pkt_len, buf->data_len);
+
+ kaf_send(buf, 1, 0);
+ }
+ return 0;
+}
+
+static void print_stats(void)
+{
+ struct rte_eth_stats eth_stats;
+ unsigned i;
+
+ printf("\nThread stats:\n");
+ printf(" - Received: %" PRIu64 "\n", app_stats.rx.received_pkts);
+ printf(" - Queued: %" PRIu64 "\n", app_stats.rx.enqueued_pkts);
+ printf(" - Sent: %" PRIu64 "\n", app_stats.rx.sent_pkts);
+
+ for (i = 0; i < rte_eth_dev_count(); i++) {
+ rte_eth_stats_get(i, ð_stats);
+ printf("\nPort %u stats:\n", i);
+ printf(" - Pkts in: %" PRIu64 "\n", eth_stats.ipackets);
+ printf(" - Pkts out: %" PRIu64 "\n", eth_stats.opackets);
+ printf(" - In Errs: %" PRIu64 "\n", eth_stats.ierrors);
+ printf(" - Out Errs: %" PRIu64 "\n", eth_stats.oerrors);
+ printf(" - Mbuf Errs: %" PRIu64 "\n", eth_stats.rx_nombuf);
+ }
+}
+
+/*
+ * Handles interrupt signals.
+ */
+static void sig_handler(int sig_num)
+{
+ LOG_INFO(USER1, "Exiting on signal '%d'\n", sig_num);
+
+ // set quit flag for rx thread to exit
+ quit_signal_rx = 1;
+}
+
+/**
+ * Get it going.
+ */
+int main(int argc, char* argv[])
+{
+ unsigned lcore_id;
+ unsigned nb_ports;
+ unsigned worker_id = 0;
+ unsigned nb_workers;
+ uint8_t port_id;
+ uint8_t nb_ports_available;
+
+ struct rte_mempool* mbuf_pool;
+ struct rte_distributor* d;
+
+ // catch interrupt
+ signal(SIGINT, sig_handler);
+
+ // initialize the environment
+ int ret = rte_eal_init(argc, argv);
+ if (ret < 0) {
+ rte_exit(EXIT_FAILURE, "Error: Problem during initialization: %i\n", ret);
+ }
+
+ // advance past the environmental settings
+ argc -= ret;
+ argv += ret;
+
+ // parse arguments to the application
+ ret = parse_args(argc, argv);
+ if (ret < 0) {
+ rte_exit(EXIT_FAILURE, "Error: Invalid parameters\n");
+ }
+
+ // check number of ethernet devices
+ nb_ports = rte_eth_dev_count();
+ if (nb_ports == 0) {
+ rte_exit(EXIT_FAILURE, "Error: No ethernet ports detected\n");
+ }
+
+ // check number of available logical cores for workers
+ nb_workers = rte_lcore_count() - 1;
+ if (nb_workers < 1) {
+ rte_exit(EXIT_FAILURE, "Error: Minimum 2 logical cores required. \n");
+ }
+
+ // create memory pool
+ mbuf_pool = rte_pktmbuf_pool_create("mbuf-pool", NUM_MBUFS * nb_ports,
+ MBUF_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());
+ if (mbuf_pool == NULL) {
+ rte_exit(EXIT_FAILURE, "Error: Cannot create memory pool for packets\n");
+ }
+
+ // initialize each specified ethernet ports
+ nb_ports_available = nb_ports;
+ for (port_id = 0; port_id < nb_ports; port_id++) {
+
+ // skip over ports that are not enabled
+ if ((app.enabled_port_mask & (1 << port_id)) == 0) {
+ LOG_INFO(USER1, "Skipping over disabled port '%d'\n", port_id);
+ nb_ports_available--;
+ continue;
+ }
+
+ // initialize the port
+ LOG_INFO(USER1, "Initializing port %u\n", (unsigned)port_id);
+ if (init_port(port_id, mbuf_pool) != 0) {
+ rte_exit(EXIT_FAILURE, "Cannot initialize port %" PRIu8 "\n", port_id);
+ }
+ }
+
+ // ensure that we were able to initialize enough ports
+ if (!nb_ports_available) {
+ rte_exit(EXIT_FAILURE, "Error: No available enabled ports. Portmask set?\n");
+ }
+
+ kaf_init(1);
+
+ // the distributor will dispatch packets to 1 or more workers
+ d = rte_distributor_create("master", rte_socket_id(), nb_workers);
+ if (d == NULL) {
+ rte_exit(EXIT_FAILURE, "Error: Unable to create distributor\n");
+ }
+
+ // launch workers on each logical core
+ RTE_LCORE_FOREACH_SLAVE(lcore_id) {
+
+ struct lcore_params* p = rte_malloc(NULL, sizeof(*p), 0);
+ if (!p) {
+ rte_panic("Error: rte_malloc failure\n");
+ }
+
+ // launch the worker process
+ LOG_INFO(USER1, "Launching worker on core %u\n", lcore_id);
+ *p = (struct lcore_params){ worker_id, nb_workers, d, mbuf_pool };
+ rte_eal_remote_launch((lcore_function_t*)send_packets, p, lcore_id);
+
+ worker_id++;
+ }
+
+ // start distributing packets on the master
+ struct lcore_params p = { 0, nb_workers, d, mbuf_pool };
+ receive_packets(&p);
+
+ // wait for each worker to complete
+ RTE_LCORE_FOREACH_SLAVE(lcore_id) {
+ if (rte_eal_wait_lcore(lcore_id) < 0) {
+ return -1;
+ }
+ }
+
+ print_stats();
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/packet-capture/src/main.h
----------------------------------------------------------------------
diff --git a/metron-sensors/packet-capture/src/main.h b/metron-sensors/packet-capture/src/main.h
new file mode 100644
index 0000000..abaf82a
--- /dev/null
+++ b/metron-sensors/packet-capture/src/main.h
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef METRON_MAIN_H
+#define METRON_MAIN_H
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <inttypes.h>
+#include <sys/types.h>
+#include <string.h>
+#include <sys/queue.h>
+#include <stdarg.h>
+#include <errno.h>
+#include <getopt.h>
+#include <unistd.h>
+#include <signal.h>
+#include <rte_common.h>
+#include <rte_byteorder.h>
+#include <rte_log.h>
+#include <rte_memory.h>
+#include <rte_memcpy.h>
+#include <rte_memzone.h>
+#include <rte_eal.h>
+#include <rte_per_lcore.h>
+#include <rte_launch.h>
+#include <rte_atomic.h>
+#include <rte_cycles.h>
+#include <rte_prefetch.h>
+#include <rte_lcore.h>
+#include <rte_per_lcore.h>
+#include <rte_branch_prediction.h>
+#include <rte_interrupts.h>
+#include <rte_pci.h>
+#include <rte_random.h>
+#include <rte_debug.h>
+#include <rte_ether.h>
+#include <rte_ethdev.h>
+#include <rte_ring.h>
+#include <rte_mempool.h>
+#include <rte_mbuf.h>
+#include <rte_ip.h>
+#include <rte_tcp.h>
+#include <rte_lpm.h>
+#include <rte_string_fns.h>
+#include <rte_distributor.h>
+#include <rte_malloc.h>
+
+#include "args.h"
+#include "kafka.h"
+
+#define RX_RING_SIZE 256
+#define TX_RING_SIZE 512
+#define NUM_MBUFS ((64 * 1024) - 1)
+#define MBUF_CACHE_SIZE 250
+#define BURST_SIZE 32
+#define RTE_RING_SZ 1024
+
+// uncomment below line to enable debug logs
+//#define DEBUG
+
+volatile uint8_t quit_signal;
+volatile uint8_t quit_signal_rx;
+
+/**
+ * Tracks packet processing stats.
+ */
+volatile struct app_stats {
+ struct {
+ uint64_t received_pkts;
+ uint64_t enqueued_pkts;
+ uint64_t sent_pkts;
+ } rx __rte_cache_aligned;
+} app_stats;
+
+/**
+ * Default port configuration settings.
+ */
+const struct rte_eth_conf port_conf_default = {
+ .rxmode = {
+ .mq_mode = ETH_MQ_RX_RSS,
+ .max_rx_pkt_len = ETHER_MAX_LEN,
+ },
+ .txmode = {
+ .mq_mode = ETH_MQ_TX_NONE,
+ },
+ .rx_adv_conf = {
+ .rss_conf = {
+ .rss_hf = ETH_RSS_IP | ETH_RSS_UDP | ETH_RSS_TCP | ETH_RSS_SCTP,
+ }
+ },
+};
+
+/**
+ * Configuration parameters provided to each worker.
+ */
+struct lcore_params {
+ unsigned worker_id;
+ unsigned num_workers;
+ struct rte_distributor* d;
+ struct rte_mempool* mem_pool;
+} __rte_cache_aligned;
+
+int main(int argc, char* argv[]);
+
+#endif
[2/3] incubator-metron git commit: METRON-108 Create Fast Packet
Capture Process (nickwallen) closes apache/incubator-metron#73
Posted by ni...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/packet-capture/tasks/dependencies.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/packet-capture/tasks/dependencies.yml b/deployment/roles/packet-capture/tasks/dependencies.yml
new file mode 100644
index 0000000..4d6edc4
--- /dev/null
+++ b/deployment/roles/packet-capture/tasks/dependencies.yml
@@ -0,0 +1,38 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+ - name: Install dependencies
+ yum: name={{ item }}
+ with_items:
+ - "@Development tools"
+ - pciutils
+ - net-tools
+ - glib2
+ - glib2-devel
+ - git
+
+ #
+ # install prerequisite packages and the latest kernel headers. need to
+ # ensure that the kernel headers match the current running kernel version.
+ # if this is not the case, the DPDK build process will fail
+ #
+ - name: Install latest kernel headers and source
+ yum: name={{ item }} state=latest
+ with_items:
+ - kernel
+ - kernel-devel
+ - kernel-headers
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/packet-capture/tasks/dpdk.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/packet-capture/tasks/dpdk.yml b/deployment/roles/packet-capture/tasks/dpdk.yml
new file mode 100644
index 0000000..3780be7
--- /dev/null
+++ b/deployment/roles/packet-capture/tasks/dpdk.yml
@@ -0,0 +1,59 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+ - name: "Download DPDK version {{ dpdk_version }}"
+ unarchive:
+ src: "http://dpdk.org/browse/dpdk/snapshot/dpdk-{{ dpdk_version }}.tar.gz"
+ dest: "/root"
+ creates: "{{ dpdk_sdk }}"
+ copy: no
+
+ - name: "Configure DPDK for the target environment: {{ dpdk_target }}"
+ shell: "make config T={{ dpdk_target }} DESTDIR={{ dpdk_home }}"
+ args:
+ chdir: "{{ dpdk_sdk }}"
+ creates: "{{ dpdk_home }}"
+
+ - name: "Turn on debug flags"
+ lineinfile:
+ dest: "{{ dpdk_sdk }}/config/common_linuxapp"
+ regexp: 'DEBUG=n'
+ line: 'DEBUG=y'
+ tags:
+ - debug
+
+ - name: "Build DPDK for the target environment: {{ dpdk_target }}"
+ shell: "make install T={{ dpdk_target }} DESTDIR={{ dpdk_home }} EXTRA_CFLAGS={{ extra_cflags }}"
+ args:
+ chdir: "{{ dpdk_sdk }}"
+ creates: "{{ dpdk_home }}"
+
+ - name: Load kernel modules to enable userspace IO
+ shell: "{{ item }}"
+ with_items:
+ - modprobe uio_pci_generic
+ - modprobe vfio-pci
+
+ - name: Bind the device to the loaded kernel module(s)
+ shell: "{{ dpdk_home }}/sbin/dpdk_nic_bind --force --bind=uio_pci_generic {{ item }}"
+ with_items: "{{ dpdk_device }}"
+
+ - name: Set useful environment variables
+ lineinfile: "dest=/root/.bash_profile line={{ item }}"
+ with_items:
+ - "export RTE_SDK={{ dpdk_sdk }}"
+ - "export RTE_TARGET={{ dpdk_target }}"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/packet-capture/tasks/kernel.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/packet-capture/tasks/kernel.yml b/deployment/roles/packet-capture/tasks/kernel.yml
new file mode 100644
index 0000000..cd4abe6
--- /dev/null
+++ b/deployment/roles/packet-capture/tasks/kernel.yml
@@ -0,0 +1,51 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+# DPDK requires specific kernel boot parameters. set the params and reboot
+# the host, if the actual params differ from what is expected.
+#
+---
+ - set_fact:
+ expected_kernel_params: "default_hugepagesz=1G hugepagesz=1G hugepages={{ num_huge_pages }} iommu=pt intel_iommu=on"
+
+ - name: Check kernel boot parameters
+ shell: "cat /proc/cmdline"
+ register: actual_kernel_params
+
+ - name: Alter kernel boot parameters
+ lineinfile:
+ dest: /etc/default/grub
+ regexp: '^(GRUB_CMDLINE_LINUX=\"[^\"]+)\"$'
+ line: '\1 {{ expected_kernel_params }}"'
+ backrefs: yes
+ when: not expected_kernel_params in actual_kernel_params.stdout
+
+ - name: Update grub with kernel boot parameters
+ shell: /sbin/grub2-mkconfig -o /boot/grub2/grub.cfg
+ when: not expected_kernel_params in actual_kernel_params.stdout
+
+ - name: Restart for modified kernel params
+ command: shutdown -r now "modified kernel params"
+ async: 0
+ poll: 0
+ ignore_errors: true
+ when: not expected_kernel_params in actual_kernel_params.stdout
+
+ - name: Wait for reboot of '{{ inventory_hostname }}'
+ local_action: wait_for host={{ inventory_hostname }} state=started port=22 timeout=300 delay=10
+ become: false
+ when: not expected_kernel_params in actual_kernel_params.stdout
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/packet-capture/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/packet-capture/tasks/main.yml b/deployment/roles/packet-capture/tasks/main.yml
new file mode 100644
index 0000000..f096178
--- /dev/null
+++ b/deployment/roles/packet-capture/tasks/main.yml
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+ - include: dependencies.yml
+ - include: kernel.yml
+ - include: dpdk.yml
+ - include: pcapture.yml
+ - include: debug.yml
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/packet-capture/tasks/pcapture.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/packet-capture/tasks/pcapture.yml b/deployment/roles/packet-capture/tasks/pcapture.yml
new file mode 100644
index 0000000..d00d379
--- /dev/null
+++ b/deployment/roles/packet-capture/tasks/pcapture.yml
@@ -0,0 +1,49 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Distribute pcapture
+ copy: src=../../../metron-sensors/packet-capture dest={{ pcapture_work_dir | dirname }} mode=0755
+
+- name: Build pcapture
+ shell: "{{ item }}"
+ args:
+ chdir: "{{ pcapture_work_dir }}"
+ with_items:
+ - make
+ environment:
+ RTE_SDK: "{{ dpdk_sdk }}"
+ RTE_TARGET: "{{ dpdk_target }}"
+ LD_LIBRARY_PATH: "{{ pcapture_ld_library_path }}"
+
+- name: Install pcapture
+ shell: "cp {{ pcapture_work_dir }}/src/build/app/{{ pcapture_bin }} {{ pcapture_prefix }}"
+ args:
+ chdir: "{{ pcapture_work_dir }}"
+ creates: "{{ pcapture_prefix }}/{{ pcapture_bin }}"
+
+- name: Deploy configuration
+ template: src=pcapture.conf dest={{ pcapture_kafka_config }} mode=0755
+
+- name: Deploy service
+ template: src=pcapture dest=/etc/init.d/ mode=0755
+
+- name: Register the service with systemd
+ shell: systemctl enable pcapture
+ when: ansible_distribution == "CentOS" and ansible_distribution_major_version == "7"
+
+- name: Run pcapture
+ service: name=pcapture state=restarted
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/packet-capture/templates/pcapture
----------------------------------------------------------------------
diff --git a/deployment/roles/packet-capture/templates/pcapture b/deployment/roles/packet-capture/templates/pcapture
new file mode 100644
index 0000000..8c2221a
--- /dev/null
+++ b/deployment/roles/packet-capture/templates/pcapture
@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# pcapture daemon
+# chkconfig: 345 20 80
+# description: Packet capture probe
+# processname: pcapture
+#
+
+export RTE_SDK="{{ dpdk_sdk }}"
+export RTE_TARGET="{{ dpdk_target }}"
+export LD_LIBRARY_PATH="{{ pcapture_ld_library_path }}"
+
+DAEMON_PATH="{{ dpdk_sdk }}"
+DAEMON="{{ pcapture_prefix }}/{{ pcapture_bin }}"
+DAEMONOPTS+=" -- "
+DAEMONOPTS+="-p {{ pcapture_portmask }} "
+DAEMONOPTS+="-t {{ pcapture_topic }} "
+DAEMONOPTS+="-c {{ pcapture_kafka_config }} "
+
+NAME="pcapture"
+DESC="Metron network packet capture probe"
+PIDFILE=/var/run/$NAME.pid
+SCRIPTNAME=/etc/init.d/$NAME
+DAEMONLOG=/var/log/$NAME.log
+NOW=`date`
+
+case "$1" in
+ start)
+ printf "%-50s" "Starting $NAME..."
+ echo "$NOW: Starting $NAME..." >> $DAEMONLOG
+ cd $DAEMON_PATH
+ PID=`$DAEMON $DAEMONOPTS >> $DAEMONLOG 2>&1 & echo $!`
+ if [ -z $PID ]; then
+ printf "%s\n" "Fail"
+ else
+ echo $PID > $PIDFILE
+ printf "%s\n" "Ok"
+ fi
+ ;;
+
+ status)
+ printf "%-50s" "Checking $NAME..."
+ if [ -f $PIDFILE ]; then
+ PID=`cat $PIDFILE`
+ if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then
+ printf "%s\n" "Process dead but pidfile exists"
+ else
+ echo "Running"
+ fi
+ else
+ printf "%s\n" "Service not running"
+ fi
+ ;;
+
+ stop)
+ printf "%-50s" "Stopping $NAME"
+ PID=`cat $PIDFILE`
+ cd $DAEMON_PATH
+ if [ -f $PIDFILE ]; then
+ echo "$NOW: Stopping $NAME with pid=$PID" >> $DAEMONLOG
+ kill -HUP $PID
+ printf "%s\n" "Ok"
+ rm -f $PIDFILE
+ else
+ printf "%s\n" "pidfile not found"
+ fi
+ ;;
+
+ restart)
+ $0 stop
+ $0 start
+ ;;
+
+ *)
+ echo "Usage: $0 {status|start|stop|restart}"
+ exit 1
+esac
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/packet-capture/templates/pcapture.conf
----------------------------------------------------------------------
diff --git a/deployment/roles/packet-capture/templates/pcapture.conf b/deployment/roles/packet-capture/templates/pcapture.conf
new file mode 100644
index 0000000..e404476
--- /dev/null
+++ b/deployment/roles/packet-capture/templates/pcapture.conf
@@ -0,0 +1,67 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# kafka global settings
+#
+[kafka-global]
+
+# initial list of kafka brokers
+metadata.broker.list = {{ kafka_broker_url }}
+
+# identifies the client to kafka
+client.id = metron-packet-capture
+
+# max number of messages allowed on the producer queue
+queue.buffering.max.messages = 1000
+
+# maximum time, in milliseconds, for buffering data on the producer queue
+queue.buffering.max.ms = 3000
+
+# compression codec = none, gzip or snappy
+compression.codec = snappy
+
+# maximum number of messages batched in one MessageSet (increase for better compression)
+batch.num.messages = 10
+
+# max times to retry sending a failed message set
+message.send.max.retries = 5
+
+# backoff time before retrying a message send
+retry.backoff.ms = 250
+
+# how often statistics are emitted; 0 = never
+statistics.interval.ms = 0
+
+# only provide delivery reports for failed messages
+delivery.report.only.error = false
+
+#
+# kafka topic settings
+#
+[kafka-topic]
+
+# broker acks { 1 = leader ack, 0 = no acks, -1 = in sync replica ack }
+request.required.acks = 1
+
+# local message timeout. This value is only enforced locally and limits the time a
+# produced message waits for successful delivery. A time of 0 is infinite.
+message.timeout.ms = 10000
+
+# report offset of produced message back to application. The application must be
+# use the dr_msg_cb to retrieve the offset from rd_kafka_message_t.offset
+produce.offset.report = false
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/vagrant/packet-capture/Vagrantfile
----------------------------------------------------------------------
diff --git a/deployment/vagrant/packet-capture/Vagrantfile b/deployment/vagrant/packet-capture/Vagrantfile
new file mode 100644
index 0000000..1303712
--- /dev/null
+++ b/deployment/vagrant/packet-capture/Vagrantfile
@@ -0,0 +1,69 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+Vagrant.configure("2") do |config|
+
+ # enable hostmanager
+ config.hostmanager.enabled = true
+ config.hostmanager.manage_host = true
+
+ #
+ # source
+ #
+ config.vm.define "source" do |node|
+
+ # host settings
+ node.vm.hostname = "source"
+ node.vm.box = "bento/centos-7.1"
+ node.ssh.insert_key = "true"
+ node.vm.network :private_network, ip: "192.168.33.10", netmask: "255.255.255.0"
+
+ # provider
+ node.vm.provider "virtualbox" do |vb|
+ vb.memory = 1024
+ vb.cpus = 1
+ end
+ end
+
+ #
+ # sink
+ #
+ config.vm.define "sink" do |node|
+
+ # host settings
+ node.vm.hostname = "sink"
+ node.vm.box = "bento/centos-7.1"
+ node.ssh.insert_key = "true"
+ node.vm.network "public_network"
+ node.vm.network :private_network, ip: "192.168.33.11", netmask: "255.255.255.0"
+
+ # provider
+ node.vm.provider "virtualbox" do |vb|
+ vb.memory = 4096
+ vb.cpus = 3
+
+ # network adapter settings; [Am79C970A|Am79C973|82540EM|82543GC|82545EM|virtio]
+ vb.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
+ vb.customize ["modifyvm", :id, "--nictype2","82545EM"]
+ end
+ end
+
+ # provision hosts
+ config.vm.provision :ansible do |ansible|
+ ansible.playbook = "playbook.yml"
+ end
+end
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/vagrant/packet-capture/ansible.cfg
----------------------------------------------------------------------
diff --git a/deployment/vagrant/packet-capture/ansible.cfg b/deployment/vagrant/packet-capture/ansible.cfg
new file mode 100644
index 0000000..9c650c2
--- /dev/null
+++ b/deployment/vagrant/packet-capture/ansible.cfg
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+[defaults]
+host_key_checking = false
+library = ../../extra_modules
+roles_path = ../../roles
+pipelining = True
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/vagrant/packet-capture/playbook.yml
----------------------------------------------------------------------
diff --git a/deployment/vagrant/packet-capture/playbook.yml b/deployment/vagrant/packet-capture/playbook.yml
new file mode 100644
index 0000000..7a5128c
--- /dev/null
+++ b/deployment/vagrant/packet-capture/playbook.yml
@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+#
+# produces network traffic
+#
+- hosts: source
+ become: yes
+ vars:
+ pcap_replay_interface: "enp0s8"
+ roles:
+ - role: pcap_replay
+
+#
+# consumes network traffic
+#
+- hosts: sink
+ become: yes
+ vars:
+ dpdk_device: ["00:08.0"]
+ dpdk_target: "x86_64-native-linuxapp-gcc"
+ num_huge_pages: 512
+ pcapture_portmask: 0xf
+ pcapture_topic: pcap
+ kafka_broker_url: localhost:9092
+ roles:
+ - role: librdkafka
+ - role: kafka-broker
+ - role: packet-capture
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/.gitignore
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/.gitignore b/metron-sensors/bro-plugin-kafka/.gitignore
new file mode 100644
index 0000000..28a8358
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/.gitignore
@@ -0,0 +1,31 @@
+.state
+build
+
+# Compiled Object files
+*.slo
+*.lo
+*.o
+*.obj
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
+# Fortran module files
+*.mod
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+*.lib
+
+# Executables
+*.exe
+*.out
+*.app
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/CHANGES
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/CHANGES b/metron-sensors/bro-plugin-kafka/CHANGES
new file mode 100644
index 0000000..d9e26de
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/CHANGES
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/CMakeLists.txt b/metron-sensors/bro-plugin-kafka/CMakeLists.txt
new file mode 100644
index 0000000..30bf3b5
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/CMakeLists.txt
@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+cmake_minimum_required(VERSION 2.8)
+project(Plugin)
+include(BroPlugin)
+find_package(LibRDKafka)
+find_package(OpenSSL)
+
+if (LIBRDKAFKA_FOUND AND OPENSSL_FOUND)
+ include_directories(BEFORE ${LibRDKafka_INCLUDE_DIR} ${OpenSSL_INCLUDE_DIR})
+ bro_plugin_begin(BRO KAFKA)
+ bro_plugin_cc(src/KafkaWriter.cc)
+ bro_plugin_cc(src/Plugin.cc)
+ bro_plugin_cc(src/TaggedJSON.cc)
+ bro_plugin_bif(src/kafka.bif)
+ bro_plugin_dist_files(README CHANGES COPYING VERSION)
+ bro_plugin_link_library(${LibRDKafka_LIBRARIES})
+ bro_plugin_link_library(${LibRDKafka_C_LIBRARIES})
+ bro_plugin_link_library(${OpenSSL_LIBRARIES})
+ bro_plugin_end()
+
+elseif (NOT LIBRDKAFKA_FOUND)
+ message(FATAL_ERROR "LibRDKafka not found.")
+
+elseif (NOT OPENSSL_FOUND)
+ message(FATAL_ERROR "OpenSSL not found.")
+
+endif ()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/COPYING
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/COPYING b/metron-sensors/bro-plugin-kafka/COPYING
new file mode 100644
index 0000000..5c304d1
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/COPYING
@@ -0,0 +1,201 @@
+Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "{}"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright {yyyy} {name of copyright owner}
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/MAINTAINER
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/MAINTAINER b/metron-sensors/bro-plugin-kafka/MAINTAINER
new file mode 100644
index 0000000..0ddaed7
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/MAINTAINER
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+Apache Metron <us...@metron.incubator.apache.org>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/Makefile
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/Makefile b/metron-sensors/bro-plugin-kafka/Makefile
new file mode 100644
index 0000000..50fa3ca
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/Makefile
@@ -0,0 +1,44 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Convenience Makefile providing a few common top-level targets.
+#
+
+cmake_build_dir=build
+arch=`uname -s | tr A-Z a-z`-`uname -m`
+
+all: build-it
+
+build-it:
+ @test -e $(cmake_build_dir)/config.status || ./configure
+ -@test -e $(cmake_build_dir)/CMakeCache.txt && \
+ test $(cmake_build_dir)/CMakeCache.txt -ot `cat $(cmake_build_dir)/CMakeCache.txt | grep BRO_DIST | cut -d '=' -f 2`/build/CMakeCache.txt && \
+ echo Updating stale CMake cache && \
+ touch $(cmake_build_dir)/CMakeCache.txt
+
+ ( cd $(cmake_build_dir) && make )
+
+install:
+ ( cd $(cmake_build_dir) && make install )
+
+clean:
+ ( cd $(cmake_build_dir) && make clean )
+
+distclean:
+ rm -rf $(cmake_build_dir)
+
+test:
+ make -C tests
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/README
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/README b/metron-sensors/bro-plugin-kafka/README
new file mode 100644
index 0000000..e6caa7a
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/README
@@ -0,0 +1,92 @@
+Bro Logging Output to Kafka
+===========================
+
+A Bro log writer that sends logging output to Kafka. This provides a convenient
+means for tools in the Hadoop ecosystem, such as Storm, Spark, and others, to
+process the data generated by Bro.
+
+Installation
+------------
+
+Install librdkafka (https://github.com/edenhill/librdkafka), a native client
+library for Kafka. This plugin has been tested against the latest release of
+librdkafka, which at the time of this writing is v0.8.6.
+
+ # curl -L https://github.com/edenhill/librdkafka/archive/0.8.6.tar.gz | tar xvz
+ # cd librdkafka-0.8.6/
+ # ./configure
+ # make
+ # sudo make install
+
+Then compile this Bro plugin using the following commands.
+
+ # ./configure --bro-dist=$BRO_SRC
+ # make
+ # sudo make install
+
+Run the following command to ensure that the plugin was installed successfully.
+
+ # bro -N Bro::Kafka
+ Bro::Kafka - Writes logs to Kafka (dynamic, version 0.1)
+
+Activation
+----------
+
+The easiest way to enable Kafka output is to load the plugin's
+``logs-to-kafka.bro`` script. If you are using BroControl, the following lines
+added to local.bro will activate it.
+
+```
+@load Bro/Kafka/logs-to-kafka.bro
+redef Kafka::logs_to_send = set(Conn::LOG, HTTP::LOG, DNS::LOG);
+redef Kafka::topic_name = "bro";
+redef Kafka::kafka_conf = table(
+ ["metadata.broker.list"] = "localhost:9092"
+);
+```
+
+This example will send all HTTP, DNS, and Conn logs to a Kafka broker running on
+the localhost to a topic called ``bro``. Any configuration value accepted by
+librdkafka can be added to the ``kafka_conf`` configuration table.
+
+Settings
+--------
+
+### ``kafka_conf``
+
+The global configuration settings for Kafka. These values are passed through
+directly to librdkafka. Any valid librdkafka settings can be defined in this
+table.
+
+```
+redef Kafka::kafka_conf = table(
+ ["metadata.broker.list"] = "localhost:9092",
+ ["client.id"] = "bro"
+);
+```
+
+### ``topic_name``
+
+The name of the topic in Kafka where all Bro logs will be sent to.
+
+```
+redef Kafka::topic_name = "bro";
+```
+
+### ``max_wait_on_shutdown``
+
+The maximum number of milliseconds that the plugin will wait for any backlog of
+queued messages to be sent to Kafka before forced shutdown.
+
+```
+redef Kafka::max_wait_on_shutdown = 3000;
+```
+
+### ``tag_json``
+
+If true, a log stream identifier is appended to each JSON-formatted message. For
+example, a Conn::LOG message will look like ``{ 'conn' : { ... }}``.
+
+```
+redef Kafka::tag_json = T;
+```
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/VERSION
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/VERSION b/metron-sensors/bro-plugin-kafka/VERSION
new file mode 100644
index 0000000..204e5ca
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/VERSION
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+0.1
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/cmake/FindLibRDKafka.cmake
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/cmake/FindLibRDKafka.cmake b/metron-sensors/bro-plugin-kafka/cmake/FindLibRDKafka.cmake
new file mode 100644
index 0000000..c64d8f9
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/cmake/FindLibRDKafka.cmake
@@ -0,0 +1,49 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+find_path(LibRDKafka_ROOT_DIR
+ NAMES include/librdkafka/rdkafkacpp.h
+)
+
+find_library(LibRDKafka_LIBRARIES
+ NAMES rdkafka++
+ HINTS ${LibRDKafka_ROOT_DIR}/lib
+)
+
+find_library(LibRDKafka_C_LIBRARIES
+ NAMES rdkafka
+ HINTS ${LibRDKafka_ROT_DIR}/lib
+)
+
+find_path(LibRDKafka_INCLUDE_DIR
+ NAMES librdkafka/rdkafkacpp.h
+ HINTS ${LibRDKafka_ROOT_DIR}/include
+)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(LibRDKafka DEFAULT_MSG
+ LibRDKafka_LIBRARIES
+ LibRDKafka_C_LIBRARIES
+ LibRDKafka_INCLUDE_DIR
+)
+
+mark_as_advanced(
+ LibRDKafka_ROOT_DIR
+ LibRDKafka_LIBRARIES
+ LibRDKafka_C_LIBRARIES
+ LibRDKafka_INCLUDE_DIR
+)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/cmake/FindOpenSSL.cmake
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/cmake/FindOpenSSL.cmake b/metron-sensors/bro-plugin-kafka/cmake/FindOpenSSL.cmake
new file mode 100644
index 0000000..5ed955c
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/cmake/FindOpenSSL.cmake
@@ -0,0 +1,72 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# - Try to find openssl include dirs and libraries
+#
+# Usage of this module as follows:
+#
+# find_package(OpenSSL)
+#
+# Variables used by this module, they can change the default behaviour and need
+# to be set before calling find_package:
+#
+# OpenSSL_ROOT_DIR Set this variable to the root installation of
+# openssl if the module has problems finding the
+# proper installation path.
+#
+# Variables defined by this module:
+#
+# OPENSSL_FOUND System has openssl, include and library dirs found
+# OpenSSL_INCLUDE_DIR The openssl include directories.
+# OpenSSL_LIBRARIES The openssl libraries.
+# OpenSSL_CYRPTO_LIBRARY The openssl crypto library.
+# OpenSSL_SSL_LIBRARY The openssl ssl library.
+
+find_path(OpenSSL_ROOT_DIR
+ NAMES include/openssl/ssl.h
+)
+
+find_path(OpenSSL_INCLUDE_DIR
+ NAMES openssl/ssl.h
+ HINTS ${OpenSSL_ROOT_DIR}/include
+)
+
+find_library(OpenSSL_SSL_LIBRARY
+ NAMES ssl ssleay32 ssleay32MD
+ HINTS ${OpenSSL_ROOT_DIR}/lib
+)
+
+find_library(OpenSSL_CRYPTO_LIBRARY
+ NAMES crypto
+ HINTS ${OpenSSL_ROOT_DIR}/lib
+)
+
+set(OpenSSL_LIBRARIES ${OpenSSL_SSL_LIBRARY} ${OpenSSL_CRYPTO_LIBRARY}
+ CACHE STRING "OpenSSL SSL and crypto libraries" FORCE)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(OpenSSL DEFAULT_MSG
+ OpenSSL_LIBRARIES
+ OpenSSL_INCLUDE_DIR
+)
+
+mark_as_advanced(
+ OpenSSL_ROOT_DIR
+ OpenSSL_INCLUDE_DIR
+ OpenSSL_LIBRARIES
+ OpenSSL_CRYPTO_LIBRARY
+ OpenSSL_SSL_LIBRARY
+)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/configure
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/configure b/metron-sensors/bro-plugin-kafka/configure
new file mode 100755
index 0000000..d053488
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/configure
@@ -0,0 +1,130 @@
+#!/bin/sh
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Wrapper for viewing/setting options that the plugin's CMake
+# scripts will recognize.
+#
+# Don't edit this. Edit configure.plugin to add plugin-specific options.
+#
+
+set -e
+command="$0 $*"
+
+if [ -e `dirname $0`/configure.plugin ]; then
+ # Include custom additions.
+ . `dirname $0`/configure.plugin
+fi
+
+# Check for `cmake` command.
+type cmake > /dev/null 2>&1 || {
+ echo "\
+This package requires CMake, please install it first, then you may
+use this configure script to access CMake equivalent functionality.\
+" >&2;
+ exit 1;
+}
+
+usage() {
+
+cat 1>&2 <<EOF
+Usage: $0 [OPTIONS]
+
+ Plugin Options:
+ --bro-dist=DIR Path to Bro source tree
+ --install-root=DIR Path where to install plugin into
+EOF
+
+if type plugin_usage >/dev/null 2>&1; then
+ plugin_usage 1>&2
+fi
+
+echo
+
+exit 1
+}
+
+# Function to append a CMake cache entry definition to the
+# CMakeCacheEntries variable
+# $1 is the cache entry variable name
+# $2 is the cache entry variable type
+# $3 is the cache entry variable value
+append_cache_entry () {
+ CMakeCacheEntries="$CMakeCacheEntries -D $1:$2=$3"
+}
+
+# set defaults
+builddir=build
+brodist=`cd ../../.. && pwd`
+installroot="default"
+CMakeCacheEntries=""
+
+while [ $# -ne 0 ]; do
+ case "$1" in
+ -*=*) optarg=`echo "$1" | sed 's/[-_a-zA-Z0-9]*=//'` ;;
+ *) optarg= ;;
+ esac
+
+ case "$1" in
+ --help|-h)
+ usage
+ ;;
+ --bro-dist=*)
+ brodist=`cd $optarg && pwd`
+ ;;
+ --install-root=*)
+ installroot=$optarg
+ ;;
+ --with-openssl=*)
+ append_cache_entry OpenSSL_ROOT_DIR PATH $optarg
+ ;;
+ *)
+ if type plugin_option >/dev/null 2>&1; then
+ plugin_option $1 && shift && continue;
+ fi
+
+ echo "Invalid option '$1'. Try $0 --help to see available options."
+ exit 1
+ ;;
+ esac
+ shift
+done
+
+if [ ! -e "$brodist/bro-path-dev.in" ]; then
+ echo "Cannot determine Bro source directory, use --bro-dist=DIR."
+ exit 1
+fi
+
+append_cache_entry BRO_DIST PATH $brodist
+append_cache_entry CMAKE_MODULE_PATH PATH $brodist/cmake
+
+if [ "$installroot" != "default" ]; then
+ mkdir -p $installroot
+ append_cache_entry BRO_PLUGIN_INSTALL_ROOT PATH $installroot
+fi
+
+echo "Build Directory : $builddir"
+echo "Bro Source Directory : $brodist"
+
+mkdir -p $builddir
+cd $builddir
+
+cmake $CMakeCacheEntries ..
+
+echo "# This is the command used to configure this build" > config.status
+echo $command >> config.status
+chmod u+x config.status
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/configure.plugin
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/configure.plugin b/metron-sensors/bro-plugin-kafka/configure.plugin
new file mode 100644
index 0000000..1cb2086
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/configure.plugin
@@ -0,0 +1,43 @@
+#!/bin/sh
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Hooks to add custom options to the configure script.
+#
+
+plugin_usage()
+{
+ cat <<EOF
+ --with-librdkafka=PATH path to librdkafka
+ --with-openssl=PATH path to OpenSSL install root
+EOF
+}
+
+plugin_option()
+{
+ case "$1" in
+ --with-librdkafka=*)
+ append_cache_entry LibRdKafka_ROOT_DIR PATH $optarg
+ ;;
+ --with-openssl=*)
+ append_cache_entry OpenSSL_ROOT_DIR PATH $optarg
+ ;;
+ *)
+ return 1;
+ ;;
+ esac
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/scripts/Bro/Kafka/__load__.bro
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/scripts/Bro/Kafka/__load__.bro b/metron-sensors/bro-plugin-kafka/scripts/Bro/Kafka/__load__.bro
new file mode 100644
index 0000000..12295a9
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/scripts/Bro/Kafka/__load__.bro
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This is loaded when a user activates the plugin. Include scripts here that should be
+# loaded automatically at that point.
+#
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/scripts/Bro/Kafka/logs-to-kafka.bro
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/scripts/Bro/Kafka/logs-to-kafka.bro b/metron-sensors/bro-plugin-kafka/scripts/Bro/Kafka/logs-to-kafka.bro
new file mode 100644
index 0000000..84e390c
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/scripts/Bro/Kafka/logs-to-kafka.bro
@@ -0,0 +1,44 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+##! load this script to enable log output to kafka
+
+module Kafka;
+
+export {
+ ##
+ ## which log streams should be sent to kafka?
+ ## example:
+ ## redef Kafka::logs_to_send = set(Conn::Log, HTTP::LOG, DNS::LOG);
+ ##
+ const logs_to_send: set[Log::ID] &redef;
+}
+
+event bro_init() &priority=-5
+{
+ for (stream_id in Log::active_streams)
+ {
+ if (stream_id in Kafka::logs_to_send)
+ {
+ local filter: Log::Filter = [
+ $name = fmt("kafka-%s", stream_id),
+ $writer = Log::WRITER_KAFKAWRITER
+ ];
+
+ Log::add_filter(stream_id, filter);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/scripts/__load__.bro
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/scripts/__load__.bro b/metron-sensors/bro-plugin-kafka/scripts/__load__.bro
new file mode 100644
index 0000000..fee9549
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/scripts/__load__.bro
@@ -0,0 +1,25 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This is loaded unconditionally at Bro startup. Include scripts here that should
+# always be loaded.
+#
+# Normally, that will be only code that initializes built-in elements. Load
+# your standard scripts in
+# scripts/<plugin-namespace>/<plugin-name>/__load__.bro instead.
+#
+
+@load ./init.bro
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/scripts/init.bro
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/scripts/init.bro b/metron-sensors/bro-plugin-kafka/scripts/init.bro
new file mode 100644
index 0000000..c76b2a6
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/scripts/init.bro
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Kafka;
+
+export {
+ const topic_name: string = "bro" &redef;
+ const max_wait_on_shutdown: count = 3000 &redef;
+ const tag_json: bool = F &redef;
+ const kafka_conf: table[string] of string = table(
+ ["metadata.broker.list"] = "localhost:9092"
+ ) &redef;
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/src/KafkaWriter.cc
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/src/KafkaWriter.cc b/metron-sensors/bro-plugin-kafka/src/KafkaWriter.cc
new file mode 100644
index 0000000..9019790
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/src/KafkaWriter.cc
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <Type.h>
+#include <threading/Formatter.h>
+#include <threading/formatters/JSON.h>
+#include "kafka.bif.h"
+#include "TaggedJSON.h"
+#include "KafkaWriter.h"
+
+using namespace logging;
+using namespace writer;
+
+KafkaWriter::KafkaWriter(WriterFrontend* frontend): WriterBackend(frontend), formatter(NULL), producer(NULL), topic(NULL)
+{
+ // TODO do we need this??
+ topic_name.assign((const char*)BifConst::Kafka::topic_name->Bytes(),
+ BifConst::Kafka::topic_name->Len());
+}
+
+KafkaWriter::~KafkaWriter()
+{}
+
+bool KafkaWriter::DoInit(const WriterInfo& info, int num_fields, const threading::Field* const* fields)
+{
+ // initialize the formatter
+ if(BifConst::Kafka::tag_json) {
+ formatter = new threading::formatter::TaggedJSON(info.path, this, threading::formatter::JSON::TS_EPOCH);
+ } else {
+ formatter = new threading::formatter::JSON(this, threading::formatter::JSON::TS_EPOCH);
+ }
+
+ // kafka global configuration
+ string err;
+ conf = RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL);
+
+ // apply the user-defined settings to kafka
+ Val* val = BifConst::Kafka::kafka_conf->AsTableVal();
+ IterCookie* c = val->AsTable()->InitForIteration();
+ HashKey* k;
+ TableEntryVal* v;
+ while ((v = val->AsTable()->NextEntry(k, c))) {
+
+ // fetch the key and value
+ ListVal* index = val->AsTableVal()->RecoverIndex(k);
+ string key = index->Index(0)->AsString()->CheckString();
+ string val = v->Value()->AsString()->CheckString();
+
+ // apply setting to kafka
+ if (RdKafka::Conf::CONF_OK != conf->set(key, val, err)) {
+ reporter->Error("Failed to set '%s'='%s': %s", key.c_str(), val.c_str(), err.c_str());
+ return false;
+ }
+
+ // cleanup
+ Unref(index);
+ delete k;
+ }
+
+ // create kafka producer
+ producer = RdKafka::Producer::create(conf, err);
+ if (!producer) {
+ reporter->Error("Failed to create producer: %s", err.c_str());
+ return false;
+ }
+
+ // create handle to topic
+ topic_conf = RdKafka::Conf::create(RdKafka::Conf::CONF_TOPIC);
+ topic = RdKafka::Topic::create(producer, topic_name, topic_conf, err);
+ if (!topic) {
+ reporter->Error("Failed to create topic handle: %s", err.c_str());
+ return false;
+ }
+
+ return true;
+}
+
+/**
+ * Writer-specific method called just before the threading system is
+ * going to shutdown. It is assumed that once this messages returns,
+ * the thread can be safely terminated.
+ */
+bool KafkaWriter::DoFinish(double network_time)
+{
+ bool success = false;
+ int poll_interval = 1000;
+ int waited = 0;
+ int max_wait = BifConst::Kafka::max_wait_on_shutdown;
+
+ // wait a bit for queued messages to be delivered
+ while (producer->outq_len() > 0 && waited <= max_wait) {
+ producer->poll(poll_interval);
+ waited += poll_interval;
+ }
+
+ // successful only if all messages delivered
+ if (producer->outq_len() == 0) {
+ reporter->Error("Unable to deliver %0d message(s)", producer->outq_len());
+ success = true;
+ }
+
+ delete topic;
+ delete producer;
+ delete formatter;
+
+ return success;
+}
+
+/**
+ * Writer-specific output method implementing recording of one log
+ * entry.
+ */
+bool KafkaWriter::DoWrite(int num_fields, const threading::Field* const* fields, threading::Value** vals)
+{
+ ODesc buff;
+ buff.Clear();
+
+ // format the log entry
+ formatter->Describe(&buff, num_fields, fields, vals);
+
+ // send the formatted log entry to kafka
+ const char* raw = (const char*)buff.Bytes();
+ RdKafka::ErrorCode resp = producer->produce(
+ topic, RdKafka::Topic::PARTITION_UA, RdKafka::Producer::RK_MSG_COPY,
+ const_cast<char*>(raw), strlen(raw), NULL, NULL);
+
+ if (RdKafka::ERR_NO_ERROR == resp) {
+ producer->poll(0);
+ }
+ else {
+ string err = RdKafka::err2str(resp);
+ reporter->Error("Kafka send failed: %s", err.c_str());
+ }
+
+ return true;
+}
+
+/**
+ * Writer-specific method implementing a change of fthe buffering
+ * state. If buffering is disabled, the writer should attempt to
+ * write out information as quickly as possible even if doing so may
+ * have a performance impact. If enabled (which is the default), it
+ * may buffer data as helpful and write it out later in a way
+ * optimized for performance. The current buffering state can be
+ * queried via IsBuf().
+ */
+bool KafkaWriter::DoSetBuf(bool enabled)
+{
+ // no change in behavior
+ return true;
+}
+
+/**
+ * Writer-specific method implementing flushing of its output. A writer
+ * implementation must override this method but it can just
+ * ignore calls if flushing doesn't align with its semantics.
+ */
+bool KafkaWriter::DoFlush(double network_time)
+{
+ producer->poll(0);
+ return true;
+}
+
+/**
+ * Writer-specific method implementing log rotation. Most directly
+ * this only applies to writers writing into files, which should then
+ * close the current file and open a new one. However, a writer may
+ * also trigger other apppropiate actions if semantics are similar.
+ * Once rotation has finished, the implementation *must* call
+ * FinishedRotation() to signal the log manager that potential
+ * postprocessors can now run.
+ */
+bool KafkaWriter::DoRotate(const char* rotated_path, double open, double close, bool terminating)
+{
+ // no need to perform log rotation
+ return FinishedRotation();
+}
+
+/**
+ * Triggered by regular heartbeat messages from the main thread.
+ */
+bool KafkaWriter::DoHeartbeat(double network_time, double current_time)
+{
+ producer->poll(0);
+ return true;
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/src/KafkaWriter.h
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/src/KafkaWriter.h b/metron-sensors/bro-plugin-kafka/src/KafkaWriter.h
new file mode 100644
index 0000000..2299667
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/src/KafkaWriter.h
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef BRO_PLUGIN_BRO_KAFKA_KAFKAWRITER_H
+#define BRO_PLUGIN_BRO_KAFKA_KAFKAWRITER_H
+
+#include <string>
+#include <librdkafka/rdkafkacpp.h>
+#include <logging/WriterBackend.h>
+#include <threading/formatters/JSON.h>
+#include <Type.h>
+#include "kafka.bif.h"
+
+#include "TaggedJSON.h"
+
+namespace logging { namespace writer {
+
+/**
+ * A logging writer that sends data to a Kafka broker.
+ */
+class KafkaWriter : public WriterBackend {
+
+public:
+ KafkaWriter(WriterFrontend* frontend);
+ ~KafkaWriter();
+
+ static WriterBackend* Instantiate(WriterFrontend* frontend)
+ {
+ return new KafkaWriter(frontend);
+ }
+
+protected:
+ virtual bool DoInit(const WriterBackend::WriterInfo& info, int num_fields, const threading::Field* const* fields);
+ virtual bool DoWrite(int num_fields, const threading::Field* const* fields, threading::Value** vals);
+ virtual bool DoSetBuf(bool enabled);
+ virtual bool DoRotate(const char* rotated_path, double open, double close, bool terminating);
+ virtual bool DoFlush(double network_time);
+ virtual bool DoFinish(double network_time);
+ virtual bool DoHeartbeat(double network_time, double current_time);
+
+private:
+ string topic_name;
+ threading::formatter::Formatter *formatter;
+ RdKafka::Producer* producer;
+ RdKafka::Topic* topic;
+ RdKafka::Conf* conf;
+ RdKafka::Conf* topic_conf;
+};
+
+}}
+
+#endif
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/src/Plugin.cc
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/src/Plugin.cc b/metron-sensors/bro-plugin-kafka/src/Plugin.cc
new file mode 100644
index 0000000..d523d23
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/src/Plugin.cc
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Plugin.h"
+#include "KafkaWriter.h"
+
+namespace plugin { namespace Bro_Kafka {
+ Plugin plugin;
+}}
+
+using namespace plugin::Bro_Kafka;
+
+plugin::Configuration Plugin::Configure()
+{
+ AddComponent(new ::logging::Component("KafkaWriter", ::logging::writer::KafkaWriter::Instantiate));
+
+ plugin::Configuration config;
+ config.name = "Bro::Kafka";
+ config.description = "Writes logs to Kafka";
+ config.version.major = 0;
+ config.version.minor = 1;
+ return config;
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/src/Plugin.h
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/src/Plugin.h b/metron-sensors/bro-plugin-kafka/src/Plugin.h
new file mode 100644
index 0000000..8adeb18
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/src/Plugin.h
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef BRO_PLUGIN_BRO_KAFKA
+#define BRO_PLUGIN_BRO_KAFKA
+
+#include <plugin/Plugin.h>
+
+namespace plugin { namespace Bro_Kafka {
+
+ class Plugin : public ::plugin::Plugin {
+ protected:
+ // Overridden from plugin::Plugin.
+ virtual plugin::Configuration Configure();
+ };
+
+ extern Plugin plugin;
+}}
+
+#endif
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/src/TaggedJSON.cc
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/src/TaggedJSON.cc b/metron-sensors/bro-plugin-kafka/src/TaggedJSON.cc
new file mode 100644
index 0000000..db3f305
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/src/TaggedJSON.cc
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "TaggedJSON.h"
+
+namespace threading { namespace formatter {
+
+TaggedJSON::TaggedJSON(string sn, MsgThread* t, JSON::TimeFormat tf): JSON(t, tf), stream_name(sn)
+{}
+
+TaggedJSON::~TaggedJSON()
+{}
+
+bool TaggedJSON::Describe(ODesc* desc, int num_fields, const Field* const* fields, Value** vals) const
+{
+ desc->AddRaw("{");
+
+ // 'tag' the json; aka prepend the stream name to the json-formatted log content
+ desc->AddRaw("\"");
+ desc->AddRaw(stream_name);
+ desc->AddRaw("\": ");
+
+ // append the JSON formatted log record itself
+ JSON::Describe(desc, num_fields, fields, vals);
+
+ desc->AddRaw("}");
+ return true;
+}
+}}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/src/TaggedJSON.h
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/src/TaggedJSON.h b/metron-sensors/bro-plugin-kafka/src/TaggedJSON.h
new file mode 100644
index 0000000..08a50df
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/src/TaggedJSON.h
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef BRO_PLUGIN_BRO_KAFKA_TAGGEDJSON_H
+#define BRO_PLUGIN_BRO_KAFKA_TAGGEDJSON_H
+
+#include <string>
+#include <threading/Formatter.h>
+#include <threading/formatters/JSON.h>
+
+using threading::formatter::JSON;
+using threading::MsgThread;
+using threading::Value;
+using threading::Field;
+
+namespace threading { namespace formatter {
+
+/*
+ * A JSON formatter that prepends or 'tags' the content with a log stream
+ * identifier. For example,
+ * { 'conn' : { ... }}
+ * { 'http' : { ... }}
+ */
+class TaggedJSON : public JSON {
+
+public:
+ TaggedJSON(string stream_name, MsgThread* t, JSON::TimeFormat tf);
+ virtual ~TaggedJSON();
+ virtual bool Describe(ODesc* desc, int num_fields, const Field* const* fields, Value** vals) const;
+
+private:
+ string stream_name;
+};
+
+}}
+#endif
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/src/kafka.bif
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/src/kafka.bif b/metron-sensors/bro-plugin-kafka/src/kafka.bif
new file mode 100644
index 0000000..8a8070c
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/src/kafka.bif
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Kafka;
+
+const kafka_conf: config;
+const topic_name: string;
+const max_wait_on_shutdown: count;
+const tag_json: bool;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/src/kafka_const.bif
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/src/kafka_const.bif b/metron-sensors/bro-plugin-kafka/src/kafka_const.bif
new file mode 100644
index 0000000..989c0ae
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/src/kafka_const.bif
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Kafka;
+
+type config : table[string] of string;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/tests/Makefile
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/tests/Makefile b/metron-sensors/bro-plugin-kafka/tests/Makefile
new file mode 100644
index 0000000..a637cd3
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/tests/Makefile
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+test:
+ @btest
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/tests/Scripts/get-bro-env
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/tests/Scripts/get-bro-env b/metron-sensors/bro-plugin-kafka/tests/Scripts/get-bro-env
new file mode 100755
index 0000000..8aa0ea7
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/tests/Scripts/get-bro-env
@@ -0,0 +1,36 @@
+#! /bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# BTest helper for getting values for Bro-related environment variables.
+
+base=`dirname $0`
+bro=`cat ${base}/../../build/CMakeCache.txt | grep BRO_DIST | cut -d = -f 2`
+
+if [ "$1" = "brobase" ]; then
+ echo ${bro}
+elif [ "$1" = "bropath" ]; then
+ ${bro}/build/bro-path-dev
+elif [ "$1" = "bro_plugin_path" ]; then
+ ( cd ${base}/../.. && pwd )
+elif [ "$1" = "bro_seed_file" ]; then
+ echo ${bro}/testing/btest/random.seed
+elif [ "$1" = "path" ]; then
+ echo ${bro}/build/src:${bro}/aux/btest:${base}/:${bro}/aux/bro-cut:$PATH
+else
+ echo "usage: `basename $0` <var>" >&2
+ exit 1
+fi
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/tests/btest.cfg
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/tests/btest.cfg b/metron-sensors/bro-plugin-kafka/tests/btest.cfg
new file mode 100644
index 0000000..e42fefd
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/tests/btest.cfg
@@ -0,0 +1,36 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+[btest]
+TestDirs = kafka
+TmpDir = %(testbase)s/.tmp
+BaselineDir = %(testbase)s/Baseline
+IgnoreDirs = .svn CVS .tmp
+IgnoreFiles = *.tmp *.swp #* *.trace .DS_Store
+
+[environment]
+BROBASE=`%(testbase)s/Scripts/get-bro-env brobase`
+BROPATH=`%(testbase)s/Scripts/get-bro-env bropath`
+BRO_PLUGIN_PATH=`%(testbase)s/Scripts/get-bro-env bro_plugin_path`
+BRO_SEED_FILE=`%(testbase)s/Scripts/get-bro-env bro_seed_file`
+PATH=`%(testbase)s/Scripts/get-bro-env path`
+TZ=UTC
+LC_ALL=C
+TRACES=%(testbase)s/Traces
+TMPDIR=%(testbase)s/.tmp
+BRO_TRACES=`%(testbase)s/Scripts/get-bro-env brobase`/testing/btest/Traces
+TEST_DIFF_CANONIFIER=`%(testbase)s/Scripts/get-bro-env brobase`/testing/scripts/diff-canonifier
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/bro-plugin-kafka/tests/kafka/show-plugin.bro
----------------------------------------------------------------------
diff --git a/metron-sensors/bro-plugin-kafka/tests/kafka/show-plugin.bro b/metron-sensors/bro-plugin-kafka/tests/kafka/show-plugin.bro
new file mode 100644
index 0000000..4e8dd6a
--- /dev/null
+++ b/metron-sensors/bro-plugin-kafka/tests/kafka/show-plugin.bro
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# @TEST-EXEC: bro -NN Bro::Kafka >output
+# @TEST-EXEC: btest-diff output
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/metron-sensors/packet-capture/.gitignore
----------------------------------------------------------------------
diff --git a/metron-sensors/packet-capture/.gitignore b/metron-sensors/packet-capture/.gitignore
new file mode 100644
index 0000000..2efc4c0
--- /dev/null
+++ b/metron-sensors/packet-capture/.gitignore
@@ -0,0 +1,3 @@
+roles
+.vagrant
+*.retry
[3/3] incubator-metron git commit: METRON-108 Create Fast Packet
Capture Process (nickwallen) closes apache/incubator-metron#73
Posted by ni...@apache.org.
METRON-108 Create Fast Packet Capture Process (nickwallen) closes apache/incubator-metron#73
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/564272e6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/564272e6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/564272e6
Branch: refs/heads/master
Commit: 564272e6ad086bd290836184c5691dbceaeab5cf
Parents: 8d5fb1b
Author: nickwallen <ni...@nickallen.org>
Authored: Wed Apr 13 10:47:54 2016 -0400
Committer: Nick Allen <ni...@nickallen.org>
Committed: Wed Apr 13 10:47:54 2016 -0400
----------------------------------------------------------------------
bro-plugin-kafka/.gitignore | 31 --
bro-plugin-kafka/CHANGES | 16 -
bro-plugin-kafka/CMakeLists.txt | 43 ---
bro-plugin-kafka/COPYING | 201 -----------
bro-plugin-kafka/MAINTAINER | 18 -
bro-plugin-kafka/Makefile | 44 ---
bro-plugin-kafka/README | 96 -----
bro-plugin-kafka/VERSION | 18 -
bro-plugin-kafka/cmake/FindLibRDKafka.cmake | 49 ---
bro-plugin-kafka/cmake/FindOpenSSL.cmake | 72 ----
bro-plugin-kafka/configure | 130 -------
bro-plugin-kafka/configure.plugin | 43 ---
bro-plugin-kafka/scripts/Bro/Kafka/__load__.bro | 19 -
.../scripts/Bro/Kafka/logs-to-kafka.bro | 44 ---
bro-plugin-kafka/scripts/__load__.bro | 25 --
bro-plugin-kafka/scripts/init.bro | 27 --
bro-plugin-kafka/src/KafkaWriter.cc | 200 -----------
bro-plugin-kafka/src/KafkaWriter.h | 66 ----
bro-plugin-kafka/src/Plugin.cc | 37 --
bro-plugin-kafka/src/Plugin.h | 34 --
bro-plugin-kafka/src/TaggedJSON.cc | 43 ---
bro-plugin-kafka/src/TaggedJSON.h | 50 ---
bro-plugin-kafka/src/kafka.bif | 23 --
bro-plugin-kafka/src/kafka_const.bif | 20 --
bro-plugin-kafka/tests/Makefile | 19 -
bro-plugin-kafka/tests/Scripts/get-bro-env | 36 --
bro-plugin-kafka/tests/btest.cfg | 36 --
bro-plugin-kafka/tests/kafka/show-plugin.bro | 19 -
deployment/roles/bro/meta/main.yml | 4 +-
deployment/roles/bro/tasks/bro-plugin-kafka.yml | 2 +-
deployment/roles/bro/vars/main.yml | 5 -
deployment/roles/kafka-broker/defaults/main.yml | 18 +
deployment/roles/kafka-broker/meta/main.yml | 18 +
deployment/roles/kafka-broker/tasks/main.yml | 41 +++
deployment/roles/kafka-broker/vars/main.yml | 18 +
deployment/roles/librdkafka/defaults/main.yml | 20 ++
.../roles/librdkafka/tasks/dependencies.yml | 37 ++
.../roles/librdkafka/tasks/librdkafka.yml | 39 +++
deployment/roles/librdkafka/tasks/main.yml | 19 +
.../roles/packet-capture/defaults/main.yml | 32 ++
deployment/roles/packet-capture/meta/main.yml | 19 +
deployment/roles/packet-capture/tasks/debug.yml | 26 ++
.../roles/packet-capture/tasks/dependencies.yml | 38 ++
deployment/roles/packet-capture/tasks/dpdk.yml | 59 ++++
.../roles/packet-capture/tasks/kernel.yml | 51 +++
deployment/roles/packet-capture/tasks/main.yml | 22 ++
.../roles/packet-capture/tasks/pcapture.yml | 49 +++
.../roles/packet-capture/templates/pcapture | 93 +++++
.../packet-capture/templates/pcapture.conf | 67 ++++
deployment/vagrant/packet-capture/Vagrantfile | 69 ++++
deployment/vagrant/packet-capture/ansible.cfg | 22 ++
deployment/vagrant/packet-capture/playbook.yml | 43 +++
metron-sensors/bro-plugin-kafka/.gitignore | 31 ++
metron-sensors/bro-plugin-kafka/CHANGES | 16 +
metron-sensors/bro-plugin-kafka/CMakeLists.txt | 43 +++
metron-sensors/bro-plugin-kafka/COPYING | 201 +++++++++++
metron-sensors/bro-plugin-kafka/MAINTAINER | 18 +
metron-sensors/bro-plugin-kafka/Makefile | 44 +++
metron-sensors/bro-plugin-kafka/README | 92 +++++
metron-sensors/bro-plugin-kafka/VERSION | 18 +
.../bro-plugin-kafka/cmake/FindLibRDKafka.cmake | 49 +++
.../bro-plugin-kafka/cmake/FindOpenSSL.cmake | 72 ++++
metron-sensors/bro-plugin-kafka/configure | 130 +++++++
.../bro-plugin-kafka/configure.plugin | 43 +++
.../scripts/Bro/Kafka/__load__.bro | 19 +
.../scripts/Bro/Kafka/logs-to-kafka.bro | 44 +++
.../bro-plugin-kafka/scripts/__load__.bro | 25 ++
.../bro-plugin-kafka/scripts/init.bro | 27 ++
.../bro-plugin-kafka/src/KafkaWriter.cc | 200 +++++++++++
.../bro-plugin-kafka/src/KafkaWriter.h | 66 ++++
metron-sensors/bro-plugin-kafka/src/Plugin.cc | 37 ++
metron-sensors/bro-plugin-kafka/src/Plugin.h | 34 ++
.../bro-plugin-kafka/src/TaggedJSON.cc | 43 +++
.../bro-plugin-kafka/src/TaggedJSON.h | 50 +++
metron-sensors/bro-plugin-kafka/src/kafka.bif | 23 ++
.../bro-plugin-kafka/src/kafka_const.bif | 20 ++
metron-sensors/bro-plugin-kafka/tests/Makefile | 19 +
.../bro-plugin-kafka/tests/Scripts/get-bro-env | 36 ++
metron-sensors/bro-plugin-kafka/tests/btest.cfg | 36 ++
.../tests/kafka/show-plugin.bro | 19 +
metron-sensors/packet-capture/.gitignore | 3 +
metron-sensors/packet-capture/Makefile | 26 ++
metron-sensors/packet-capture/README.md | 12 +
.../packet-capture/conf/localhost.kafka | 67 ++++
metron-sensors/packet-capture/src/.gitignore | 1 +
metron-sensors/packet-capture/src/Makefile | 52 +++
metron-sensors/packet-capture/src/args.c | 145 ++++++++
metron-sensors/packet-capture/src/args.h | 107 ++++++
metron-sensors/packet-capture/src/kafka.c | 224 ++++++++++++
metron-sensors/packet-capture/src/kafka.h | 45 +++
metron-sensors/packet-capture/src/main.c | 349 +++++++++++++++++++
metron-sensors/packet-capture/src/main.h | 122 +++++++
92 files changed, 3411 insertions(+), 1467 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/.gitignore
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/.gitignore b/bro-plugin-kafka/.gitignore
deleted file mode 100644
index 28a8358..0000000
--- a/bro-plugin-kafka/.gitignore
+++ /dev/null
@@ -1,31 +0,0 @@
-.state
-build
-
-# Compiled Object files
-*.slo
-*.lo
-*.o
-*.obj
-
-# Precompiled Headers
-*.gch
-*.pch
-
-# Compiled Dynamic libraries
-*.so
-*.dylib
-*.dll
-
-# Fortran module files
-*.mod
-
-# Compiled Static libraries
-*.lai
-*.la
-*.a
-*.lib
-
-# Executables
-*.exe
-*.out
-*.app
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/CHANGES
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/CHANGES b/bro-plugin-kafka/CHANGES
deleted file mode 100644
index d9e26de..0000000
--- a/bro-plugin-kafka/CHANGES
+++ /dev/null
@@ -1,16 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/CMakeLists.txt b/bro-plugin-kafka/CMakeLists.txt
deleted file mode 100644
index 30bf3b5..0000000
--- a/bro-plugin-kafka/CMakeLists.txt
+++ /dev/null
@@ -1,43 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-cmake_minimum_required(VERSION 2.8)
-project(Plugin)
-include(BroPlugin)
-find_package(LibRDKafka)
-find_package(OpenSSL)
-
-if (LIBRDKAFKA_FOUND AND OPENSSL_FOUND)
- include_directories(BEFORE ${LibRDKafka_INCLUDE_DIR} ${OpenSSL_INCLUDE_DIR})
- bro_plugin_begin(BRO KAFKA)
- bro_plugin_cc(src/KafkaWriter.cc)
- bro_plugin_cc(src/Plugin.cc)
- bro_plugin_cc(src/TaggedJSON.cc)
- bro_plugin_bif(src/kafka.bif)
- bro_plugin_dist_files(README CHANGES COPYING VERSION)
- bro_plugin_link_library(${LibRDKafka_LIBRARIES})
- bro_plugin_link_library(${LibRDKafka_C_LIBRARIES})
- bro_plugin_link_library(${OpenSSL_LIBRARIES})
- bro_plugin_end()
-
-elseif (NOT LIBRDKAFKA_FOUND)
- message(FATAL_ERROR "LibRDKafka not found.")
-
-elseif (NOT OPENSSL_FOUND)
- message(FATAL_ERROR "OpenSSL not found.")
-
-endif ()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/COPYING
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/COPYING b/bro-plugin-kafka/COPYING
deleted file mode 100644
index 5c304d1..0000000
--- a/bro-plugin-kafka/COPYING
+++ /dev/null
@@ -1,201 +0,0 @@
-Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "{}"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright {yyyy} {name of copyright owner}
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/MAINTAINER
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/MAINTAINER b/bro-plugin-kafka/MAINTAINER
deleted file mode 100644
index 0ddaed7..0000000
--- a/bro-plugin-kafka/MAINTAINER
+++ /dev/null
@@ -1,18 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-Apache Metron <us...@metron.incubator.apache.org>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/Makefile
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/Makefile b/bro-plugin-kafka/Makefile
deleted file mode 100644
index 50fa3ca..0000000
--- a/bro-plugin-kafka/Makefile
+++ /dev/null
@@ -1,44 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Convenience Makefile providing a few common top-level targets.
-#
-
-cmake_build_dir=build
-arch=`uname -s | tr A-Z a-z`-`uname -m`
-
-all: build-it
-
-build-it:
- @test -e $(cmake_build_dir)/config.status || ./configure
- -@test -e $(cmake_build_dir)/CMakeCache.txt && \
- test $(cmake_build_dir)/CMakeCache.txt -ot `cat $(cmake_build_dir)/CMakeCache.txt | grep BRO_DIST | cut -d '=' -f 2`/build/CMakeCache.txt && \
- echo Updating stale CMake cache && \
- touch $(cmake_build_dir)/CMakeCache.txt
-
- ( cd $(cmake_build_dir) && make )
-
-install:
- ( cd $(cmake_build_dir) && make install )
-
-clean:
- ( cd $(cmake_build_dir) && make clean )
-
-distclean:
- rm -rf $(cmake_build_dir)
-
-test:
- make -C tests
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/README
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/README b/bro-plugin-kafka/README
deleted file mode 100644
index c672bd0..0000000
--- a/bro-plugin-kafka/README
+++ /dev/null
@@ -1,96 +0,0 @@
-
-===============================
-Writing Logging Output to Kafka
-===============================
-
-A log writer that sends logging output to Kafka. This provides a convenient
-means for tools in the Hadoop ecosystem, such as Storm, Spark, and others, to
-process the data generated by Bro.
-
-.. contents::
-
-Installation
-------------
-
-Install librdkafka (https://github.com/edenhill/librdkafka), a native client
-library for Kafka. This plugin has been tested against the latest release of
-librdkafka, which at the time of this writing is v0.8.6.
-
- # curl -L https://github.com/edenhill/librdkafka/archive/0.8.6.tar.gz | tar xvz
- # cd librdkafka-0.8.6/
- # ./configure
- # make
- # sudo make install
-
-Then compile this Bro plugin using the following commands.
-
- # ./configure --bro-dist=$BRO_SRC
- # make
- # sudo make install
-
-Run the following command to ensure that the plugin was installed successfully.
-
- # bro -N Bro::Kafka
- Bro::Kafka - Writes logs to Kafka (dynamic, version 0.1)
-
-Activation
-----------
-
-The easiest way to enable Kafka output is to load the plugin's
-``logs-to-kafka.bro`` script. If you are using BroControl, the following lines
-added to local.bro will activate it.
-
-.. console::
-
- @load Bro/Kafka/logs-to-kafka.bro
- redef Kafka::logs_to_send = set(Conn::LOG, HTTP::LOG, DNS::LOG);
- redef Kafka::topic_name = "bro";
- redef Kafka::kafka_conf = table(
- ["metadata.broker.list"] = "localhost:9092"
- );
-
-This example will send all HTTP, DNS, and Conn logs to a Kafka broker running on
-the localhost to a topic called ``bro``. Any configuration value accepted by
-librdkafka can be added to the ``kafka_conf`` configuration table.
-
-Settings
---------
-
-``kafka_conf``
-
-The global configuration settings for Kafka. These values are passed through
-directly to librdkafka. Any valid librdkafka settings can be defined in this
-table.
-
-.. console::
-
- redef Kafka::kafka_conf = table(
- ["metadata.broker.list"] = "localhost:9092",
- ["client.id"] = "bro"
- );
-
-``topic_name``
-
-The name of the topic in Kafka where all Bro logs will be sent to.
-
-.. console::
-
- redef Kafka::topic_name = "bro";
-
-``max_wait_on_shutdown``
-
-The maximum number of milliseconds that the plugin will wait for any backlog of
-queued messages to be sent to Kafka before forced shutdown.
-
-.. console::
-
- redef Kafka::max_wait_on_shutdown = 3000;
-
-``tag_json``
-
-If true, a log stream identifier is appended to each JSON-formatted message. For
-example, a Conn::LOG message will look like ``{ 'conn' : { ... }}``.
-
-.. console::
-
- redef Kafka::tag_json = T;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/VERSION
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/VERSION b/bro-plugin-kafka/VERSION
deleted file mode 100644
index 204e5ca..0000000
--- a/bro-plugin-kafka/VERSION
+++ /dev/null
@@ -1,18 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-0.1
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/cmake/FindLibRDKafka.cmake
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/cmake/FindLibRDKafka.cmake b/bro-plugin-kafka/cmake/FindLibRDKafka.cmake
deleted file mode 100644
index c64d8f9..0000000
--- a/bro-plugin-kafka/cmake/FindLibRDKafka.cmake
+++ /dev/null
@@ -1,49 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-find_path(LibRDKafka_ROOT_DIR
- NAMES include/librdkafka/rdkafkacpp.h
-)
-
-find_library(LibRDKafka_LIBRARIES
- NAMES rdkafka++
- HINTS ${LibRDKafka_ROOT_DIR}/lib
-)
-
-find_library(LibRDKafka_C_LIBRARIES
- NAMES rdkafka
- HINTS ${LibRDKafka_ROT_DIR}/lib
-)
-
-find_path(LibRDKafka_INCLUDE_DIR
- NAMES librdkafka/rdkafkacpp.h
- HINTS ${LibRDKafka_ROOT_DIR}/include
-)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(LibRDKafka DEFAULT_MSG
- LibRDKafka_LIBRARIES
- LibRDKafka_C_LIBRARIES
- LibRDKafka_INCLUDE_DIR
-)
-
-mark_as_advanced(
- LibRDKafka_ROOT_DIR
- LibRDKafka_LIBRARIES
- LibRDKafka_C_LIBRARIES
- LibRDKafka_INCLUDE_DIR
-)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/cmake/FindOpenSSL.cmake
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/cmake/FindOpenSSL.cmake b/bro-plugin-kafka/cmake/FindOpenSSL.cmake
deleted file mode 100644
index 5ed955c..0000000
--- a/bro-plugin-kafka/cmake/FindOpenSSL.cmake
+++ /dev/null
@@ -1,72 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# - Try to find openssl include dirs and libraries
-#
-# Usage of this module as follows:
-#
-# find_package(OpenSSL)
-#
-# Variables used by this module, they can change the default behaviour and need
-# to be set before calling find_package:
-#
-# OpenSSL_ROOT_DIR Set this variable to the root installation of
-# openssl if the module has problems finding the
-# proper installation path.
-#
-# Variables defined by this module:
-#
-# OPENSSL_FOUND System has openssl, include and library dirs found
-# OpenSSL_INCLUDE_DIR The openssl include directories.
-# OpenSSL_LIBRARIES The openssl libraries.
-# OpenSSL_CYRPTO_LIBRARY The openssl crypto library.
-# OpenSSL_SSL_LIBRARY The openssl ssl library.
-
-find_path(OpenSSL_ROOT_DIR
- NAMES include/openssl/ssl.h
-)
-
-find_path(OpenSSL_INCLUDE_DIR
- NAMES openssl/ssl.h
- HINTS ${OpenSSL_ROOT_DIR}/include
-)
-
-find_library(OpenSSL_SSL_LIBRARY
- NAMES ssl ssleay32 ssleay32MD
- HINTS ${OpenSSL_ROOT_DIR}/lib
-)
-
-find_library(OpenSSL_CRYPTO_LIBRARY
- NAMES crypto
- HINTS ${OpenSSL_ROOT_DIR}/lib
-)
-
-set(OpenSSL_LIBRARIES ${OpenSSL_SSL_LIBRARY} ${OpenSSL_CRYPTO_LIBRARY}
- CACHE STRING "OpenSSL SSL and crypto libraries" FORCE)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(OpenSSL DEFAULT_MSG
- OpenSSL_LIBRARIES
- OpenSSL_INCLUDE_DIR
-)
-
-mark_as_advanced(
- OpenSSL_ROOT_DIR
- OpenSSL_INCLUDE_DIR
- OpenSSL_LIBRARIES
- OpenSSL_CRYPTO_LIBRARY
- OpenSSL_SSL_LIBRARY
-)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/configure
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/configure b/bro-plugin-kafka/configure
deleted file mode 100755
index d053488..0000000
--- a/bro-plugin-kafka/configure
+++ /dev/null
@@ -1,130 +0,0 @@
-#!/bin/sh
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Wrapper for viewing/setting options that the plugin's CMake
-# scripts will recognize.
-#
-# Don't edit this. Edit configure.plugin to add plugin-specific options.
-#
-
-set -e
-command="$0 $*"
-
-if [ -e `dirname $0`/configure.plugin ]; then
- # Include custom additions.
- . `dirname $0`/configure.plugin
-fi
-
-# Check for `cmake` command.
-type cmake > /dev/null 2>&1 || {
- echo "\
-This package requires CMake, please install it first, then you may
-use this configure script to access CMake equivalent functionality.\
-" >&2;
- exit 1;
-}
-
-usage() {
-
-cat 1>&2 <<EOF
-Usage: $0 [OPTIONS]
-
- Plugin Options:
- --bro-dist=DIR Path to Bro source tree
- --install-root=DIR Path where to install plugin into
-EOF
-
-if type plugin_usage >/dev/null 2>&1; then
- plugin_usage 1>&2
-fi
-
-echo
-
-exit 1
-}
-
-# Function to append a CMake cache entry definition to the
-# CMakeCacheEntries variable
-# $1 is the cache entry variable name
-# $2 is the cache entry variable type
-# $3 is the cache entry variable value
-append_cache_entry () {
- CMakeCacheEntries="$CMakeCacheEntries -D $1:$2=$3"
-}
-
-# set defaults
-builddir=build
-brodist=`cd ../../.. && pwd`
-installroot="default"
-CMakeCacheEntries=""
-
-while [ $# -ne 0 ]; do
- case "$1" in
- -*=*) optarg=`echo "$1" | sed 's/[-_a-zA-Z0-9]*=//'` ;;
- *) optarg= ;;
- esac
-
- case "$1" in
- --help|-h)
- usage
- ;;
- --bro-dist=*)
- brodist=`cd $optarg && pwd`
- ;;
- --install-root=*)
- installroot=$optarg
- ;;
- --with-openssl=*)
- append_cache_entry OpenSSL_ROOT_DIR PATH $optarg
- ;;
- *)
- if type plugin_option >/dev/null 2>&1; then
- plugin_option $1 && shift && continue;
- fi
-
- echo "Invalid option '$1'. Try $0 --help to see available options."
- exit 1
- ;;
- esac
- shift
-done
-
-if [ ! -e "$brodist/bro-path-dev.in" ]; then
- echo "Cannot determine Bro source directory, use --bro-dist=DIR."
- exit 1
-fi
-
-append_cache_entry BRO_DIST PATH $brodist
-append_cache_entry CMAKE_MODULE_PATH PATH $brodist/cmake
-
-if [ "$installroot" != "default" ]; then
- mkdir -p $installroot
- append_cache_entry BRO_PLUGIN_INSTALL_ROOT PATH $installroot
-fi
-
-echo "Build Directory : $builddir"
-echo "Bro Source Directory : $brodist"
-
-mkdir -p $builddir
-cd $builddir
-
-cmake $CMakeCacheEntries ..
-
-echo "# This is the command used to configure this build" > config.status
-echo $command >> config.status
-chmod u+x config.status
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/configure.plugin
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/configure.plugin b/bro-plugin-kafka/configure.plugin
deleted file mode 100644
index 1cb2086..0000000
--- a/bro-plugin-kafka/configure.plugin
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/bin/sh
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Hooks to add custom options to the configure script.
-#
-
-plugin_usage()
-{
- cat <<EOF
- --with-librdkafka=PATH path to librdkafka
- --with-openssl=PATH path to OpenSSL install root
-EOF
-}
-
-plugin_option()
-{
- case "$1" in
- --with-librdkafka=*)
- append_cache_entry LibRdKafka_ROOT_DIR PATH $optarg
- ;;
- --with-openssl=*)
- append_cache_entry OpenSSL_ROOT_DIR PATH $optarg
- ;;
- *)
- return 1;
- ;;
- esac
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/scripts/Bro/Kafka/__load__.bro
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/scripts/Bro/Kafka/__load__.bro b/bro-plugin-kafka/scripts/Bro/Kafka/__load__.bro
deleted file mode 100644
index 12295a9..0000000
--- a/bro-plugin-kafka/scripts/Bro/Kafka/__load__.bro
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# This is loaded when a user activates the plugin. Include scripts here that should be
-# loaded automatically at that point.
-#
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/scripts/Bro/Kafka/logs-to-kafka.bro
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/scripts/Bro/Kafka/logs-to-kafka.bro b/bro-plugin-kafka/scripts/Bro/Kafka/logs-to-kafka.bro
deleted file mode 100644
index 84e390c..0000000
--- a/bro-plugin-kafka/scripts/Bro/Kafka/logs-to-kafka.bro
+++ /dev/null
@@ -1,44 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-##! load this script to enable log output to kafka
-
-module Kafka;
-
-export {
- ##
- ## which log streams should be sent to kafka?
- ## example:
- ## redef Kafka::logs_to_send = set(Conn::Log, HTTP::LOG, DNS::LOG);
- ##
- const logs_to_send: set[Log::ID] &redef;
-}
-
-event bro_init() &priority=-5
-{
- for (stream_id in Log::active_streams)
- {
- if (stream_id in Kafka::logs_to_send)
- {
- local filter: Log::Filter = [
- $name = fmt("kafka-%s", stream_id),
- $writer = Log::WRITER_KAFKAWRITER
- ];
-
- Log::add_filter(stream_id, filter);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/scripts/__load__.bro
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/scripts/__load__.bro b/bro-plugin-kafka/scripts/__load__.bro
deleted file mode 100644
index fee9549..0000000
--- a/bro-plugin-kafka/scripts/__load__.bro
+++ /dev/null
@@ -1,25 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# This is loaded unconditionally at Bro startup. Include scripts here that should
-# always be loaded.
-#
-# Normally, that will be only code that initializes built-in elements. Load
-# your standard scripts in
-# scripts/<plugin-namespace>/<plugin-name>/__load__.bro instead.
-#
-
-@load ./init.bro
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/scripts/init.bro
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/scripts/init.bro b/bro-plugin-kafka/scripts/init.bro
deleted file mode 100644
index c76b2a6..0000000
--- a/bro-plugin-kafka/scripts/init.bro
+++ /dev/null
@@ -1,27 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-module Kafka;
-
-export {
- const topic_name: string = "bro" &redef;
- const max_wait_on_shutdown: count = 3000 &redef;
- const tag_json: bool = F &redef;
- const kafka_conf: table[string] of string = table(
- ["metadata.broker.list"] = "localhost:9092"
- ) &redef;
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/src/KafkaWriter.cc
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/src/KafkaWriter.cc b/bro-plugin-kafka/src/KafkaWriter.cc
deleted file mode 100644
index 9019790..0000000
--- a/bro-plugin-kafka/src/KafkaWriter.cc
+++ /dev/null
@@ -1,200 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <Type.h>
-#include <threading/Formatter.h>
-#include <threading/formatters/JSON.h>
-#include "kafka.bif.h"
-#include "TaggedJSON.h"
-#include "KafkaWriter.h"
-
-using namespace logging;
-using namespace writer;
-
-KafkaWriter::KafkaWriter(WriterFrontend* frontend): WriterBackend(frontend), formatter(NULL), producer(NULL), topic(NULL)
-{
- // TODO do we need this??
- topic_name.assign((const char*)BifConst::Kafka::topic_name->Bytes(),
- BifConst::Kafka::topic_name->Len());
-}
-
-KafkaWriter::~KafkaWriter()
-{}
-
-bool KafkaWriter::DoInit(const WriterInfo& info, int num_fields, const threading::Field* const* fields)
-{
- // initialize the formatter
- if(BifConst::Kafka::tag_json) {
- formatter = new threading::formatter::TaggedJSON(info.path, this, threading::formatter::JSON::TS_EPOCH);
- } else {
- formatter = new threading::formatter::JSON(this, threading::formatter::JSON::TS_EPOCH);
- }
-
- // kafka global configuration
- string err;
- conf = RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL);
-
- // apply the user-defined settings to kafka
- Val* val = BifConst::Kafka::kafka_conf->AsTableVal();
- IterCookie* c = val->AsTable()->InitForIteration();
- HashKey* k;
- TableEntryVal* v;
- while ((v = val->AsTable()->NextEntry(k, c))) {
-
- // fetch the key and value
- ListVal* index = val->AsTableVal()->RecoverIndex(k);
- string key = index->Index(0)->AsString()->CheckString();
- string val = v->Value()->AsString()->CheckString();
-
- // apply setting to kafka
- if (RdKafka::Conf::CONF_OK != conf->set(key, val, err)) {
- reporter->Error("Failed to set '%s'='%s': %s", key.c_str(), val.c_str(), err.c_str());
- return false;
- }
-
- // cleanup
- Unref(index);
- delete k;
- }
-
- // create kafka producer
- producer = RdKafka::Producer::create(conf, err);
- if (!producer) {
- reporter->Error("Failed to create producer: %s", err.c_str());
- return false;
- }
-
- // create handle to topic
- topic_conf = RdKafka::Conf::create(RdKafka::Conf::CONF_TOPIC);
- topic = RdKafka::Topic::create(producer, topic_name, topic_conf, err);
- if (!topic) {
- reporter->Error("Failed to create topic handle: %s", err.c_str());
- return false;
- }
-
- return true;
-}
-
-/**
- * Writer-specific method called just before the threading system is
- * going to shutdown. It is assumed that once this messages returns,
- * the thread can be safely terminated.
- */
-bool KafkaWriter::DoFinish(double network_time)
-{
- bool success = false;
- int poll_interval = 1000;
- int waited = 0;
- int max_wait = BifConst::Kafka::max_wait_on_shutdown;
-
- // wait a bit for queued messages to be delivered
- while (producer->outq_len() > 0 && waited <= max_wait) {
- producer->poll(poll_interval);
- waited += poll_interval;
- }
-
- // successful only if all messages delivered
- if (producer->outq_len() == 0) {
- reporter->Error("Unable to deliver %0d message(s)", producer->outq_len());
- success = true;
- }
-
- delete topic;
- delete producer;
- delete formatter;
-
- return success;
-}
-
-/**
- * Writer-specific output method implementing recording of one log
- * entry.
- */
-bool KafkaWriter::DoWrite(int num_fields, const threading::Field* const* fields, threading::Value** vals)
-{
- ODesc buff;
- buff.Clear();
-
- // format the log entry
- formatter->Describe(&buff, num_fields, fields, vals);
-
- // send the formatted log entry to kafka
- const char* raw = (const char*)buff.Bytes();
- RdKafka::ErrorCode resp = producer->produce(
- topic, RdKafka::Topic::PARTITION_UA, RdKafka::Producer::RK_MSG_COPY,
- const_cast<char*>(raw), strlen(raw), NULL, NULL);
-
- if (RdKafka::ERR_NO_ERROR == resp) {
- producer->poll(0);
- }
- else {
- string err = RdKafka::err2str(resp);
- reporter->Error("Kafka send failed: %s", err.c_str());
- }
-
- return true;
-}
-
-/**
- * Writer-specific method implementing a change of fthe buffering
- * state. If buffering is disabled, the writer should attempt to
- * write out information as quickly as possible even if doing so may
- * have a performance impact. If enabled (which is the default), it
- * may buffer data as helpful and write it out later in a way
- * optimized for performance. The current buffering state can be
- * queried via IsBuf().
- */
-bool KafkaWriter::DoSetBuf(bool enabled)
-{
- // no change in behavior
- return true;
-}
-
-/**
- * Writer-specific method implementing flushing of its output. A writer
- * implementation must override this method but it can just
- * ignore calls if flushing doesn't align with its semantics.
- */
-bool KafkaWriter::DoFlush(double network_time)
-{
- producer->poll(0);
- return true;
-}
-
-/**
- * Writer-specific method implementing log rotation. Most directly
- * this only applies to writers writing into files, which should then
- * close the current file and open a new one. However, a writer may
- * also trigger other apppropiate actions if semantics are similar.
- * Once rotation has finished, the implementation *must* call
- * FinishedRotation() to signal the log manager that potential
- * postprocessors can now run.
- */
-bool KafkaWriter::DoRotate(const char* rotated_path, double open, double close, bool terminating)
-{
- // no need to perform log rotation
- return FinishedRotation();
-}
-
-/**
- * Triggered by regular heartbeat messages from the main thread.
- */
-bool KafkaWriter::DoHeartbeat(double network_time, double current_time)
-{
- producer->poll(0);
- return true;
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/src/KafkaWriter.h
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/src/KafkaWriter.h b/bro-plugin-kafka/src/KafkaWriter.h
deleted file mode 100644
index 2299667..0000000
--- a/bro-plugin-kafka/src/KafkaWriter.h
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef BRO_PLUGIN_BRO_KAFKA_KAFKAWRITER_H
-#define BRO_PLUGIN_BRO_KAFKA_KAFKAWRITER_H
-
-#include <string>
-#include <librdkafka/rdkafkacpp.h>
-#include <logging/WriterBackend.h>
-#include <threading/formatters/JSON.h>
-#include <Type.h>
-#include "kafka.bif.h"
-
-#include "TaggedJSON.h"
-
-namespace logging { namespace writer {
-
-/**
- * A logging writer that sends data to a Kafka broker.
- */
-class KafkaWriter : public WriterBackend {
-
-public:
- KafkaWriter(WriterFrontend* frontend);
- ~KafkaWriter();
-
- static WriterBackend* Instantiate(WriterFrontend* frontend)
- {
- return new KafkaWriter(frontend);
- }
-
-protected:
- virtual bool DoInit(const WriterBackend::WriterInfo& info, int num_fields, const threading::Field* const* fields);
- virtual bool DoWrite(int num_fields, const threading::Field* const* fields, threading::Value** vals);
- virtual bool DoSetBuf(bool enabled);
- virtual bool DoRotate(const char* rotated_path, double open, double close, bool terminating);
- virtual bool DoFlush(double network_time);
- virtual bool DoFinish(double network_time);
- virtual bool DoHeartbeat(double network_time, double current_time);
-
-private:
- string topic_name;
- threading::formatter::Formatter *formatter;
- RdKafka::Producer* producer;
- RdKafka::Topic* topic;
- RdKafka::Conf* conf;
- RdKafka::Conf* topic_conf;
-};
-
-}}
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/src/Plugin.cc
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/src/Plugin.cc b/bro-plugin-kafka/src/Plugin.cc
deleted file mode 100644
index d523d23..0000000
--- a/bro-plugin-kafka/src/Plugin.cc
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "Plugin.h"
-#include "KafkaWriter.h"
-
-namespace plugin { namespace Bro_Kafka {
- Plugin plugin;
-}}
-
-using namespace plugin::Bro_Kafka;
-
-plugin::Configuration Plugin::Configure()
-{
- AddComponent(new ::logging::Component("KafkaWriter", ::logging::writer::KafkaWriter::Instantiate));
-
- plugin::Configuration config;
- config.name = "Bro::Kafka";
- config.description = "Writes logs to Kafka";
- config.version.major = 0;
- config.version.minor = 1;
- return config;
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/src/Plugin.h
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/src/Plugin.h b/bro-plugin-kafka/src/Plugin.h
deleted file mode 100644
index 8adeb18..0000000
--- a/bro-plugin-kafka/src/Plugin.h
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef BRO_PLUGIN_BRO_KAFKA
-#define BRO_PLUGIN_BRO_KAFKA
-
-#include <plugin/Plugin.h>
-
-namespace plugin { namespace Bro_Kafka {
-
- class Plugin : public ::plugin::Plugin {
- protected:
- // Overridden from plugin::Plugin.
- virtual plugin::Configuration Configure();
- };
-
- extern Plugin plugin;
-}}
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/src/TaggedJSON.cc
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/src/TaggedJSON.cc b/bro-plugin-kafka/src/TaggedJSON.cc
deleted file mode 100644
index db3f305..0000000
--- a/bro-plugin-kafka/src/TaggedJSON.cc
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "TaggedJSON.h"
-
-namespace threading { namespace formatter {
-
-TaggedJSON::TaggedJSON(string sn, MsgThread* t, JSON::TimeFormat tf): JSON(t, tf), stream_name(sn)
-{}
-
-TaggedJSON::~TaggedJSON()
-{}
-
-bool TaggedJSON::Describe(ODesc* desc, int num_fields, const Field* const* fields, Value** vals) const
-{
- desc->AddRaw("{");
-
- // 'tag' the json; aka prepend the stream name to the json-formatted log content
- desc->AddRaw("\"");
- desc->AddRaw(stream_name);
- desc->AddRaw("\": ");
-
- // append the JSON formatted log record itself
- JSON::Describe(desc, num_fields, fields, vals);
-
- desc->AddRaw("}");
- return true;
-}
-}}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/src/TaggedJSON.h
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/src/TaggedJSON.h b/bro-plugin-kafka/src/TaggedJSON.h
deleted file mode 100644
index 08a50df..0000000
--- a/bro-plugin-kafka/src/TaggedJSON.h
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef BRO_PLUGIN_BRO_KAFKA_TAGGEDJSON_H
-#define BRO_PLUGIN_BRO_KAFKA_TAGGEDJSON_H
-
-#include <string>
-#include <threading/Formatter.h>
-#include <threading/formatters/JSON.h>
-
-using threading::formatter::JSON;
-using threading::MsgThread;
-using threading::Value;
-using threading::Field;
-
-namespace threading { namespace formatter {
-
-/*
- * A JSON formatter that prepends or 'tags' the content with a log stream
- * identifier. For example,
- * { 'conn' : { ... }}
- * { 'http' : { ... }}
- */
-class TaggedJSON : public JSON {
-
-public:
- TaggedJSON(string stream_name, MsgThread* t, JSON::TimeFormat tf);
- virtual ~TaggedJSON();
- virtual bool Describe(ODesc* desc, int num_fields, const Field* const* fields, Value** vals) const;
-
-private:
- string stream_name;
-};
-
-}}
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/src/kafka.bif
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/src/kafka.bif b/bro-plugin-kafka/src/kafka.bif
deleted file mode 100644
index 8a8070c..0000000
--- a/bro-plugin-kafka/src/kafka.bif
+++ /dev/null
@@ -1,23 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-module Kafka;
-
-const kafka_conf: config;
-const topic_name: string;
-const max_wait_on_shutdown: count;
-const tag_json: bool;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/src/kafka_const.bif
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/src/kafka_const.bif b/bro-plugin-kafka/src/kafka_const.bif
deleted file mode 100644
index 989c0ae..0000000
--- a/bro-plugin-kafka/src/kafka_const.bif
+++ /dev/null
@@ -1,20 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-module Kafka;
-
-type config : table[string] of string;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/tests/Makefile
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/tests/Makefile b/bro-plugin-kafka/tests/Makefile
deleted file mode 100644
index a637cd3..0000000
--- a/bro-plugin-kafka/tests/Makefile
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-test:
- @btest
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/tests/Scripts/get-bro-env
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/tests/Scripts/get-bro-env b/bro-plugin-kafka/tests/Scripts/get-bro-env
deleted file mode 100755
index 8aa0ea7..0000000
--- a/bro-plugin-kafka/tests/Scripts/get-bro-env
+++ /dev/null
@@ -1,36 +0,0 @@
-#! /bin/sh
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# BTest helper for getting values for Bro-related environment variables.
-
-base=`dirname $0`
-bro=`cat ${base}/../../build/CMakeCache.txt | grep BRO_DIST | cut -d = -f 2`
-
-if [ "$1" = "brobase" ]; then
- echo ${bro}
-elif [ "$1" = "bropath" ]; then
- ${bro}/build/bro-path-dev
-elif [ "$1" = "bro_plugin_path" ]; then
- ( cd ${base}/../.. && pwd )
-elif [ "$1" = "bro_seed_file" ]; then
- echo ${bro}/testing/btest/random.seed
-elif [ "$1" = "path" ]; then
- echo ${bro}/build/src:${bro}/aux/btest:${base}/:${bro}/aux/bro-cut:$PATH
-else
- echo "usage: `basename $0` <var>" >&2
- exit 1
-fi
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/tests/btest.cfg
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/tests/btest.cfg b/bro-plugin-kafka/tests/btest.cfg
deleted file mode 100644
index e42fefd..0000000
--- a/bro-plugin-kafka/tests/btest.cfg
+++ /dev/null
@@ -1,36 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-[btest]
-TestDirs = kafka
-TmpDir = %(testbase)s/.tmp
-BaselineDir = %(testbase)s/Baseline
-IgnoreDirs = .svn CVS .tmp
-IgnoreFiles = *.tmp *.swp #* *.trace .DS_Store
-
-[environment]
-BROBASE=`%(testbase)s/Scripts/get-bro-env brobase`
-BROPATH=`%(testbase)s/Scripts/get-bro-env bropath`
-BRO_PLUGIN_PATH=`%(testbase)s/Scripts/get-bro-env bro_plugin_path`
-BRO_SEED_FILE=`%(testbase)s/Scripts/get-bro-env bro_seed_file`
-PATH=`%(testbase)s/Scripts/get-bro-env path`
-TZ=UTC
-LC_ALL=C
-TRACES=%(testbase)s/Traces
-TMPDIR=%(testbase)s/.tmp
-BRO_TRACES=`%(testbase)s/Scripts/get-bro-env brobase`/testing/btest/Traces
-TEST_DIFF_CANONIFIER=`%(testbase)s/Scripts/get-bro-env brobase`/testing/scripts/diff-canonifier
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/tests/kafka/show-plugin.bro
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/tests/kafka/show-plugin.bro b/bro-plugin-kafka/tests/kafka/show-plugin.bro
deleted file mode 100644
index 4e8dd6a..0000000
--- a/bro-plugin-kafka/tests/kafka/show-plugin.bro
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# @TEST-EXEC: bro -NN Bro::Kafka >output
-# @TEST-EXEC: btest-diff output
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/bro/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/bro/meta/main.yml b/deployment/roles/bro/meta/main.yml
index 320ce8d..9c9286f 100644
--- a/deployment/roles/bro/meta/main.yml
+++ b/deployment/roles/bro/meta/main.yml
@@ -16,8 +16,8 @@
#
---
dependencies:
- - ambari_gather_facts
- libselinux-python
+ - ambari_gather_facts
- build-tools
- kafka-client
-
+ - librdkafka
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/bro/tasks/bro-plugin-kafka.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/bro/tasks/bro-plugin-kafka.yml b/deployment/roles/bro/tasks/bro-plugin-kafka.yml
index d920688..d8e887d 100644
--- a/deployment/roles/bro/tasks/bro-plugin-kafka.yml
+++ b/deployment/roles/bro/tasks/bro-plugin-kafka.yml
@@ -16,7 +16,7 @@
#
---
- name: Distribute bro-kafka plugin
- copy: src=../../../bro-plugin-kafka dest=/tmp/ mode=0755
+ copy: src=../../../metron-sensors/bro-plugin-kafka dest=/tmp/ mode=0755
- name: Compile and install the plugin
shell: "{{ item }}"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/bro/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/bro/vars/main.yml b/deployment/roles/bro/vars/main.yml
index 1e1d1ab..2ff5177 100644
--- a/deployment/roles/bro/vars/main.yml
+++ b/deployment/roles/bro/vars/main.yml
@@ -24,8 +24,3 @@ bro_topic: bro
bro_crontab_minutes: 0-59/5
bro_crontab_job: "{{ bro_home }}/bin/broctl cron"
bro_clean_job: "rm -rf {{ bro_home }}/spool/tmp/*"
-
-# librdkafka
-librdkafka_version: 0.8.6
-librdkafka_url: https://github.com/edenhill/librdkafka/archive/0.8.6.tar.gz
-librdkafka_home: /usr/local
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/kafka-broker/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/kafka-broker/defaults/main.yml b/deployment/roles/kafka-broker/defaults/main.yml
new file mode 100644
index 0000000..351d125
--- /dev/null
+++ b/deployment/roles/kafka-broker/defaults/main.yml
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+hdp_repo_def: http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.2.0/hdp.repo
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/kafka-broker/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/kafka-broker/meta/main.yml b/deployment/roles/kafka-broker/meta/main.yml
new file mode 100644
index 0000000..9587e79
--- /dev/null
+++ b/deployment/roles/kafka-broker/meta/main.yml
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+dependencies:
+ - libselinux-python
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/kafka-broker/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/kafka-broker/tasks/main.yml b/deployment/roles/kafka-broker/tasks/main.yml
new file mode 100644
index 0000000..db05cb0
--- /dev/null
+++ b/deployment/roles/kafka-broker/tasks/main.yml
@@ -0,0 +1,41 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Retrieve HDP repository definition
+ get_url:
+ url: "{{ hdp_repo_def }}"
+ dest: /etc/yum.repos.d/hdp.repo
+ mode: 0644
+
+- name: Install kafka
+ yum: name={{item}}
+ with_items:
+ - java-1.8.0-openjdk
+ - kafka
+ - zookeeper-server
+
+- name: Create pid directories
+ file: path={{ item }} state=directory mode=0755
+ with_items:
+ - /var/run/zookeeper
+ - /var/run/kafka
+
+- name: Start zookeeper
+ shell: /usr/hdp/current/zookeeper-server/bin/zookeeper-server start
+
+- name: Start kafka
+ shell: /usr/hdp/current/kafka-broker/bin/kafka start
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/kafka-broker/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/kafka-broker/vars/main.yml b/deployment/roles/kafka-broker/vars/main.yml
new file mode 100644
index 0000000..351d125
--- /dev/null
+++ b/deployment/roles/kafka-broker/vars/main.yml
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+hdp_repo_def: http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.2.0/hdp.repo
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/librdkafka/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/librdkafka/defaults/main.yml b/deployment/roles/librdkafka/defaults/main.yml
new file mode 100644
index 0000000..d920883
--- /dev/null
+++ b/deployment/roles/librdkafka/defaults/main.yml
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+librdkafka_version: 0.8.6
+librdkafka_url: https://github.com/edenhill/librdkafka/archive/0.8.6.tar.gz
+librdkafka_home: /usr/local
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/librdkafka/tasks/dependencies.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/librdkafka/tasks/dependencies.yml b/deployment/roles/librdkafka/tasks/dependencies.yml
new file mode 100644
index 0000000..431e861
--- /dev/null
+++ b/deployment/roles/librdkafka/tasks/dependencies.yml
@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Install prerequisites
+ yum: name={{ item }}
+ with_items:
+ - cmake
+ - make
+ - gcc
+ - gcc-c++
+ - flex
+ - bison
+ - libpcap
+ - libpcap-devel
+ - openssl-devel
+ - python-devel
+ - swig
+ - zlib-devel
+ - perl
+ register: result
+ until: result.rc == 0
+ retries: 5
+ delay: 10
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/librdkafka/tasks/librdkafka.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/librdkafka/tasks/librdkafka.yml b/deployment/roles/librdkafka/tasks/librdkafka.yml
new file mode 100644
index 0000000..652d319
--- /dev/null
+++ b/deployment/roles/librdkafka/tasks/librdkafka.yml
@@ -0,0 +1,39 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Download librdkafka
+ get_url:
+ url: "{{ librdkafka_url }}"
+ dest: "/tmp/librdkafka-{{ librdkafka_version }}.tar.gz"
+
+- name: Extract librdkafka tarball
+ unarchive:
+ src: "/tmp/librdkafka-{{ librdkafka_version }}.tar.gz"
+ dest: /tmp
+ copy: no
+ creates: "/tmp/librdkafka-{{ librdkafka_version }}"
+
+- name: Compile and install librdkafka
+ shell: "{{ item }}"
+ args:
+ chdir: "/tmp/librdkafka-{{ librdkafka_version }}"
+ creates: "{{ librdkafka_home }}/lib/librdkafka.so"
+ with_items:
+ - rm -rf build/
+ - "./configure --prefix={{ librdkafka_home }}"
+ - make
+ - make install
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/librdkafka/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/librdkafka/tasks/main.yml b/deployment/roles/librdkafka/tasks/main.yml
new file mode 100644
index 0000000..2144d7f
--- /dev/null
+++ b/deployment/roles/librdkafka/tasks/main.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- include: dependencies.yml
+- include: librdkafka.yml
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/packet-capture/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/packet-capture/defaults/main.yml b/deployment/roles/packet-capture/defaults/main.yml
new file mode 100644
index 0000000..3e6358c
--- /dev/null
+++ b/deployment/roles/packet-capture/defaults/main.yml
@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+# dpdk
+dpdk_home: "/usr/local/dpdk"
+dpdk_version: "2.2.0"
+dpdk_sdk: "/root/dpdk-{{ dpdk_version }}"
+dpdk_target: "x86_64-native-linuxapp-gcc"
+num_huge_pages: 512
+extra_cflags: -g
+
+# pcapture
+pcapture_work_dir: /root/packet-capture
+pcapture_prefix: /usr/local/bin
+pcapture_ld_library_path: /usr/local/lib
+pcapture_portmask: 0x01
+pcapture_kafka_config: /etc/pcapture.conf
+pcapture_bin: pcapture
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/packet-capture/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/packet-capture/meta/main.yml b/deployment/roles/packet-capture/meta/main.yml
new file mode 100644
index 0000000..d253e88
--- /dev/null
+++ b/deployment/roles/packet-capture/meta/main.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+dependencies:
+ - librdkafka
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/packet-capture/tasks/debug.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/packet-capture/tasks/debug.yml b/deployment/roles/packet-capture/tasks/debug.yml
new file mode 100644
index 0000000..06f1526
--- /dev/null
+++ b/deployment/roles/packet-capture/tasks/debug.yml
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+ - name: Install debug utilities
+ yum: name=yum-utils
+ tags:
+ - debug
+
+ - name: Install debug symbols
+ shell: debuginfo-install -y glibc glib2 zlib
+ tags:
+ - debug