You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ni...@apache.org on 2016/04/13 16:48:29 UTC
[3/3] incubator-metron git commit: METRON-108 Create Fast Packet
Capture Process (nickwallen) closes apache/incubator-metron#73
METRON-108 Create Fast Packet Capture Process (nickwallen) closes apache/incubator-metron#73
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/564272e6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/564272e6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/564272e6
Branch: refs/heads/master
Commit: 564272e6ad086bd290836184c5691dbceaeab5cf
Parents: 8d5fb1b
Author: nickwallen <ni...@nickallen.org>
Authored: Wed Apr 13 10:47:54 2016 -0400
Committer: Nick Allen <ni...@nickallen.org>
Committed: Wed Apr 13 10:47:54 2016 -0400
----------------------------------------------------------------------
bro-plugin-kafka/.gitignore | 31 --
bro-plugin-kafka/CHANGES | 16 -
bro-plugin-kafka/CMakeLists.txt | 43 ---
bro-plugin-kafka/COPYING | 201 -----------
bro-plugin-kafka/MAINTAINER | 18 -
bro-plugin-kafka/Makefile | 44 ---
bro-plugin-kafka/README | 96 -----
bro-plugin-kafka/VERSION | 18 -
bro-plugin-kafka/cmake/FindLibRDKafka.cmake | 49 ---
bro-plugin-kafka/cmake/FindOpenSSL.cmake | 72 ----
bro-plugin-kafka/configure | 130 -------
bro-plugin-kafka/configure.plugin | 43 ---
bro-plugin-kafka/scripts/Bro/Kafka/__load__.bro | 19 -
.../scripts/Bro/Kafka/logs-to-kafka.bro | 44 ---
bro-plugin-kafka/scripts/__load__.bro | 25 --
bro-plugin-kafka/scripts/init.bro | 27 --
bro-plugin-kafka/src/KafkaWriter.cc | 200 -----------
bro-plugin-kafka/src/KafkaWriter.h | 66 ----
bro-plugin-kafka/src/Plugin.cc | 37 --
bro-plugin-kafka/src/Plugin.h | 34 --
bro-plugin-kafka/src/TaggedJSON.cc | 43 ---
bro-plugin-kafka/src/TaggedJSON.h | 50 ---
bro-plugin-kafka/src/kafka.bif | 23 --
bro-plugin-kafka/src/kafka_const.bif | 20 --
bro-plugin-kafka/tests/Makefile | 19 -
bro-plugin-kafka/tests/Scripts/get-bro-env | 36 --
bro-plugin-kafka/tests/btest.cfg | 36 --
bro-plugin-kafka/tests/kafka/show-plugin.bro | 19 -
deployment/roles/bro/meta/main.yml | 4 +-
deployment/roles/bro/tasks/bro-plugin-kafka.yml | 2 +-
deployment/roles/bro/vars/main.yml | 5 -
deployment/roles/kafka-broker/defaults/main.yml | 18 +
deployment/roles/kafka-broker/meta/main.yml | 18 +
deployment/roles/kafka-broker/tasks/main.yml | 41 +++
deployment/roles/kafka-broker/vars/main.yml | 18 +
deployment/roles/librdkafka/defaults/main.yml | 20 ++
.../roles/librdkafka/tasks/dependencies.yml | 37 ++
.../roles/librdkafka/tasks/librdkafka.yml | 39 +++
deployment/roles/librdkafka/tasks/main.yml | 19 +
.../roles/packet-capture/defaults/main.yml | 32 ++
deployment/roles/packet-capture/meta/main.yml | 19 +
deployment/roles/packet-capture/tasks/debug.yml | 26 ++
.../roles/packet-capture/tasks/dependencies.yml | 38 ++
deployment/roles/packet-capture/tasks/dpdk.yml | 59 ++++
.../roles/packet-capture/tasks/kernel.yml | 51 +++
deployment/roles/packet-capture/tasks/main.yml | 22 ++
.../roles/packet-capture/tasks/pcapture.yml | 49 +++
.../roles/packet-capture/templates/pcapture | 93 +++++
.../packet-capture/templates/pcapture.conf | 67 ++++
deployment/vagrant/packet-capture/Vagrantfile | 69 ++++
deployment/vagrant/packet-capture/ansible.cfg | 22 ++
deployment/vagrant/packet-capture/playbook.yml | 43 +++
metron-sensors/bro-plugin-kafka/.gitignore | 31 ++
metron-sensors/bro-plugin-kafka/CHANGES | 16 +
metron-sensors/bro-plugin-kafka/CMakeLists.txt | 43 +++
metron-sensors/bro-plugin-kafka/COPYING | 201 +++++++++++
metron-sensors/bro-plugin-kafka/MAINTAINER | 18 +
metron-sensors/bro-plugin-kafka/Makefile | 44 +++
metron-sensors/bro-plugin-kafka/README | 92 +++++
metron-sensors/bro-plugin-kafka/VERSION | 18 +
.../bro-plugin-kafka/cmake/FindLibRDKafka.cmake | 49 +++
.../bro-plugin-kafka/cmake/FindOpenSSL.cmake | 72 ++++
metron-sensors/bro-plugin-kafka/configure | 130 +++++++
.../bro-plugin-kafka/configure.plugin | 43 +++
.../scripts/Bro/Kafka/__load__.bro | 19 +
.../scripts/Bro/Kafka/logs-to-kafka.bro | 44 +++
.../bro-plugin-kafka/scripts/__load__.bro | 25 ++
.../bro-plugin-kafka/scripts/init.bro | 27 ++
.../bro-plugin-kafka/src/KafkaWriter.cc | 200 +++++++++++
.../bro-plugin-kafka/src/KafkaWriter.h | 66 ++++
metron-sensors/bro-plugin-kafka/src/Plugin.cc | 37 ++
metron-sensors/bro-plugin-kafka/src/Plugin.h | 34 ++
.../bro-plugin-kafka/src/TaggedJSON.cc | 43 +++
.../bro-plugin-kafka/src/TaggedJSON.h | 50 +++
metron-sensors/bro-plugin-kafka/src/kafka.bif | 23 ++
.../bro-plugin-kafka/src/kafka_const.bif | 20 ++
metron-sensors/bro-plugin-kafka/tests/Makefile | 19 +
.../bro-plugin-kafka/tests/Scripts/get-bro-env | 36 ++
metron-sensors/bro-plugin-kafka/tests/btest.cfg | 36 ++
.../tests/kafka/show-plugin.bro | 19 +
metron-sensors/packet-capture/.gitignore | 3 +
metron-sensors/packet-capture/Makefile | 26 ++
metron-sensors/packet-capture/README.md | 12 +
.../packet-capture/conf/localhost.kafka | 67 ++++
metron-sensors/packet-capture/src/.gitignore | 1 +
metron-sensors/packet-capture/src/Makefile | 52 +++
metron-sensors/packet-capture/src/args.c | 145 ++++++++
metron-sensors/packet-capture/src/args.h | 107 ++++++
metron-sensors/packet-capture/src/kafka.c | 224 ++++++++++++
metron-sensors/packet-capture/src/kafka.h | 45 +++
metron-sensors/packet-capture/src/main.c | 349 +++++++++++++++++++
metron-sensors/packet-capture/src/main.h | 122 +++++++
92 files changed, 3411 insertions(+), 1467 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/.gitignore
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/.gitignore b/bro-plugin-kafka/.gitignore
deleted file mode 100644
index 28a8358..0000000
--- a/bro-plugin-kafka/.gitignore
+++ /dev/null
@@ -1,31 +0,0 @@
-.state
-build
-
-# Compiled Object files
-*.slo
-*.lo
-*.o
-*.obj
-
-# Precompiled Headers
-*.gch
-*.pch
-
-# Compiled Dynamic libraries
-*.so
-*.dylib
-*.dll
-
-# Fortran module files
-*.mod
-
-# Compiled Static libraries
-*.lai
-*.la
-*.a
-*.lib
-
-# Executables
-*.exe
-*.out
-*.app
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/CHANGES
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/CHANGES b/bro-plugin-kafka/CHANGES
deleted file mode 100644
index d9e26de..0000000
--- a/bro-plugin-kafka/CHANGES
+++ /dev/null
@@ -1,16 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/CMakeLists.txt b/bro-plugin-kafka/CMakeLists.txt
deleted file mode 100644
index 30bf3b5..0000000
--- a/bro-plugin-kafka/CMakeLists.txt
+++ /dev/null
@@ -1,43 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-cmake_minimum_required(VERSION 2.8)
-project(Plugin)
-include(BroPlugin)
-find_package(LibRDKafka)
-find_package(OpenSSL)
-
-if (LIBRDKAFKA_FOUND AND OPENSSL_FOUND)
- include_directories(BEFORE ${LibRDKafka_INCLUDE_DIR} ${OpenSSL_INCLUDE_DIR})
- bro_plugin_begin(BRO KAFKA)
- bro_plugin_cc(src/KafkaWriter.cc)
- bro_plugin_cc(src/Plugin.cc)
- bro_plugin_cc(src/TaggedJSON.cc)
- bro_plugin_bif(src/kafka.bif)
- bro_plugin_dist_files(README CHANGES COPYING VERSION)
- bro_plugin_link_library(${LibRDKafka_LIBRARIES})
- bro_plugin_link_library(${LibRDKafka_C_LIBRARIES})
- bro_plugin_link_library(${OpenSSL_LIBRARIES})
- bro_plugin_end()
-
-elseif (NOT LIBRDKAFKA_FOUND)
- message(FATAL_ERROR "LibRDKafka not found.")
-
-elseif (NOT OPENSSL_FOUND)
- message(FATAL_ERROR "OpenSSL not found.")
-
-endif ()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/COPYING
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/COPYING b/bro-plugin-kafka/COPYING
deleted file mode 100644
index 5c304d1..0000000
--- a/bro-plugin-kafka/COPYING
+++ /dev/null
@@ -1,201 +0,0 @@
-Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "{}"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright {yyyy} {name of copyright owner}
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/MAINTAINER
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/MAINTAINER b/bro-plugin-kafka/MAINTAINER
deleted file mode 100644
index 0ddaed7..0000000
--- a/bro-plugin-kafka/MAINTAINER
+++ /dev/null
@@ -1,18 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-Apache Metron <us...@metron.incubator.apache.org>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/Makefile
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/Makefile b/bro-plugin-kafka/Makefile
deleted file mode 100644
index 50fa3ca..0000000
--- a/bro-plugin-kafka/Makefile
+++ /dev/null
@@ -1,44 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Convenience Makefile providing a few common top-level targets.
-#
-
-cmake_build_dir=build
-arch=`uname -s | tr A-Z a-z`-`uname -m`
-
-all: build-it
-
-build-it:
- @test -e $(cmake_build_dir)/config.status || ./configure
- -@test -e $(cmake_build_dir)/CMakeCache.txt && \
- test $(cmake_build_dir)/CMakeCache.txt -ot `cat $(cmake_build_dir)/CMakeCache.txt | grep BRO_DIST | cut -d '=' -f 2`/build/CMakeCache.txt && \
- echo Updating stale CMake cache && \
- touch $(cmake_build_dir)/CMakeCache.txt
-
- ( cd $(cmake_build_dir) && make )
-
-install:
- ( cd $(cmake_build_dir) && make install )
-
-clean:
- ( cd $(cmake_build_dir) && make clean )
-
-distclean:
- rm -rf $(cmake_build_dir)
-
-test:
- make -C tests
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/README
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/README b/bro-plugin-kafka/README
deleted file mode 100644
index c672bd0..0000000
--- a/bro-plugin-kafka/README
+++ /dev/null
@@ -1,96 +0,0 @@
-
-===============================
-Writing Logging Output to Kafka
-===============================
-
-A log writer that sends logging output to Kafka. This provides a convenient
-means for tools in the Hadoop ecosystem, such as Storm, Spark, and others, to
-process the data generated by Bro.
-
-.. contents::
-
-Installation
-------------
-
-Install librdkafka (https://github.com/edenhill/librdkafka), a native client
-library for Kafka. This plugin has been tested against the latest release of
-librdkafka, which at the time of this writing is v0.8.6.
-
- # curl -L https://github.com/edenhill/librdkafka/archive/0.8.6.tar.gz | tar xvz
- # cd librdkafka-0.8.6/
- # ./configure
- # make
- # sudo make install
-
-Then compile this Bro plugin using the following commands.
-
- # ./configure --bro-dist=$BRO_SRC
- # make
- # sudo make install
-
-Run the following command to ensure that the plugin was installed successfully.
-
- # bro -N Bro::Kafka
- Bro::Kafka - Writes logs to Kafka (dynamic, version 0.1)
-
-Activation
-----------
-
-The easiest way to enable Kafka output is to load the plugin's
-``logs-to-kafka.bro`` script. If you are using BroControl, the following lines
-added to local.bro will activate it.
-
-.. console::
-
- @load Bro/Kafka/logs-to-kafka.bro
- redef Kafka::logs_to_send = set(Conn::LOG, HTTP::LOG, DNS::LOG);
- redef Kafka::topic_name = "bro";
- redef Kafka::kafka_conf = table(
- ["metadata.broker.list"] = "localhost:9092"
- );
-
-This example will send all HTTP, DNS, and Conn logs to a Kafka broker running on
-the localhost to a topic called ``bro``. Any configuration value accepted by
-librdkafka can be added to the ``kafka_conf`` configuration table.
-
-Settings
---------
-
-``kafka_conf``
-
-The global configuration settings for Kafka. These values are passed through
-directly to librdkafka. Any valid librdkafka settings can be defined in this
-table.
-
-.. console::
-
- redef Kafka::kafka_conf = table(
- ["metadata.broker.list"] = "localhost:9092",
- ["client.id"] = "bro"
- );
-
-``topic_name``
-
-The name of the topic in Kafka where all Bro logs will be sent to.
-
-.. console::
-
- redef Kafka::topic_name = "bro";
-
-``max_wait_on_shutdown``
-
-The maximum number of milliseconds that the plugin will wait for any backlog of
-queued messages to be sent to Kafka before forced shutdown.
-
-.. console::
-
- redef Kafka::max_wait_on_shutdown = 3000;
-
-``tag_json``
-
-If true, a log stream identifier is appended to each JSON-formatted message. For
-example, a Conn::LOG message will look like ``{ 'conn' : { ... }}``.
-
-.. console::
-
- redef Kafka::tag_json = T;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/VERSION
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/VERSION b/bro-plugin-kafka/VERSION
deleted file mode 100644
index 204e5ca..0000000
--- a/bro-plugin-kafka/VERSION
+++ /dev/null
@@ -1,18 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-0.1
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/cmake/FindLibRDKafka.cmake
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/cmake/FindLibRDKafka.cmake b/bro-plugin-kafka/cmake/FindLibRDKafka.cmake
deleted file mode 100644
index c64d8f9..0000000
--- a/bro-plugin-kafka/cmake/FindLibRDKafka.cmake
+++ /dev/null
@@ -1,49 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-find_path(LibRDKafka_ROOT_DIR
- NAMES include/librdkafka/rdkafkacpp.h
-)
-
-find_library(LibRDKafka_LIBRARIES
- NAMES rdkafka++
- HINTS ${LibRDKafka_ROOT_DIR}/lib
-)
-
-find_library(LibRDKafka_C_LIBRARIES
- NAMES rdkafka
- HINTS ${LibRDKafka_ROT_DIR}/lib
-)
-
-find_path(LibRDKafka_INCLUDE_DIR
- NAMES librdkafka/rdkafkacpp.h
- HINTS ${LibRDKafka_ROOT_DIR}/include
-)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(LibRDKafka DEFAULT_MSG
- LibRDKafka_LIBRARIES
- LibRDKafka_C_LIBRARIES
- LibRDKafka_INCLUDE_DIR
-)
-
-mark_as_advanced(
- LibRDKafka_ROOT_DIR
- LibRDKafka_LIBRARIES
- LibRDKafka_C_LIBRARIES
- LibRDKafka_INCLUDE_DIR
-)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/cmake/FindOpenSSL.cmake
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/cmake/FindOpenSSL.cmake b/bro-plugin-kafka/cmake/FindOpenSSL.cmake
deleted file mode 100644
index 5ed955c..0000000
--- a/bro-plugin-kafka/cmake/FindOpenSSL.cmake
+++ /dev/null
@@ -1,72 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# - Try to find openssl include dirs and libraries
-#
-# Usage of this module as follows:
-#
-# find_package(OpenSSL)
-#
-# Variables used by this module, they can change the default behaviour and need
-# to be set before calling find_package:
-#
-# OpenSSL_ROOT_DIR Set this variable to the root installation of
-# openssl if the module has problems finding the
-# proper installation path.
-#
-# Variables defined by this module:
-#
-# OPENSSL_FOUND System has openssl, include and library dirs found
-# OpenSSL_INCLUDE_DIR The openssl include directories.
-# OpenSSL_LIBRARIES The openssl libraries.
-# OpenSSL_CYRPTO_LIBRARY The openssl crypto library.
-# OpenSSL_SSL_LIBRARY The openssl ssl library.
-
-find_path(OpenSSL_ROOT_DIR
- NAMES include/openssl/ssl.h
-)
-
-find_path(OpenSSL_INCLUDE_DIR
- NAMES openssl/ssl.h
- HINTS ${OpenSSL_ROOT_DIR}/include
-)
-
-find_library(OpenSSL_SSL_LIBRARY
- NAMES ssl ssleay32 ssleay32MD
- HINTS ${OpenSSL_ROOT_DIR}/lib
-)
-
-find_library(OpenSSL_CRYPTO_LIBRARY
- NAMES crypto
- HINTS ${OpenSSL_ROOT_DIR}/lib
-)
-
-set(OpenSSL_LIBRARIES ${OpenSSL_SSL_LIBRARY} ${OpenSSL_CRYPTO_LIBRARY}
- CACHE STRING "OpenSSL SSL and crypto libraries" FORCE)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(OpenSSL DEFAULT_MSG
- OpenSSL_LIBRARIES
- OpenSSL_INCLUDE_DIR
-)
-
-mark_as_advanced(
- OpenSSL_ROOT_DIR
- OpenSSL_INCLUDE_DIR
- OpenSSL_LIBRARIES
- OpenSSL_CRYPTO_LIBRARY
- OpenSSL_SSL_LIBRARY
-)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/configure
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/configure b/bro-plugin-kafka/configure
deleted file mode 100755
index d053488..0000000
--- a/bro-plugin-kafka/configure
+++ /dev/null
@@ -1,130 +0,0 @@
-#!/bin/sh
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Wrapper for viewing/setting options that the plugin's CMake
-# scripts will recognize.
-#
-# Don't edit this. Edit configure.plugin to add plugin-specific options.
-#
-
-set -e
-command="$0 $*"
-
-if [ -e `dirname $0`/configure.plugin ]; then
- # Include custom additions.
- . `dirname $0`/configure.plugin
-fi
-
-# Check for `cmake` command.
-type cmake > /dev/null 2>&1 || {
- echo "\
-This package requires CMake, please install it first, then you may
-use this configure script to access CMake equivalent functionality.\
-" >&2;
- exit 1;
-}
-
-usage() {
-
-cat 1>&2 <<EOF
-Usage: $0 [OPTIONS]
-
- Plugin Options:
- --bro-dist=DIR Path to Bro source tree
- --install-root=DIR Path where to install plugin into
-EOF
-
-if type plugin_usage >/dev/null 2>&1; then
- plugin_usage 1>&2
-fi
-
-echo
-
-exit 1
-}
-
-# Function to append a CMake cache entry definition to the
-# CMakeCacheEntries variable
-# $1 is the cache entry variable name
-# $2 is the cache entry variable type
-# $3 is the cache entry variable value
-append_cache_entry () {
- CMakeCacheEntries="$CMakeCacheEntries -D $1:$2=$3"
-}
-
-# set defaults
-builddir=build
-brodist=`cd ../../.. && pwd`
-installroot="default"
-CMakeCacheEntries=""
-
-while [ $# -ne 0 ]; do
- case "$1" in
- -*=*) optarg=`echo "$1" | sed 's/[-_a-zA-Z0-9]*=//'` ;;
- *) optarg= ;;
- esac
-
- case "$1" in
- --help|-h)
- usage
- ;;
- --bro-dist=*)
- brodist=`cd $optarg && pwd`
- ;;
- --install-root=*)
- installroot=$optarg
- ;;
- --with-openssl=*)
- append_cache_entry OpenSSL_ROOT_DIR PATH $optarg
- ;;
- *)
- if type plugin_option >/dev/null 2>&1; then
- plugin_option $1 && shift && continue;
- fi
-
- echo "Invalid option '$1'. Try $0 --help to see available options."
- exit 1
- ;;
- esac
- shift
-done
-
-if [ ! -e "$brodist/bro-path-dev.in" ]; then
- echo "Cannot determine Bro source directory, use --bro-dist=DIR."
- exit 1
-fi
-
-append_cache_entry BRO_DIST PATH $brodist
-append_cache_entry CMAKE_MODULE_PATH PATH $brodist/cmake
-
-if [ "$installroot" != "default" ]; then
- mkdir -p $installroot
- append_cache_entry BRO_PLUGIN_INSTALL_ROOT PATH $installroot
-fi
-
-echo "Build Directory : $builddir"
-echo "Bro Source Directory : $brodist"
-
-mkdir -p $builddir
-cd $builddir
-
-cmake $CMakeCacheEntries ..
-
-echo "# This is the command used to configure this build" > config.status
-echo $command >> config.status
-chmod u+x config.status
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/configure.plugin
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/configure.plugin b/bro-plugin-kafka/configure.plugin
deleted file mode 100644
index 1cb2086..0000000
--- a/bro-plugin-kafka/configure.plugin
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/bin/sh
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Hooks to add custom options to the configure script.
-#
-
-plugin_usage()
-{
- cat <<EOF
- --with-librdkafka=PATH path to librdkafka
- --with-openssl=PATH path to OpenSSL install root
-EOF
-}
-
-plugin_option()
-{
- case "$1" in
- --with-librdkafka=*)
- append_cache_entry LibRdKafka_ROOT_DIR PATH $optarg
- ;;
- --with-openssl=*)
- append_cache_entry OpenSSL_ROOT_DIR PATH $optarg
- ;;
- *)
- return 1;
- ;;
- esac
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/scripts/Bro/Kafka/__load__.bro
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/scripts/Bro/Kafka/__load__.bro b/bro-plugin-kafka/scripts/Bro/Kafka/__load__.bro
deleted file mode 100644
index 12295a9..0000000
--- a/bro-plugin-kafka/scripts/Bro/Kafka/__load__.bro
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# This is loaded when a user activates the plugin. Include scripts here that should be
-# loaded automatically at that point.
-#
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/scripts/Bro/Kafka/logs-to-kafka.bro
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/scripts/Bro/Kafka/logs-to-kafka.bro b/bro-plugin-kafka/scripts/Bro/Kafka/logs-to-kafka.bro
deleted file mode 100644
index 84e390c..0000000
--- a/bro-plugin-kafka/scripts/Bro/Kafka/logs-to-kafka.bro
+++ /dev/null
@@ -1,44 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-##! load this script to enable log output to kafka
-
-module Kafka;
-
-export {
- ##
- ## which log streams should be sent to kafka?
- ## example:
- ## redef Kafka::logs_to_send = set(Conn::Log, HTTP::LOG, DNS::LOG);
- ##
- const logs_to_send: set[Log::ID] &redef;
-}
-
-event bro_init() &priority=-5
-{
- for (stream_id in Log::active_streams)
- {
- if (stream_id in Kafka::logs_to_send)
- {
- local filter: Log::Filter = [
- $name = fmt("kafka-%s", stream_id),
- $writer = Log::WRITER_KAFKAWRITER
- ];
-
- Log::add_filter(stream_id, filter);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/scripts/__load__.bro
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/scripts/__load__.bro b/bro-plugin-kafka/scripts/__load__.bro
deleted file mode 100644
index fee9549..0000000
--- a/bro-plugin-kafka/scripts/__load__.bro
+++ /dev/null
@@ -1,25 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# This is loaded unconditionally at Bro startup. Include scripts here that should
-# always be loaded.
-#
-# Normally, that will be only code that initializes built-in elements. Load
-# your standard scripts in
-# scripts/<plugin-namespace>/<plugin-name>/__load__.bro instead.
-#
-
-@load ./init.bro
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/scripts/init.bro
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/scripts/init.bro b/bro-plugin-kafka/scripts/init.bro
deleted file mode 100644
index c76b2a6..0000000
--- a/bro-plugin-kafka/scripts/init.bro
+++ /dev/null
@@ -1,27 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-module Kafka;
-
-export {
- const topic_name: string = "bro" &redef;
- const max_wait_on_shutdown: count = 3000 &redef;
- const tag_json: bool = F &redef;
- const kafka_conf: table[string] of string = table(
- ["metadata.broker.list"] = "localhost:9092"
- ) &redef;
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/src/KafkaWriter.cc
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/src/KafkaWriter.cc b/bro-plugin-kafka/src/KafkaWriter.cc
deleted file mode 100644
index 9019790..0000000
--- a/bro-plugin-kafka/src/KafkaWriter.cc
+++ /dev/null
@@ -1,200 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <Type.h>
-#include <threading/Formatter.h>
-#include <threading/formatters/JSON.h>
-#include "kafka.bif.h"
-#include "TaggedJSON.h"
-#include "KafkaWriter.h"
-
-using namespace logging;
-using namespace writer;
-
-KafkaWriter::KafkaWriter(WriterFrontend* frontend): WriterBackend(frontend), formatter(NULL), producer(NULL), topic(NULL)
-{
- // TODO do we need this??
- topic_name.assign((const char*)BifConst::Kafka::topic_name->Bytes(),
- BifConst::Kafka::topic_name->Len());
-}
-
-KafkaWriter::~KafkaWriter()
-{}
-
-bool KafkaWriter::DoInit(const WriterInfo& info, int num_fields, const threading::Field* const* fields)
-{
- // initialize the formatter
- if(BifConst::Kafka::tag_json) {
- formatter = new threading::formatter::TaggedJSON(info.path, this, threading::formatter::JSON::TS_EPOCH);
- } else {
- formatter = new threading::formatter::JSON(this, threading::formatter::JSON::TS_EPOCH);
- }
-
- // kafka global configuration
- string err;
- conf = RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL);
-
- // apply the user-defined settings to kafka
- Val* val = BifConst::Kafka::kafka_conf->AsTableVal();
- IterCookie* c = val->AsTable()->InitForIteration();
- HashKey* k;
- TableEntryVal* v;
- while ((v = val->AsTable()->NextEntry(k, c))) {
-
- // fetch the key and value
- ListVal* index = val->AsTableVal()->RecoverIndex(k);
- string key = index->Index(0)->AsString()->CheckString();
- string val = v->Value()->AsString()->CheckString();
-
- // apply setting to kafka
- if (RdKafka::Conf::CONF_OK != conf->set(key, val, err)) {
- reporter->Error("Failed to set '%s'='%s': %s", key.c_str(), val.c_str(), err.c_str());
- return false;
- }
-
- // cleanup
- Unref(index);
- delete k;
- }
-
- // create kafka producer
- producer = RdKafka::Producer::create(conf, err);
- if (!producer) {
- reporter->Error("Failed to create producer: %s", err.c_str());
- return false;
- }
-
- // create handle to topic
- topic_conf = RdKafka::Conf::create(RdKafka::Conf::CONF_TOPIC);
- topic = RdKafka::Topic::create(producer, topic_name, topic_conf, err);
- if (!topic) {
- reporter->Error("Failed to create topic handle: %s", err.c_str());
- return false;
- }
-
- return true;
-}
-
-/**
- * Writer-specific method called just before the threading system is
- * going to shutdown. It is assumed that once this messages returns,
- * the thread can be safely terminated.
- */
-bool KafkaWriter::DoFinish(double network_time)
-{
- bool success = false;
- int poll_interval = 1000;
- int waited = 0;
- int max_wait = BifConst::Kafka::max_wait_on_shutdown;
-
- // wait a bit for queued messages to be delivered
- while (producer->outq_len() > 0 && waited <= max_wait) {
- producer->poll(poll_interval);
- waited += poll_interval;
- }
-
- // successful only if all messages delivered
- if (producer->outq_len() == 0) {
- reporter->Error("Unable to deliver %0d message(s)", producer->outq_len());
- success = true;
- }
-
- delete topic;
- delete producer;
- delete formatter;
-
- return success;
-}
-
-/**
- * Writer-specific output method implementing recording of one log
- * entry.
- */
-bool KafkaWriter::DoWrite(int num_fields, const threading::Field* const* fields, threading::Value** vals)
-{
- ODesc buff;
- buff.Clear();
-
- // format the log entry
- formatter->Describe(&buff, num_fields, fields, vals);
-
- // send the formatted log entry to kafka
- const char* raw = (const char*)buff.Bytes();
- RdKafka::ErrorCode resp = producer->produce(
- topic, RdKafka::Topic::PARTITION_UA, RdKafka::Producer::RK_MSG_COPY,
- const_cast<char*>(raw), strlen(raw), NULL, NULL);
-
- if (RdKafka::ERR_NO_ERROR == resp) {
- producer->poll(0);
- }
- else {
- string err = RdKafka::err2str(resp);
- reporter->Error("Kafka send failed: %s", err.c_str());
- }
-
- return true;
-}
-
-/**
- * Writer-specific method implementing a change of fthe buffering
- * state. If buffering is disabled, the writer should attempt to
- * write out information as quickly as possible even if doing so may
- * have a performance impact. If enabled (which is the default), it
- * may buffer data as helpful and write it out later in a way
- * optimized for performance. The current buffering state can be
- * queried via IsBuf().
- */
-bool KafkaWriter::DoSetBuf(bool enabled)
-{
- // no change in behavior
- return true;
-}
-
-/**
- * Writer-specific method implementing flushing of its output. A writer
- * implementation must override this method but it can just
- * ignore calls if flushing doesn't align with its semantics.
- */
-bool KafkaWriter::DoFlush(double network_time)
-{
- producer->poll(0);
- return true;
-}
-
-/**
- * Writer-specific method implementing log rotation. Most directly
- * this only applies to writers writing into files, which should then
- * close the current file and open a new one. However, a writer may
- * also trigger other apppropiate actions if semantics are similar.
- * Once rotation has finished, the implementation *must* call
- * FinishedRotation() to signal the log manager that potential
- * postprocessors can now run.
- */
-bool KafkaWriter::DoRotate(const char* rotated_path, double open, double close, bool terminating)
-{
- // no need to perform log rotation
- return FinishedRotation();
-}
-
-/**
- * Triggered by regular heartbeat messages from the main thread.
- */
-bool KafkaWriter::DoHeartbeat(double network_time, double current_time)
-{
- producer->poll(0);
- return true;
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/src/KafkaWriter.h
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/src/KafkaWriter.h b/bro-plugin-kafka/src/KafkaWriter.h
deleted file mode 100644
index 2299667..0000000
--- a/bro-plugin-kafka/src/KafkaWriter.h
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef BRO_PLUGIN_BRO_KAFKA_KAFKAWRITER_H
-#define BRO_PLUGIN_BRO_KAFKA_KAFKAWRITER_H
-
-#include <string>
-#include <librdkafka/rdkafkacpp.h>
-#include <logging/WriterBackend.h>
-#include <threading/formatters/JSON.h>
-#include <Type.h>
-#include "kafka.bif.h"
-
-#include "TaggedJSON.h"
-
-namespace logging { namespace writer {
-
-/**
- * A logging writer that sends data to a Kafka broker.
- */
-class KafkaWriter : public WriterBackend {
-
-public:
- KafkaWriter(WriterFrontend* frontend);
- ~KafkaWriter();
-
- static WriterBackend* Instantiate(WriterFrontend* frontend)
- {
- return new KafkaWriter(frontend);
- }
-
-protected:
- virtual bool DoInit(const WriterBackend::WriterInfo& info, int num_fields, const threading::Field* const* fields);
- virtual bool DoWrite(int num_fields, const threading::Field* const* fields, threading::Value** vals);
- virtual bool DoSetBuf(bool enabled);
- virtual bool DoRotate(const char* rotated_path, double open, double close, bool terminating);
- virtual bool DoFlush(double network_time);
- virtual bool DoFinish(double network_time);
- virtual bool DoHeartbeat(double network_time, double current_time);
-
-private:
- string topic_name;
- threading::formatter::Formatter *formatter;
- RdKafka::Producer* producer;
- RdKafka::Topic* topic;
- RdKafka::Conf* conf;
- RdKafka::Conf* topic_conf;
-};
-
-}}
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/src/Plugin.cc
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/src/Plugin.cc b/bro-plugin-kafka/src/Plugin.cc
deleted file mode 100644
index d523d23..0000000
--- a/bro-plugin-kafka/src/Plugin.cc
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "Plugin.h"
-#include "KafkaWriter.h"
-
-namespace plugin { namespace Bro_Kafka {
- Plugin plugin;
-}}
-
-using namespace plugin::Bro_Kafka;
-
-plugin::Configuration Plugin::Configure()
-{
- AddComponent(new ::logging::Component("KafkaWriter", ::logging::writer::KafkaWriter::Instantiate));
-
- plugin::Configuration config;
- config.name = "Bro::Kafka";
- config.description = "Writes logs to Kafka";
- config.version.major = 0;
- config.version.minor = 1;
- return config;
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/src/Plugin.h
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/src/Plugin.h b/bro-plugin-kafka/src/Plugin.h
deleted file mode 100644
index 8adeb18..0000000
--- a/bro-plugin-kafka/src/Plugin.h
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef BRO_PLUGIN_BRO_KAFKA
-#define BRO_PLUGIN_BRO_KAFKA
-
-#include <plugin/Plugin.h>
-
-namespace plugin { namespace Bro_Kafka {
-
- class Plugin : public ::plugin::Plugin {
- protected:
- // Overridden from plugin::Plugin.
- virtual plugin::Configuration Configure();
- };
-
- extern Plugin plugin;
-}}
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/src/TaggedJSON.cc
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/src/TaggedJSON.cc b/bro-plugin-kafka/src/TaggedJSON.cc
deleted file mode 100644
index db3f305..0000000
--- a/bro-plugin-kafka/src/TaggedJSON.cc
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "TaggedJSON.h"
-
-namespace threading { namespace formatter {
-
-TaggedJSON::TaggedJSON(string sn, MsgThread* t, JSON::TimeFormat tf): JSON(t, tf), stream_name(sn)
-{}
-
-TaggedJSON::~TaggedJSON()
-{}
-
-bool TaggedJSON::Describe(ODesc* desc, int num_fields, const Field* const* fields, Value** vals) const
-{
- desc->AddRaw("{");
-
- // 'tag' the json; aka prepend the stream name to the json-formatted log content
- desc->AddRaw("\"");
- desc->AddRaw(stream_name);
- desc->AddRaw("\": ");
-
- // append the JSON formatted log record itself
- JSON::Describe(desc, num_fields, fields, vals);
-
- desc->AddRaw("}");
- return true;
-}
-}}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/src/TaggedJSON.h
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/src/TaggedJSON.h b/bro-plugin-kafka/src/TaggedJSON.h
deleted file mode 100644
index 08a50df..0000000
--- a/bro-plugin-kafka/src/TaggedJSON.h
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef BRO_PLUGIN_BRO_KAFKA_TAGGEDJSON_H
-#define BRO_PLUGIN_BRO_KAFKA_TAGGEDJSON_H
-
-#include <string>
-#include <threading/Formatter.h>
-#include <threading/formatters/JSON.h>
-
-using threading::formatter::JSON;
-using threading::MsgThread;
-using threading::Value;
-using threading::Field;
-
-namespace threading { namespace formatter {
-
-/*
- * A JSON formatter that prepends or 'tags' the content with a log stream
- * identifier. For example,
- * { 'conn' : { ... }}
- * { 'http' : { ... }}
- */
-class TaggedJSON : public JSON {
-
-public:
- TaggedJSON(string stream_name, MsgThread* t, JSON::TimeFormat tf);
- virtual ~TaggedJSON();
- virtual bool Describe(ODesc* desc, int num_fields, const Field* const* fields, Value** vals) const;
-
-private:
- string stream_name;
-};
-
-}}
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/src/kafka.bif
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/src/kafka.bif b/bro-plugin-kafka/src/kafka.bif
deleted file mode 100644
index 8a8070c..0000000
--- a/bro-plugin-kafka/src/kafka.bif
+++ /dev/null
@@ -1,23 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-module Kafka;
-
-const kafka_conf: config;
-const topic_name: string;
-const max_wait_on_shutdown: count;
-const tag_json: bool;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/src/kafka_const.bif
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/src/kafka_const.bif b/bro-plugin-kafka/src/kafka_const.bif
deleted file mode 100644
index 989c0ae..0000000
--- a/bro-plugin-kafka/src/kafka_const.bif
+++ /dev/null
@@ -1,20 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-module Kafka;
-
-type config : table[string] of string;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/tests/Makefile
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/tests/Makefile b/bro-plugin-kafka/tests/Makefile
deleted file mode 100644
index a637cd3..0000000
--- a/bro-plugin-kafka/tests/Makefile
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-test:
- @btest
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/tests/Scripts/get-bro-env
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/tests/Scripts/get-bro-env b/bro-plugin-kafka/tests/Scripts/get-bro-env
deleted file mode 100755
index 8aa0ea7..0000000
--- a/bro-plugin-kafka/tests/Scripts/get-bro-env
+++ /dev/null
@@ -1,36 +0,0 @@
-#! /bin/sh
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# BTest helper for getting values for Bro-related environment variables.
-
-base=`dirname $0`
-bro=`cat ${base}/../../build/CMakeCache.txt | grep BRO_DIST | cut -d = -f 2`
-
-if [ "$1" = "brobase" ]; then
- echo ${bro}
-elif [ "$1" = "bropath" ]; then
- ${bro}/build/bro-path-dev
-elif [ "$1" = "bro_plugin_path" ]; then
- ( cd ${base}/../.. && pwd )
-elif [ "$1" = "bro_seed_file" ]; then
- echo ${bro}/testing/btest/random.seed
-elif [ "$1" = "path" ]; then
- echo ${bro}/build/src:${bro}/aux/btest:${base}/:${bro}/aux/bro-cut:$PATH
-else
- echo "usage: `basename $0` <var>" >&2
- exit 1
-fi
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/tests/btest.cfg
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/tests/btest.cfg b/bro-plugin-kafka/tests/btest.cfg
deleted file mode 100644
index e42fefd..0000000
--- a/bro-plugin-kafka/tests/btest.cfg
+++ /dev/null
@@ -1,36 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-[btest]
-TestDirs = kafka
-TmpDir = %(testbase)s/.tmp
-BaselineDir = %(testbase)s/Baseline
-IgnoreDirs = .svn CVS .tmp
-IgnoreFiles = *.tmp *.swp #* *.trace .DS_Store
-
-[environment]
-BROBASE=`%(testbase)s/Scripts/get-bro-env brobase`
-BROPATH=`%(testbase)s/Scripts/get-bro-env bropath`
-BRO_PLUGIN_PATH=`%(testbase)s/Scripts/get-bro-env bro_plugin_path`
-BRO_SEED_FILE=`%(testbase)s/Scripts/get-bro-env bro_seed_file`
-PATH=`%(testbase)s/Scripts/get-bro-env path`
-TZ=UTC
-LC_ALL=C
-TRACES=%(testbase)s/Traces
-TMPDIR=%(testbase)s/.tmp
-BRO_TRACES=`%(testbase)s/Scripts/get-bro-env brobase`/testing/btest/Traces
-TEST_DIFF_CANONIFIER=`%(testbase)s/Scripts/get-bro-env brobase`/testing/scripts/diff-canonifier
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/bro-plugin-kafka/tests/kafka/show-plugin.bro
----------------------------------------------------------------------
diff --git a/bro-plugin-kafka/tests/kafka/show-plugin.bro b/bro-plugin-kafka/tests/kafka/show-plugin.bro
deleted file mode 100644
index 4e8dd6a..0000000
--- a/bro-plugin-kafka/tests/kafka/show-plugin.bro
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# @TEST-EXEC: bro -NN Bro::Kafka >output
-# @TEST-EXEC: btest-diff output
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/bro/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/bro/meta/main.yml b/deployment/roles/bro/meta/main.yml
index 320ce8d..9c9286f 100644
--- a/deployment/roles/bro/meta/main.yml
+++ b/deployment/roles/bro/meta/main.yml
@@ -16,8 +16,8 @@
#
---
dependencies:
- - ambari_gather_facts
- libselinux-python
+ - ambari_gather_facts
- build-tools
- kafka-client
-
+ - librdkafka
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/bro/tasks/bro-plugin-kafka.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/bro/tasks/bro-plugin-kafka.yml b/deployment/roles/bro/tasks/bro-plugin-kafka.yml
index d920688..d8e887d 100644
--- a/deployment/roles/bro/tasks/bro-plugin-kafka.yml
+++ b/deployment/roles/bro/tasks/bro-plugin-kafka.yml
@@ -16,7 +16,7 @@
#
---
- name: Distribute bro-kafka plugin
- copy: src=../../../bro-plugin-kafka dest=/tmp/ mode=0755
+ copy: src=../../../metron-sensors/bro-plugin-kafka dest=/tmp/ mode=0755
- name: Compile and install the plugin
shell: "{{ item }}"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/bro/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/bro/vars/main.yml b/deployment/roles/bro/vars/main.yml
index 1e1d1ab..2ff5177 100644
--- a/deployment/roles/bro/vars/main.yml
+++ b/deployment/roles/bro/vars/main.yml
@@ -24,8 +24,3 @@ bro_topic: bro
bro_crontab_minutes: 0-59/5
bro_crontab_job: "{{ bro_home }}/bin/broctl cron"
bro_clean_job: "rm -rf {{ bro_home }}/spool/tmp/*"
-
-# librdkafka
-librdkafka_version: 0.8.6
-librdkafka_url: https://github.com/edenhill/librdkafka/archive/0.8.6.tar.gz
-librdkafka_home: /usr/local
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/kafka-broker/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/kafka-broker/defaults/main.yml b/deployment/roles/kafka-broker/defaults/main.yml
new file mode 100644
index 0000000..351d125
--- /dev/null
+++ b/deployment/roles/kafka-broker/defaults/main.yml
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+hdp_repo_def: http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.2.0/hdp.repo
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/kafka-broker/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/kafka-broker/meta/main.yml b/deployment/roles/kafka-broker/meta/main.yml
new file mode 100644
index 0000000..9587e79
--- /dev/null
+++ b/deployment/roles/kafka-broker/meta/main.yml
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+dependencies:
+ - libselinux-python
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/kafka-broker/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/kafka-broker/tasks/main.yml b/deployment/roles/kafka-broker/tasks/main.yml
new file mode 100644
index 0000000..db05cb0
--- /dev/null
+++ b/deployment/roles/kafka-broker/tasks/main.yml
@@ -0,0 +1,41 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Retrieve HDP repository definition
+ get_url:
+ url: "{{ hdp_repo_def }}"
+ dest: /etc/yum.repos.d/hdp.repo
+ mode: 0644
+
+- name: Install kafka
+ yum: name={{item}}
+ with_items:
+ - java-1.8.0-openjdk
+ - kafka
+ - zookeeper-server
+
+- name: Create pid directories
+ file: path={{ item }} state=directory mode=0755
+ with_items:
+ - /var/run/zookeeper
+ - /var/run/kafka
+
+- name: Start zookeeper
+ shell: /usr/hdp/current/zookeeper-server/bin/zookeeper-server start
+
+- name: Start kafka
+ shell: /usr/hdp/current/kafka-broker/bin/kafka start
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/kafka-broker/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/kafka-broker/vars/main.yml b/deployment/roles/kafka-broker/vars/main.yml
new file mode 100644
index 0000000..351d125
--- /dev/null
+++ b/deployment/roles/kafka-broker/vars/main.yml
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+hdp_repo_def: http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.2.0/hdp.repo
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/librdkafka/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/librdkafka/defaults/main.yml b/deployment/roles/librdkafka/defaults/main.yml
new file mode 100644
index 0000000..d920883
--- /dev/null
+++ b/deployment/roles/librdkafka/defaults/main.yml
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+librdkafka_version: 0.8.6
+librdkafka_url: https://github.com/edenhill/librdkafka/archive/0.8.6.tar.gz
+librdkafka_home: /usr/local
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/librdkafka/tasks/dependencies.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/librdkafka/tasks/dependencies.yml b/deployment/roles/librdkafka/tasks/dependencies.yml
new file mode 100644
index 0000000..431e861
--- /dev/null
+++ b/deployment/roles/librdkafka/tasks/dependencies.yml
@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Install prerequisites
+ yum: name={{ item }}
+ with_items:
+ - cmake
+ - make
+ - gcc
+ - gcc-c++
+ - flex
+ - bison
+ - libpcap
+ - libpcap-devel
+ - openssl-devel
+ - python-devel
+ - swig
+ - zlib-devel
+ - perl
+ register: result
+ until: result.rc == 0
+ retries: 5
+ delay: 10
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/librdkafka/tasks/librdkafka.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/librdkafka/tasks/librdkafka.yml b/deployment/roles/librdkafka/tasks/librdkafka.yml
new file mode 100644
index 0000000..652d319
--- /dev/null
+++ b/deployment/roles/librdkafka/tasks/librdkafka.yml
@@ -0,0 +1,39 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Download librdkafka
+ get_url:
+ url: "{{ librdkafka_url }}"
+ dest: "/tmp/librdkafka-{{ librdkafka_version }}.tar.gz"
+
+- name: Extract librdkafka tarball
+ unarchive:
+ src: "/tmp/librdkafka-{{ librdkafka_version }}.tar.gz"
+ dest: /tmp
+ copy: no
+ creates: "/tmp/librdkafka-{{ librdkafka_version }}"
+
+- name: Compile and install librdkafka
+ shell: "{{ item }}"
+ args:
+ chdir: "/tmp/librdkafka-{{ librdkafka_version }}"
+ creates: "{{ librdkafka_home }}/lib/librdkafka.so"
+ with_items:
+ - rm -rf build/
+ - "./configure --prefix={{ librdkafka_home }}"
+ - make
+ - make install
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/librdkafka/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/librdkafka/tasks/main.yml b/deployment/roles/librdkafka/tasks/main.yml
new file mode 100644
index 0000000..2144d7f
--- /dev/null
+++ b/deployment/roles/librdkafka/tasks/main.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- include: dependencies.yml
+- include: librdkafka.yml
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/packet-capture/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/packet-capture/defaults/main.yml b/deployment/roles/packet-capture/defaults/main.yml
new file mode 100644
index 0000000..3e6358c
--- /dev/null
+++ b/deployment/roles/packet-capture/defaults/main.yml
@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+# dpdk
+dpdk_home: "/usr/local/dpdk"
+dpdk_version: "2.2.0"
+dpdk_sdk: "/root/dpdk-{{ dpdk_version }}"
+dpdk_target: "x86_64-native-linuxapp-gcc"
+num_huge_pages: 512
+extra_cflags: -g
+
+# pcapture
+pcapture_work_dir: /root/packet-capture
+pcapture_prefix: /usr/local/bin
+pcapture_ld_library_path: /usr/local/lib
+pcapture_portmask: 0x01
+pcapture_kafka_config: /etc/pcapture.conf
+pcapture_bin: pcapture
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/packet-capture/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/packet-capture/meta/main.yml b/deployment/roles/packet-capture/meta/main.yml
new file mode 100644
index 0000000..d253e88
--- /dev/null
+++ b/deployment/roles/packet-capture/meta/main.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+dependencies:
+ - librdkafka
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/564272e6/deployment/roles/packet-capture/tasks/debug.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/packet-capture/tasks/debug.yml b/deployment/roles/packet-capture/tasks/debug.yml
new file mode 100644
index 0000000..06f1526
--- /dev/null
+++ b/deployment/roles/packet-capture/tasks/debug.yml
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+ - name: Install debug utilities
+ yum: name=yum-utils
+ tags:
+ - debug
+
+ - name: Install debug symbols
+ shell: debuginfo-install -y glibc glib2 zlib
+ tags:
+ - debug