You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by jo...@apache.org on 2019/03/22 18:41:22 UTC
[metron-bro-plugin-kafka] branch METRON-2045 created (now dfc7884)
This is an automated email from the ASF dual-hosted git repository.
jonzeolla pushed a change to branch METRON-2045
in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git.
at dfc7884 METRON-2045: Pass a version argument to the bro plugin docker scripts
This branch includes the following new commits:
new dfc7884 METRON-2045: Pass a version argument to the bro plugin docker scripts
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[metron-bro-plugin-kafka] 01/01: METRON-2045: Pass a version
argument to the bro plugin docker scripts
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jonzeolla pushed a commit to branch METRON-2045
in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git
commit dfc78843b9c827d8bca474e1f27894e5d8c585f2
Author: Jon Zeolla <ze...@gmail.com>
AuthorDate: Fri Mar 22 12:23:29 2019 -0400
METRON-2045: Pass a version argument to the bro plugin docker scripts
---
docker/README.md | 2 +-
docker/in_docker_scripts/build_bro_plugin.sh | 59 ++++++++++++++++++++++-
docker/run_end_to_end.sh | 30 +++++++++---
docker/scripts/docker_execute_build_bro_plugin.sh | 23 +++++++--
docker/scripts/docker_execute_shell.sh | 2 +-
5 files changed, 103 insertions(+), 13 deletions(-)
diff --git a/docker/README.md b/docker/README.md
index a965d8b..47295cf 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -50,7 +50,7 @@ testing scripts to be added to a pull request, and subsequently to a test suite.
└── wait_for_zk.sh
```
-- `build_bro_plugin.sh`: Runs `bro-pkg` to build and install the plugin.
+- `build_bro_plugin.sh`: Runs `bro-pkg` to build and install the provided version of the plugin.
- `configure_bro_plugin.sh`: Configures the plugin for the kafka container, and routes all traffic types.
- `process_data_file.sh`: Runs `bro -r` on the passed file
- `wait-for-it.sh`: Waits for a port to be open, so we know something is available.
diff --git a/docker/in_docker_scripts/build_bro_plugin.sh b/docker/in_docker_scripts/build_bro_plugin.sh
index 46277bd..064fe91 100755
--- a/docker/in_docker_scripts/build_bro_plugin.sh
+++ b/docker/in_docker_scripts/build_bro_plugin.sh
@@ -18,16 +18,73 @@
#
shopt -s nocasematch
+shopt -s globstar nullglob
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
#
# Runs bro-pkg to build and install the plugin
#
+function help {
+ echo " "
+ echo "usage: ${0}"
+ echo " --plugin-version [REQUIRED] The plugin version."
+ echo " -h/--help Usage information."
+ echo " "
+ echo " "
+}
+
+PLUGIN_VERSION=
+
+# Handle command line options
+for i in "$@"; do
+ case $i in
+ #
+ # PLUGIN_VERSION
+ #
+ # --plugin-version
+ #
+ --plugin-version=*)
+ PLUGIN_VERSION="${i#*=}"
+ shift # past argument=value
+ ;;
+
+ #
+ # -h/--help
+ #
+ -h | --help)
+ help
+ exit 0
+ shift # past argument with no value
+ ;;
+
+ #
+ # Unknown option
+ #
+ *)
+ UNKNOWN_OPTION="${i#*=}"
+ echo "Error: unknown option: $UNKNOWN_OPTION"
+ help
+ ;;
+ esac
+done
+
+if [[ -z "${PLUGIN_VERSION}" ]]; then
+ echo "PLUGIN_VERSION must be passed"
+ exit 1
+fi
+
+echo "PLUGIN_VERSION = ${PLUGIN_VERSION}"
+
cd /root || exit 1
echo "================================"
-bro-pkg install code --force
+bro-pkg install code --version "${PLUGIN_VERSION}" --force
rc=$?; if [[ ${rc} != 0 ]]; then
echo "ERROR running bro-pkg install ${rc}"
exit ${rc}
diff --git a/docker/run_end_to_end.sh b/docker/run_end_to_end.sh
index 6dfd146..0c94e74 100755
--- a/docker/run_end_to_end.sh
+++ b/docker/run_end_to_end.sh
@@ -29,6 +29,7 @@ function help {
echo " --skip-docker-build [OPTIONAL] Skip build of bro docker machine."
echo " --data-path [OPTIONAL] The pcap data path. Default: ./data"
echo " --kafka-topic [OPTIONAL] The kafka topic to consume from. Default: bro"
+ echo " --plugin-version [OPTIONAL] The plugin version. Default: the current branch name"
echo " -h/--help Usage information."
echo " "
echo "COMPATABILITY"
@@ -53,6 +54,7 @@ DATE=$(date)
LOG_DATE=${DATE// /_}
TEST_OUTPUT_PATH="${ROOT_DIR}/test_output/"${LOG_DATE//:/_}
KAFKA_TOPIC="bro"
+PLUGIN_VERSION=$(cd "${ROOT_DIR}" && git rev-parse --symbolic-full-name --abbrev-ref HEAD)
# Handle command line options
for i in "$@"; do
@@ -66,6 +68,7 @@ for i in "$@"; do
SKIP_REBUILD_BRO=true
shift # past argument
;;
+
#
# DATA_PATH
#
@@ -73,6 +76,7 @@ for i in "$@"; do
DATA_PATH="${i#*=}"
shift # past argument=value
;;
+
#
# KAFKA_TOPIC
#
@@ -82,6 +86,17 @@ for i in "$@"; do
KAFKA_TOPIC="${i#*=}"
shift # past argument=value
;;
+
+ #
+ # PLUGIN_VERSION
+ #
+ # --plugin-version
+ #
+ --plugin-version=*)
+ PLUGIN_VERSION="${i#*=}"
+ shift # past argument=value
+ ;;
+
#
# -h/--help
#
@@ -96,9 +111,10 @@ done
EXTRA_ARGS="$*"
echo "Running build_container with "
-echo "SKIP_REBUILD_BRO = $SKIP_REBUILD_BRO"
-echo "DATA_PATH = $DATA_PATH"
-echo "KAFKA_TOPIC = $KAFKA_TOPIC"
+echo "SKIP_REBUILD_BRO = ${SKIP_REBUILD_BRO}"
+echo "DATA_PATH = ${DATA_PATH}"
+echo "KAFKA_TOPIC = ${KAFKA_TOPIC}"
+echo "PLUGIN_VERSION = ${PLUGIN_VERSION}"
echo "==================================================="
# Create the network
@@ -132,7 +148,7 @@ rc=$?; if [[ ${rc} != 0 ]]; then
fi
# Create the kafka topic
-bash "${SCRIPT_DIR}"/docker_run_create_topic_in_kafka.sh --kafka-topic=${KAFKA_TOPIC}
+bash "${SCRIPT_DIR}"/docker_run_create_topic_in_kafka.sh --kafka-topic="${KAFKA_TOPIC}"
rc=$?; if [[ ${rc} != 0 ]]; then
exit ${rc}
fi
@@ -167,7 +183,7 @@ rc=$?; if [[ ${rc} != 0 ]]; then
fi
# Build the bro plugin
-bash "${SCRIPT_DIR}"/docker_execute_build_bro_plugin.sh
+bash "${SCRIPT_DIR}"/docker_execute_build_bro_plugin.sh --plugin-version="${PLUGIN_VERSION}"
rc=$?; if [[ ${rc} != 0 ]]; then
echo "ERROR> FAILED TO BUILD PLUGIN. CHECK LOGS ${rc}"
exit ${rc}
@@ -198,7 +214,7 @@ do
# get the current offset in kafka
# this is where we are going to _start_
- OFFSET=$(bash "${SCRIPT_DIR}"/docker_run_get_offset_kafka.sh --kafka-topic=${KAFKA_TOPIC} | sed "s/^${KAFKA_TOPIC}:0:\(.*\)$/\1/")
+ OFFSET=$(bash "${SCRIPT_DIR}"/docker_run_get_offset_kafka.sh --kafka-topic="${KAFKA_TOPIC}" | sed "s/^${KAFKA_TOPIC}:0:\(.*\)$/\1/")
echo "OFFSET------------------> ${OFFSET}"
bash "${SCRIPT_DIR}"/docker_execute_process_data_file.sh --pcap-file-name="${BASE_FILE_NAME}" --output-directory-name="${DOCKER_DIRECTORY_NAME}"
@@ -208,7 +224,7 @@ do
fi
KAFKA_OUTPUT_FILE="${TEST_OUTPUT_PATH}/${DOCKER_DIRECTORY_NAME}/kafka-output.log"
- bash "${SCRIPT_DIR}"/docker_run_consume_kafka.sh --offset=${OFFSET} --kafka-topic=${KAFKA_TOPIC} | "${ROOT_DIR}"/remove_timeout_message.sh | tee "${KAFKA_OUTPUT_FILE}"
+ bash "${SCRIPT_DIR}"/docker_run_consume_kafka.sh --offset="${OFFSET}" --kafka-topic="${KAFKA_TOPIC}" | "${ROOT_DIR}"/remove_timeout_message.sh | tee "${KAFKA_OUTPUT_FILE}"
rc=$?; if [[ ${rc} != 0 ]]; then
echo "ERROR> FAILED TO PROCESS ${DATA_PATH} DATA. CHECK LOGS"
diff --git a/docker/scripts/docker_execute_build_bro_plugin.sh b/docker/scripts/docker_execute_build_bro_plugin.sh
index 649fd4e..2db600d 100755
--- a/docker/scripts/docker_execute_build_bro_plugin.sh
+++ b/docker/scripts/docker_execute_build_bro_plugin.sh
@@ -31,12 +31,14 @@ function help {
echo " "
echo "usage: ${0}"
echo " --container-name [OPTIONAL] The Docker container name. Default: bro"
+ echo " --plugin-version [REQUIRED] The plugin version."
echo " -h/--help Usage information."
echo " "
echo " "
}
-CONTAINER_NAME=bro
+CONTAINER_NAME="bro"
+PLUGIN_VERSION=
# handle command line options
for i in "$@"; do
@@ -52,6 +54,16 @@ for i in "$@"; do
;;
#
+ # PLUGIN_VERSION
+ #
+ # --plugin-version
+ #
+ --plugin-version=*)
+ PLUGIN_VERSION="${i#*=}"
+ shift # past argument=value
+ ;;
+
+ #
# -h/--help
#
-h | --help)
@@ -71,11 +83,16 @@ for i in "$@"; do
esac
done
-echo "Running build_bro_plugin_docker with "
+if [[ -z "${PLUGIN_VERSION}" ]]; then
+ echo "PLUGIN_VERSION must be passed"
+ exit 1
+fi
+
+echo "Running build_bro_plugin with "
echo "CONTAINER_NAME = $CONTAINER_NAME"
echo "==================================================="
-docker exec -w /root "${CONTAINER_NAME}" bash -c /root/built_in_scripts/build_bro_plugin.sh
+docker exec -w /root "${CONTAINER_NAME}" bash -c "/root/built_in_scripts/build_bro_plugin.sh --plugin-version=${PLUGIN_VERSION}"
rc=$?; if [[ ${rc} != 0 ]]; then
exit ${rc};
fi
diff --git a/docker/scripts/docker_execute_shell.sh b/docker/scripts/docker_execute_shell.sh
index f7c55de..1c7ff9f 100755
--- a/docker/scripts/docker_execute_shell.sh
+++ b/docker/scripts/docker_execute_shell.sh
@@ -71,7 +71,7 @@ for i in "$@"; do
esac
done
-echo "Running build_bro_plugin_docker with "
+echo "Running bash on "
echo "CONTAINER_NAME = $CONTAINER_NAME"
echo "==================================================="