You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by jo...@apache.org on 2019/04/29 12:25:39 UTC
[metron-bro-plugin-kafka] branch master updated: METRON-2045 Pass a
version argument to the bro plugin docker scripts (JonZeolla) closes
apache/metron-bro-plugin-kafka#35
This is an automated email from the ASF dual-hosted git repository.
jonzeolla pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git
The following commit(s) were added to refs/heads/master by this push:
new 43c9166 METRON-2045 Pass a version argument to the bro plugin docker scripts (JonZeolla) closes apache/metron-bro-plugin-kafka#35
43c9166 is described below
commit 43c9166787649e4ac2ab295a1baba94d54903651
Author: JonZeolla <ze...@gmail.com>
AuthorDate: Mon Apr 29 08:24:12 2019 -0400
METRON-2045 Pass a version argument to the bro plugin docker scripts (JonZeolla) closes apache/metron-bro-plugin-kafka#35
---
docker/README.md | 12 +++--
docker/in_docker_scripts/build_bro_plugin.sh | 59 ++++++++++++++++++++++-
docker/run_end_to_end.sh | 30 +++++++++---
docker/scripts/docker_execute_build_bro_plugin.sh | 23 +++++++--
docker/scripts/docker_execute_shell.sh | 2 +-
5 files changed, 110 insertions(+), 16 deletions(-)
diff --git a/docker/README.md b/docker/README.md
index bde7b5e..eac2919 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -50,7 +50,7 @@ testing scripts to be added to a pull request, and subsequently to a test suite.
└── wait_for_zk.sh
```
-- `build_bro_plugin.sh`: Runs `bro-pkg` to build and install the plugin.
+- `build_bro_plugin.sh`: Runs `bro-pkg` to build and install the provided version of the plugin.
- `configure_bro_plugin.sh`: Configures the plugin for the kafka container, and routes all traffic types.
- `process_data_file.sh`: Runs `bro -r` on the passed file
- `wait-for-it.sh`: Waits for a port to be open, so we know something is available.
@@ -294,7 +294,11 @@ Other scripts may then be used to do your testing, for example running:
##### `run_end_to_end.sh`
###### Parameters
```bash
---skip-docker-build [OPTIONAL] Skip build of bro docker machine.
---data-path [OPTIONAL] The pcap data path. Default: ./data
---kafka-topic [OPTIONAL] The kafka topic name to use. Default: bro
+--skip-docker-build [OPTIONAL] Skip build of bro docker machine.
+--data-path [OPTIONAL] The pcap data path. Default: ./data
+--kafka-topic [OPTIONAL] The kafka topic name to use. Default: bro
+--plugin-version [OPTIONAL] The plugin version. Default: the current branch name
```
+
+> NOTE: The provided `--plugin-version` is passed to the [`bro-pkg install`](https://docs.zeek.org/projects/package-manager/en/stable/bro-pkg.html#install-command) command within the container, which allows you to specify a version tag, branch name, or commit hash. However, that tag, branch, or commit *must* be available in the currently checked out plugin repository.
+
diff --git a/docker/in_docker_scripts/build_bro_plugin.sh b/docker/in_docker_scripts/build_bro_plugin.sh
index 46277bd..064fe91 100755
--- a/docker/in_docker_scripts/build_bro_plugin.sh
+++ b/docker/in_docker_scripts/build_bro_plugin.sh
@@ -18,16 +18,73 @@
#
shopt -s nocasematch
+shopt -s globstar nullglob
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
#
# Runs bro-pkg to build and install the plugin
#
+function help {
+ echo " "
+ echo "usage: ${0}"
+ echo " --plugin-version [REQUIRED] The plugin version."
+ echo " -h/--help Usage information."
+ echo " "
+ echo " "
+}
+
+PLUGIN_VERSION=
+
+# Handle command line options
+for i in "$@"; do
+ case $i in
+ #
+ # PLUGIN_VERSION
+ #
+ # --plugin-version
+ #
+ --plugin-version=*)
+ PLUGIN_VERSION="${i#*=}"
+ shift # past argument=value
+ ;;
+
+ #
+ # -h/--help
+ #
+ -h | --help)
+ help
+ exit 0
+ shift # past argument with no value
+ ;;
+
+ #
+ # Unknown option
+ #
+ *)
+ UNKNOWN_OPTION="${i#*=}"
+ echo "Error: unknown option: $UNKNOWN_OPTION"
+ help
+ ;;
+ esac
+done
+
+if [[ -z "${PLUGIN_VERSION}" ]]; then
+ echo "PLUGIN_VERSION must be passed"
+ exit 1
+fi
+
+echo "PLUGIN_VERSION = ${PLUGIN_VERSION}"
+
cd /root || exit 1
echo "================================"
-bro-pkg install code --force
+bro-pkg install code --version "${PLUGIN_VERSION}" --force
rc=$?; if [[ ${rc} != 0 ]]; then
echo "ERROR running bro-pkg install ${rc}"
exit ${rc}
diff --git a/docker/run_end_to_end.sh b/docker/run_end_to_end.sh
index 4c61560..3ec0145 100755
--- a/docker/run_end_to_end.sh
+++ b/docker/run_end_to_end.sh
@@ -29,6 +29,7 @@ function help {
echo " --skip-docker-build [OPTIONAL] Skip build of bro docker machine."
echo " --data-path [OPTIONAL] The pcap data path. Default: ./data"
echo " --kafka-topic [OPTIONAL] The kafka topic to consume from. Default: bro"
+ echo " --plugin-version [OPTIONAL] The plugin version. Default: the current branch name"
echo " -h/--help Usage information."
echo " "
echo "COMPATABILITY"
@@ -53,6 +54,7 @@ DATE=$(date)
LOG_DATE=${DATE// /_}
TEST_OUTPUT_PATH="${ROOT_DIR}/test_output/"${LOG_DATE//:/_}
KAFKA_TOPIC="bro"
+PLUGIN_VERSION=$(cd "${ROOT_DIR}" && git rev-parse --symbolic-full-name --abbrev-ref HEAD)
# Handle command line options
for i in "$@"; do
@@ -66,6 +68,7 @@ for i in "$@"; do
SKIP_REBUILD_BRO=true
shift # past argument
;;
+
#
# DATA_PATH
#
@@ -73,6 +76,7 @@ for i in "$@"; do
DATA_PATH="${i#*=}"
shift # past argument=value
;;
+
#
# KAFKA_TOPIC
#
@@ -82,6 +86,17 @@ for i in "$@"; do
KAFKA_TOPIC="${i#*=}"
shift # past argument=value
;;
+
+ #
+ # PLUGIN_VERSION
+ #
+ # --plugin-version
+ #
+ --plugin-version=*)
+ PLUGIN_VERSION="${i#*=}"
+ shift # past argument=value
+ ;;
+
#
# -h/--help
#
@@ -96,9 +111,10 @@ done
EXTRA_ARGS="$*"
echo "Running build_container with "
-echo "SKIP_REBUILD_BRO = $SKIP_REBUILD_BRO"
-echo "DATA_PATH = $DATA_PATH"
-echo "KAFKA_TOPIC = $KAFKA_TOPIC"
+echo "SKIP_REBUILD_BRO = ${SKIP_REBUILD_BRO}"
+echo "DATA_PATH = ${DATA_PATH}"
+echo "KAFKA_TOPIC = ${KAFKA_TOPIC}"
+echo "PLUGIN_VERSION = ${PLUGIN_VERSION}"
echo "==================================================="
# Create the network
@@ -132,7 +148,7 @@ rc=$?; if [[ ${rc} != 0 ]]; then
fi
# Create the kafka topic
-bash "${SCRIPT_DIR}"/docker_run_create_topic_in_kafka.sh --kafka-topic=${KAFKA_TOPIC}
+bash "${SCRIPT_DIR}"/docker_run_create_topic_in_kafka.sh --kafka-topic="${KAFKA_TOPIC}"
rc=$?; if [[ ${rc} != 0 ]]; then
exit ${rc}
fi
@@ -167,7 +183,7 @@ rc=$?; if [[ ${rc} != 0 ]]; then
fi
# Build the bro plugin
-bash "${SCRIPT_DIR}"/docker_execute_build_bro_plugin.sh
+bash "${SCRIPT_DIR}"/docker_execute_build_bro_plugin.sh --plugin-version="${PLUGIN_VERSION}"
rc=$?; if [[ ${rc} != 0 ]]; then
echo "ERROR> FAILED TO BUILD PLUGIN. CHECK LOGS ${rc}"
exit ${rc}
@@ -198,7 +214,7 @@ do
# get the current offset in kafka
# this is where we are going to _start_
- OFFSET=$(bash "${SCRIPT_DIR}"/docker_run_get_offset_kafka.sh --kafka-topic=${KAFKA_TOPIC} | sed "s/^${KAFKA_TOPIC}:0:\(.*\)$/\1/")
+ OFFSET=$(bash "${SCRIPT_DIR}"/docker_run_get_offset_kafka.sh --kafka-topic="${KAFKA_TOPIC}" | sed "s/^${KAFKA_TOPIC}:0:\(.*\)$/\1/")
echo "OFFSET------------------> ${OFFSET}"
bash "${SCRIPT_DIR}"/docker_execute_process_data_file.sh --pcap-file-name="${BASE_FILE_NAME}" --output-directory-name="${DOCKER_DIRECTORY_NAME}"
@@ -208,7 +224,7 @@ do
fi
KAFKA_OUTPUT_FILE="${TEST_OUTPUT_PATH}/${DOCKER_DIRECTORY_NAME}/kafka-output.log"
- bash "${SCRIPT_DIR}"/docker_run_consume_kafka.sh --offset=${OFFSET} --kafka-topic=${KAFKA_TOPIC} | "${ROOT_DIR}"/remove_timeout_message.sh | tee "${KAFKA_OUTPUT_FILE}"
+ bash "${SCRIPT_DIR}"/docker_run_consume_kafka.sh --offset="${OFFSET}" --kafka-topic="${KAFKA_TOPIC}" | "${ROOT_DIR}"/remove_timeout_message.sh | tee "${KAFKA_OUTPUT_FILE}"
rc=$?; if [[ ${rc} != 0 ]]; then
echo "ERROR> FAILED TO PROCESS ${DATA_PATH} DATA. CHECK LOGS"
diff --git a/docker/scripts/docker_execute_build_bro_plugin.sh b/docker/scripts/docker_execute_build_bro_plugin.sh
index 649fd4e..2db600d 100755
--- a/docker/scripts/docker_execute_build_bro_plugin.sh
+++ b/docker/scripts/docker_execute_build_bro_plugin.sh
@@ -31,12 +31,14 @@ function help {
echo " "
echo "usage: ${0}"
echo " --container-name [OPTIONAL] The Docker container name. Default: bro"
+ echo " --plugin-version [REQUIRED] The plugin version."
echo " -h/--help Usage information."
echo " "
echo " "
}
-CONTAINER_NAME=bro
+CONTAINER_NAME="bro"
+PLUGIN_VERSION=
# handle command line options
for i in "$@"; do
@@ -52,6 +54,16 @@ for i in "$@"; do
;;
#
+ # PLUGIN_VERSION
+ #
+ # --plugin-version
+ #
+ --plugin-version=*)
+ PLUGIN_VERSION="${i#*=}"
+ shift # past argument=value
+ ;;
+
+ #
# -h/--help
#
-h | --help)
@@ -71,11 +83,16 @@ for i in "$@"; do
esac
done
-echo "Running build_bro_plugin_docker with "
+if [[ -z "${PLUGIN_VERSION}" ]]; then
+ echo "PLUGIN_VERSION must be passed"
+ exit 1
+fi
+
+echo "Running build_bro_plugin with "
echo "CONTAINER_NAME = $CONTAINER_NAME"
echo "==================================================="
-docker exec -w /root "${CONTAINER_NAME}" bash -c /root/built_in_scripts/build_bro_plugin.sh
+docker exec -w /root "${CONTAINER_NAME}" bash -c "/root/built_in_scripts/build_bro_plugin.sh --plugin-version=${PLUGIN_VERSION}"
rc=$?; if [[ ${rc} != 0 ]]; then
exit ${rc};
fi
diff --git a/docker/scripts/docker_execute_shell.sh b/docker/scripts/docker_execute_shell.sh
index f7c55de..1c7ff9f 100755
--- a/docker/scripts/docker_execute_shell.sh
+++ b/docker/scripts/docker_execute_shell.sh
@@ -71,7 +71,7 @@ for i in "$@"; do
esac
done
-echo "Running build_bro_plugin_docker with "
+echo "Running bash on "
echo "CONTAINER_NAME = $CONTAINER_NAME"
echo "==================================================="