You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by sz...@apache.org on 2023/05/23 03:51:31 UTC
[nifi-minifi-cpp] 03/04: MINIFICPP-2113 Add support for multi arch docker build
This is an automated email from the ASF dual-hosted git repository.
szaszm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 1d4c0718261a6d7610c4990cbc67e7ccae7be97d
Author: Gabor Gyimesi <ga...@gmail.com>
AuthorDate: Mon May 22 14:14:05 2023 +0200
MINIFICPP-2113 Add support for multi arch docker build
Closes #1569
Signed-off-by: Marton Szasz <sz...@apache.org>
---
CMakeLists.txt | 2 +
bootstrap.sh | 2 +-
cmake/DockerConfig.cmake | 28 ++++++++++-
cmake/MiNiFiOptions.cmake | 3 +-
docker/DockerBuild.sh | 116 ++++++++++++++++++++++++++++++++--------------
5 files changed, 112 insertions(+), 39 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index ce9589bfa..2e76bc7f7 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -33,6 +33,8 @@ set(BUILD_NUMBER "" CACHE STRING "Build number")
# Base Alpine image to be used in the Docker build instead of the default Alpine image
set(DOCKER_BASE_IMAGE "" CACHE STRING "Docker build Alpine base image")
set(DOCKER_CCACHE_DUMP_LOCATION "" CACHE STRING "Directory to dump ccache to after docker build for later reuse")
+set(DOCKER_PLATFORMS "" CACHE STRING "Build platforms for docker image build")
+set(DOCKER_TAGS "" CACHE STRING "Comma separated tags to override default docker build image:tag pairs with")
include(CMakeDependentOption)
include(CheckIncludeFile)
diff --git a/bootstrap.sh b/bootstrap.sh
index b6c1390f4..985f183c5 100755
--- a/bootstrap.sh
+++ b/bootstrap.sh
@@ -151,7 +151,7 @@ elif [ -f /etc/SUSE-release ]; then
OS=$(tr '\n' ' ' < /etc/SUSE-release | sed s/VERSION.*//)
elif [ -f /etc/redhat-release ]; then
# Older Red Hat, CentOS, etc.
- ...
+ echo "Unsupported old release!"
else
OS=$(uname -s)
VER=$(uname -r)
diff --git a/cmake/DockerConfig.cmake b/cmake/DockerConfig.cmake
index 8721cdd47..c70539106 100644
--- a/cmake/DockerConfig.cmake
+++ b/cmake/DockerConfig.cmake
@@ -28,6 +28,9 @@ add_custom_target(
-c DOCKER_SKIP_TESTS=${DOCKER_SKIP_TESTS}
-c CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-c BUILD_NUMBER=${BUILD_NUMBER}
+ -c DOCKER_PLATFORMS=${DOCKER_PLATFORMS}
+ -c DOCKER_PUSH=${DOCKER_PUSH}
+ -c DOCKER_TAGS=${DOCKER_TAGS}
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/)
# Create minimal docker image
@@ -36,7 +39,7 @@ add_custom_target(
COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerBuild.sh
-u 1000
-g 1000
- -t minimal
+ -p minimal
-v ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}
-o \"-DENABLE_LIBRDKAFKA=ON
-DENABLE_AWS=ON
@@ -47,12 +50,15 @@ add_custom_target(
-DENABLE_ELASTICSEARCH=OFF
-DENABLE_LUA_SCRIPTING=OFF
-DENABLE_PYTHON_SCRIPTING=OFF
- -DENABLE_OPC=OFF
+ -DENABLE_OPC=OFF
-DENABLE_ENCRYPT_CONFIG=OFF \"
-c DOCKER_BASE_IMAGE=${DOCKER_BASE_IMAGE}
-c DOCKER_SKIP_TESTS=${DOCKER_SKIP_TESTS}
-c BUILD_NUMBER=${BUILD_NUMBER}
-c CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+ -c DOCKER_PLATFORMS=${DOCKER_PLATFORMS}
+ -c DOCKER_PUSH=${DOCKER_PUSH}
+ -c DOCKER_TAGS=${DOCKER_TAGS}
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/)
add_custom_target(
@@ -68,6 +74,9 @@ add_custom_target(
-c DOCKER_CCACHE_DUMP_LOCATION=${DOCKER_CCACHE_DUMP_LOCATION}
-c DOCKER_SKIP_TESTS=${DOCKER_SKIP_TESTS}
-c CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+ -c DOCKER_PLATFORMS=${DOCKER_PLATFORMS}
+ -c DOCKER_PUSH=${DOCKER_PUSH}
+ -c DOCKER_TAGS=${DOCKER_TAGS}
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/)
add_custom_target(
@@ -81,6 +90,9 @@ add_custom_target(
-c BUILD_NUMBER=${BUILD_NUMBER}
-c DOCKER_CCACHE_DUMP_LOCATION=${DOCKER_CCACHE_DUMP_LOCATION}
-c DOCKER_SKIP_TESTS=OFF
+ -c DOCKER_PLATFORMS=${DOCKER_PLATFORMS}
+ -c DOCKER_PUSH=${DOCKER_PUSH}
+ -c DOCKER_TAGS=${DOCKER_TAGS}
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/)
add_custom_target(
@@ -94,6 +106,9 @@ add_custom_target(
-d fedora
-c BUILD_NUMBER=${BUILD_NUMBER}
-c CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+ -c DOCKER_PLATFORMS=${DOCKER_PLATFORMS}
+ -c DOCKER_PUSH=${DOCKER_PUSH}
+ -c DOCKER_TAGS=${DOCKER_TAGS}
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/)
add_custom_target(
@@ -107,6 +122,9 @@ add_custom_target(
-d bionic
-c BUILD_NUMBER=${BUILD_NUMBER}
-c CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+ -c DOCKER_PLATFORMS=${DOCKER_PLATFORMS}
+ -c DOCKER_PUSH=${DOCKER_PUSH}
+ -c DOCKER_TAGS=${DOCKER_TAGS}
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/)
add_custom_target(
@@ -120,6 +138,9 @@ add_custom_target(
-d focal
-c BUILD_NUMBER=${BUILD_NUMBER}
-c CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+ -c DOCKER_PLATFORMS=${DOCKER_PLATFORMS}
+ -c DOCKER_PUSH=${DOCKER_PUSH}
+ -c DOCKER_TAGS=${DOCKER_TAGS}
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/)
add_custom_target(
@@ -135,6 +156,9 @@ add_custom_target(
-c DOCKER_CCACHE_DUMP_LOCATION=${DOCKER_CCACHE_DUMP_LOCATION}
-c DOCKER_SKIP_TESTS=${DOCKER_SKIP_TESTS}
-c CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+ -c DOCKER_PLATFORMS=${DOCKER_PLATFORMS}
+ -c DOCKER_PUSH=${DOCKER_PUSH}
+ -c DOCKER_TAGS=${DOCKER_TAGS}
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/)
if (EXISTS ${CMAKE_SOURCE_DIR}/docker/test/integration/features)
diff --git a/cmake/MiNiFiOptions.cmake b/cmake/MiNiFiOptions.cmake
index 1f4c5d5e5..1da46a8da 100644
--- a/cmake/MiNiFiOptions.cmake
+++ b/cmake/MiNiFiOptions.cmake
@@ -38,6 +38,7 @@ add_minifi_option(CI_BUILD "Build is used for CI." OFF)
add_minifi_option(SKIP_TESTS "Skips building all tests." OFF)
add_minifi_option(DOCKER_BUILD_ONLY "Disables all targets except docker build scripts. Ideal for systems without an up-to-date compiler." OFF)
add_minifi_option(DOCKER_SKIP_TESTS "Skip building tests in docker image targets." ON)
+add_minifi_option(DOCKER_PUSH "Push created images to the specified tags" OFF)
add_minifi_option(PORTABLE "Instructs the compiler to remove architecture specific optimizations" ON)
add_minifi_option(USE_SHARED_LIBS "Builds using shared libraries" ON)
@@ -134,7 +135,7 @@ endif()
function(get_minifi_docker_options RET_VALUE)
set(MINIFI_DOCKER_OPTIONS_STR ${MINIFI_EXTERNAL_DOCKER_OPTIONS_STR})
foreach(MINIFI_OPTION ${MINIFI_OPTIONS})
- if (MINIFI_OPTION STREQUAL "CI_BUILD" OR MINIFI_OPTION STREQUAL "SKIP_TESTS" OR MINIFI_OPTION STREQUAL "DOCKER_BUILD_ONLY" OR MINIFI_OPTION STREQUAL "DOCKER_SKIP_TESTS")
+ if (MINIFI_OPTION STREQUAL "CI_BUILD" OR MINIFI_OPTION STREQUAL "SKIP_TESTS" OR MINIFI_OPTION STREQUAL "DOCKER_BUILD_ONLY" OR MINIFI_OPTION STREQUAL "DOCKER_SKIP_TESTS" OR MINIFI_OPTION STREQUAL "DOCKER_PUSH")
continue()
endif()
set(MINIFI_DOCKER_OPTIONS_STR "${MINIFI_DOCKER_OPTIONS_STR} -D${MINIFI_OPTION}=${${MINIFI_OPTION}}")
diff --git a/docker/DockerBuild.sh b/docker/DockerBuild.sh
index 4ac0c4050..0bfeeb6bb 100755
--- a/docker/DockerBuild.sh
+++ b/docker/DockerBuild.sh
@@ -24,19 +24,23 @@ set -euo pipefail
UID_ARG=1000
GID_ARG=1000
MINIFI_VERSION=
-IMAGE_TAG=
+PREFIX=
DUMP_LOCATION=
DISTRO_NAME=
BUILD_NUMBER=
DOCKER_CCACHE_DUMP_LOCATION=
DOCKER_SKIP_TESTS=ON
CMAKE_BUILD_TYPE=Release
+PLATFORMS=
+PUSH_OR_LOAD="--load"
+TAGS=()
+MULTI_ARCH=no
function usage {
echo "Usage: ./DockerBuild.sh -v <MINIFI_VERSION> [additional options]"
echo "Options:"
echo "-v, --minifi-version Minifi version number to be used (required)"
- echo "-t, --tag Additional prefix added to the image tag"
+ echo "-p, --prefix Additional prefix added to the image tag"
echo "-u, --uid User id to be used in the Docker image (default: 1000)"
echo "-g, --gid Group id to be used in the Docker image (default: 1000)"
echo "-d, --distro-name Linux distribution build to be used for alternative builds (bionic|focal|fedora|centos)"
@@ -50,7 +54,7 @@ function usage {
function dump_ccache() {
ccache_source_image=$1
docker_ccache_dump_location=$2
- container_id=$(docker run --rm -d "${ccache_source_image}" sh -c "while true; do sleep 1; done")
+ container_id=$(docker create "${ccache_source_image}")
mkdir -p "${docker_ccache_dump_location}"
docker cp "${container_id}:/opt/minifi/.ccache/." "${docker_ccache_dump_location}"
docker rm -f "${container_id}"
@@ -75,8 +79,8 @@ while [[ $# -gt 0 ]]; do
shift
shift
;;
- -t | --tag)
- IMAGE_TAG="$2"
+ -p | --prefix)
+ PREFIX="$2"
shift
shift
;;
@@ -103,6 +107,14 @@ while [[ $# -gt 0 ]]; do
DOCKER_SKIP_TESTS="${ARR[1]}"
elif [ "${ARR[0]}" == "CMAKE_BUILD_TYPE" ]; then
CMAKE_BUILD_TYPE="${ARR[1]}"
+ elif [ "${ARR[0]}" == "DOCKER_PLATFORMS" ] && [ -n "${ARR[1]}" ]; then
+ PLATFORMS="--platform ${ARR[1]}"
+ elif [ "${ARR[0]}" == "DOCKER_PUSH" ]; then
+ if [ "${ARR[1]}" == "ON" ]; then
+ PUSH_OR_LOAD="--push"
+ fi
+ elif [ "${ARR[0]}" == "DOCKER_TAGS" ] && [ -n "${ARR[1]}" ]; then
+ IFS=', ' read -r -a TAGS <<< "${ARR[1]}"
else
BUILD_ARGS+=("--build-arg" "${ARR[0]}=${ARR[1]}")
fi
@@ -129,6 +141,25 @@ if [ -z "${MINIFI_VERSION}" ]; then
usage
fi
+if [[ "${PLATFORMS}" == *,* ]]; then
+ MULTI_ARCH=yes
+fi
+
+if { [ -n "${DUMP_LOCATION}" ] || [ -n "${DOCKER_CCACHE_DUMP_LOCATION}" ]; } && [ "${PUSH_OR_LOAD}" == "--push" ]; then
+ echo "Error: Cannot save ccache dump or binaries locally when pushing image to remote registry. Turn DOCKER_PUSH flag OFF if you want to build binaries on docker."
+ exit 1
+fi
+
+if { [ -n "${DUMP_LOCATION}" ] || [ -n "${DOCKER_CCACHE_DUMP_LOCATION}" ]; } && [ "${MULTI_ARCH}" == "yes" ]; then
+ echo "Error: Cannot save ccache dump or binaries locally when using multi-arch image. Change DOCKER_PLATFORMS option to a single architecture if you want to build binaries on docker."
+ exit 1
+fi
+
+if [ "${MULTI_ARCH}" == "yes" ] && [ "${PUSH_OR_LOAD}" == "--load" ]; then
+ echo "Error: Multi-arch image cannot be locally built. Either change DOCKER_PLATFORMS flag to a single architecture or turn DOCKER_PUSH flag ON."
+ exit 1
+fi
+
echo "NiFi-MiNiFi-CPP Version: ${MINIFI_VERSION}"
if [ -n "${DISTRO_NAME}" ]; then
@@ -137,53 +168,68 @@ else
DOCKERFILE="Dockerfile"
fi
-TAG=""
-if [ -n "${IMAGE_TAG}" ]; then
- TAG="${IMAGE_TAG}-"
-fi
-if [ -n "${DISTRO_NAME}" ]; then
- TAG="${TAG}${DISTRO_NAME}-"
-fi
-TAG="${TAG}${MINIFI_VERSION}"
-if [ -n "${BUILD_NUMBER}" ]; then
- TAG="${TAG}-${BUILD_NUMBER}"
-fi
+if [ ${#TAGS[@]} -eq 0 ]; then
+ TAG="apacheminificpp:"
+ if [ -n "${PREFIX}" ]; then
+ TAG="${TAG}${PREFIX}-"
+ fi
+ if [ -n "${DISTRO_NAME}" ]; then
+ TAG="${TAG}${DISTRO_NAME}-"
+ fi
+ TAG="${TAG}${MINIFI_VERSION}"
+ if [ -n "${BUILD_NUMBER}" ]; then
+ TAG="${TAG}-${BUILD_NUMBER}"
+ fi
-TARGZ_TAG="bin"
-if [ -n "${DISTRO_NAME}" ]; then
- TARGZ_TAG="${TARGZ_TAG}-${DISTRO_NAME}"
-fi
-if [ -n "${BUILD_NUMBER}" ]; then
- TARGZ_TAG="${TARGZ_TAG}-${BUILD_NUMBER}"
+ TAGS+=("${TAG}")
+
+ TARGZ_TAG="bin"
+ if [ -n "${DISTRO_NAME}" ]; then
+ TARGZ_TAG="${TARGZ_TAG}-${DISTRO_NAME}"
+ fi
+ if [ -n "${BUILD_NUMBER}" ]; then
+ TARGZ_TAG="${TARGZ_TAG}-${BUILD_NUMBER}"
+ fi
fi
BUILD_ARGS+=("--build-arg" "UID=${UID_ARG}"
- "--build-arg" "GID=${GID_ARG}"
- "--build-arg" "MINIFI_VERSION=${MINIFI_VERSION}"
- "--build-arg" "DUMP_LOCATION=${DUMP_LOCATION}"
- "--build-arg" "DISTRO_NAME=${DISTRO_NAME}"
- "--build-arg" "DOCKER_SKIP_TESTS=${DOCKER_SKIP_TESTS}"
- "--build-arg" "CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}")
+ "--build-arg" "GID=${GID_ARG}"
+ "--build-arg" "MINIFI_VERSION=${MINIFI_VERSION}"
+ "--build-arg" "DUMP_LOCATION=${DUMP_LOCATION}"
+ "--build-arg" "DISTRO_NAME=${DISTRO_NAME}"
+ "--build-arg" "DOCKER_SKIP_TESTS=${DOCKER_SKIP_TESTS}"
+ "--build-arg" "CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}")
if [ -n "${DUMP_LOCATION}" ]; then
BUILD_ARGS+=("--build-arg" "DOCKER_MAKE_TARGET=package")
fi
+TAGGING_CMD=""
+for t in "${TAGS[@]}"; do
+ TAGGING_CMD="${TAGGING_CMD} -t ${t}"
+done
+
if [ -n "${DISTRO_NAME}" ]; then
- echo DOCKER_BUILDKIT=1 docker build "${BUILD_ARGS[@]}" -f "${DOCKERFILE}" -t apacheminificpp:"${TAG}" ..
- DOCKER_BUILDKIT=1 docker build "${BUILD_ARGS[@]}" -f "${DOCKERFILE}" -t apacheminificpp:"${TAG}" ..
+ # shellcheck disable=SC2086
+ echo docker buildx build "${BUILD_ARGS[@]}" -f "${DOCKERFILE}" ${PLATFORMS} ${PUSH_OR_LOAD} ${TAGGING_CMD} ..
+ # shellcheck disable=SC2086
+ docker buildx build "${BUILD_ARGS[@]}" -f "${DOCKERFILE}" ${PLATFORMS} ${PUSH_OR_LOAD} ${TAGGING_CMD} ..
if [ -n "${DOCKER_CCACHE_DUMP_LOCATION}" ]; then
- dump_ccache "apacheminificpp:${TAG}" "${DOCKER_CCACHE_DUMP_LOCATION}"
+ dump_ccache "${TAG[0]}" "${DOCKER_CCACHE_DUMP_LOCATION}"
fi
else
if [ -n "${DOCKER_CCACHE_DUMP_LOCATION}" ]; then
- DOCKER_BUILDKIT=1 docker build "${BUILD_ARGS[@]}" -f "${DOCKERFILE}" --target build -t minifi_build ..
+ docker buildx build "${BUILD_ARGS[@]}" -f "${DOCKERFILE}" --target build -t minifi_build ..
dump_ccache "minifi_build" "${DOCKER_CCACHE_DUMP_LOCATION}"
fi
- echo DOCKER_BUILDKIT=1 docker build "${BUILD_ARGS[@]}" -f "${DOCKERFILE}" -t apacheminificpp:"${TAG}" ..
- DOCKER_BUILDKIT=1 docker build "${BUILD_ARGS[@]}" -f "${DOCKERFILE}" -t apacheminificpp:"${TAG}" ..
+ # shellcheck disable=SC2086
+ echo docker buildx build "${BUILD_ARGS[@]}" -f "${DOCKERFILE}" ${PLATFORMS} ${PUSH_OR_LOAD} ${TAGGING_CMD} ..
+ # shellcheck disable=SC2086
+ docker buildx build "${BUILD_ARGS[@]}" -f "${DOCKERFILE}" ${PLATFORMS} ${PUSH_OR_LOAD} ${TAGGING_CMD} ..
fi
if [ -n "${DUMP_LOCATION}" ]; then
- docker run --rm --entrypoint cat "apacheminificpp:${TAG}" "/opt/minifi/build/nifi-minifi-cpp-${MINIFI_VERSION}.tar.gz" >"${DUMP_LOCATION}/nifi-minifi-cpp-${MINIFI_VERSION}-${TARGZ_TAG}.tar.gz"
+ container_id=$(docker create "${TAG}")
+ docker cp "${container_id}:/opt/minifi/build/nifi-minifi-cpp-${MINIFI_VERSION}.tar.gz" "${DUMP_LOCATION}/nifi-minifi-cpp-${MINIFI_VERSION}-${TARGZ_TAG}.tar.gz"
+ docker rm -f "${container_id}"
fi