You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by sz...@apache.org on 2023/03/03 13:23:51 UTC
[nifi-minifi-cpp] 03/05: MINIFICPP-2053 Reduce CentOS image size in CI build
This is an automated email from the ASF dual-hosted git repository.
szaszm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 7942e0e7f344d449bb8b4635b1a8737caf57bce5
Author: Gabor Gyimesi <ga...@gmail.com>
AuthorDate: Fri Mar 3 14:16:42 2023 +0100
MINIFICPP-2053 Reduce CentOS image size in CI build
It started to occur that the CentOS build exceeded the 14GB disk space
Github Actions VMs provide (for example
https://github.com/apache/nifi-minifi-cpp/actions/runs/4263514879/jobs/7420351598).
JDK and the packaged tar.gz are removed from the CI build and JNI and
SFTP builds are moved to the ubuntu build. This reduces the image size
by about 2GBs.
Closes #1513
Signed-off-by: Marton Szasz <sz...@apache.org>
---
.github/workflows/ci.yml | 22 ++++++++++++----------
cmake/DockerConfig.cmake | 13 +++++++++++++
docker/DockerBuild.sh | 7 +++++--
docker/bionic/Dockerfile | 3 ++-
docker/centos/Dockerfile | 18 +++++++++++++-----
docker/fedora/Dockerfile | 3 ++-
docker/focal/Dockerfile | 3 ++-
7 files changed, 49 insertions(+), 20 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 9aa080554..9047e87d4 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -104,7 +104,7 @@ jobs:
export CC=gcc-11
export CXX=g++-11
cmake -DUSE_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCI_BUILD=ON -DSTRICT_GSL_CHECKS=AUDIT -DFAIL_ON_WARNINGS=ON -DENABLE_SQL=OFF -DENABLE_LIBRDKAFKA=OFF -DENABLE_AWS=OFF \
- -DENABLE_AZURE=OFF -DENABLE_SPLUNK=OFF -DENABLE_GCP=OFF -DENABLE_PROCFS=OFF -DENABLE_BUSTACHE=ON -DENABLE_PCAP=ON ..
+ -DENABLE_AZURE=OFF -DENABLE_SPLUNK=OFF -DENABLE_GCP=OFF -DENABLE_PROCFS=OFF -DENABLE_BUSTACHE=ON -DENABLE_PCAP=ON -DENABLE_JNI=ON -DENABLE_SFTP=ON ..
make -j$(nproc) VERBOSE=1
- name: test
run: cd build && make test ARGS="--timeout 300 -j2 --output-on-failure"
@@ -143,8 +143,8 @@ jobs:
export CXXFLAGS="${CXXFLAGS} -stdlib=libc++"
export LDFLAGS="${LDFLAGS} -stdlib=libc++"
cmake -DUSE_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCI_BUILD=ON -DSTRICT_GSL_CHECKS=AUDIT -DFAIL_ON_WARNINGS=ON -DENABLE_AWS=ON -DENABLE_AZURE=ON -DENABLE_BUSTACHE=ON -DENABLE_COAP=ON \
- -DENABLE_ENCRYPT_CONFIG=ON -DENABLE_GPS=ON -DENABLE_JNI=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_MQTT=ON -DENABLE_NANOFI=ON -DENABLE_OPC=ON -DENABLE_OPENCV=ON \
- -DENABLE_OPENWSMAN=ON -DENABLE_OPS=ON -DENABLE_PCAP=ON -DENABLE_PYTHON=ON -DENABLE_SENSORS=ON -DENABLE_SFTP=ON -DENABLE_SQL=ON -DENABLE_SYSTEMD=ON -DENABLE_TENSORFLOW=OFF \
+ -DENABLE_ENCRYPT_CONFIG=ON -DENABLE_GPS=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_MQTT=ON -DENABLE_NANOFI=ON -DENABLE_OPC=ON -DENABLE_OPENCV=ON \
+ -DENABLE_OPENWSMAN=ON -DENABLE_OPS=ON -DENABLE_PCAP=ON -DENABLE_PYTHON=ON -DENABLE_SENSORS=ON -DENABLE_SQL=ON -DENABLE_SYSTEMD=ON -DENABLE_TENSORFLOW=OFF \
-DENABLE_USB_CAMERA=ON -DENABLE_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_GCP=ON -DENABLE_PROCFS=ON -DENABLE_PROMETHEUS=ON -DENABLE_ELASTICSEARCH=ON \
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON ..
cmake --build . --parallel $(nproc)
@@ -196,13 +196,13 @@ jobs:
- id: build
run: |
if [ -d ~/.ccache ]; then mv ~/.ccache .; fi
- mkdir build && cd build && cmake -DUSE_SHARED_LIBS=ON -DCI_BUILD=ON -DSTRICT_GSL_CHECKS=AUDIT -DFAIL_ON_WARNINGS=ON -DENABLE_AWS=ON -DENABLE_AZURE=ON -DENABLE_COAP=ON \
- -DENABLE_ENCRYPT_CONFIG=ON -DENABLE_GPS=ON -DENABLE_JNI=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_MQTT=ON -DENABLE_NANOFI=ON -DENABLE_OPC=ON -DENABLE_OPENCV=ON \
- -DENABLE_OPENWSMAN=ON -DENABLE_OPS=ON -DENABLE_PYTHON=ON -DENABLE_SENSORS=ON -DENABLE_SFTP=ON -DENABLE_SQL=ON -DENABLE_SYSTEMD=ON -DENABLE_TENSORFLOW=OFF \
- -DENABLE_USB_CAMERA=ON -DENABLE_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_GCP=ON -DENABLE_PROCFS=ON -DENABLE_PROMETHEUS=ON -DENABLE_ELASTICSEARCH=ON \
- -DDOCKER_SKIP_TESTS=OFF -DDOCKER_BUILD_ONLY=ON -DDOCKER_CCACHE_DUMP_LOCATION=$HOME/.ccache .. && make centos
+ mkdir build && cd build && cmake -DUSE_SHARED_LIBS=ON -DCI_BUILD=ON -DSTRICT_GSL_CHECKS=AUDIT -DFAIL_ON_WARNINGS=ON -DENABLE_AWS=ON -DENABLE_AZURE=ON \
+ -DENABLE_COAP=ON -DENABLE_ENCRYPT_CONFIG=ON -DENABLE_GPS=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_MQTT=ON -DENABLE_NANOFI=ON -DENABLE_OPC=ON \
+ -DENABLE_OPENCV=ON -DENABLE_OPENWSMAN=ON -DENABLE_OPS=ON -DENABLE_PYTHON=ON -DENABLE_SENSORS=ON -DENABLE_SQL=ON -DENABLE_SYSTEMD=ON \
+ -DENABLE_USB_CAMERA=ON -DENABLE_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_GCP=ON -DENABLE_PROCFS=ON -DENABLE_PROMETHEUS=ON \
+ -DENABLE_ELASTICSEARCH=ON -DDOCKER_SKIP_TESTS=OFF -DDOCKER_BUILD_ONLY=ON -DDOCKER_CCACHE_DUMP_LOCATION=$HOME/.ccache .. && make centos-test
- id: test
- run: docker run --rm apacheminificpp:$(docker images | grep apacheminificpp | grep centos | awk '{print $2}') bash -c 'cd /opt/minifi/build && make test ARGS="--timeout 300 -j8 --output-on-failure"'
+ run: docker run --rm apacheminificpp:$(docker images | grep apacheminificpp | grep centos | awk '{print $2}' | head -1) bash -c 'cd /opt/minifi/build && make test ARGS="--timeout 300 -j8 --output-on-failure"'
docker_integration_tests:
name: "Docker integration tests"
runs-on: ubuntu-20.04
@@ -223,7 +223,9 @@ jobs:
if [ -d ~/.ccache ]; then mv ~/.ccache .; fi
mkdir build
cd build
- cmake -DUSE_SHARED_LIBS= -DSTRICT_GSL_CHECKS=AUDIT -DCI_BUILD=ON -DENABLE_JNI=OFF -DDISABLE_JEMALLOC=ON -DENABLE_AWS=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_MQTT=ON -DENABLE_AZURE=ON -DENABLE_SQL=ON -DENABLE_SPLUNK=ON -DENABLE_GCP=ON -DENABLE_OPC=ON -DENABLE_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_TEST_PROCESSORS=ON -DENABLE_PROMETHEUS=ON -DDOCKER_BUILD_ONLY=ON -DDOCKER_CCACHE_DUMP_LOCATION=$HOME/.ccache ..
+ cmake -DUSE_SHARED_LIBS= -DSTRICT_GSL_CHECKS=AUDIT -DCI_BUILD=ON -DDISABLE_JEMALLOC=ON -DENABLE_AWS=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_MQTT=ON -DENABLE_AZURE=ON -DENABLE_SQL=ON \
+ -DENABLE_SPLUNK=ON -DENABLE_GCP=ON -DENABLE_OPC=ON -DENABLE_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_TEST_PROCESSORS=ON -DENABLE_PROMETHEUS=ON \
+ -DDOCKER_BUILD_ONLY=ON -DDOCKER_CCACHE_DUMP_LOCATION=$HOME/.ccache ..
make docker
- id: install_deps
run: |
diff --git a/cmake/DockerConfig.cmake b/cmake/DockerConfig.cmake
index 04b9bb92d..dc555fead 100644
--- a/cmake/DockerConfig.cmake
+++ b/cmake/DockerConfig.cmake
@@ -64,6 +64,19 @@ add_custom_target(
-c DOCKER_SKIP_TESTS=${DOCKER_SKIP_TESTS}
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/)
+add_custom_target(
+ centos-test
+ COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerBuild.sh
+ -u 1000
+ -g 1000
+ -v ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}
+ -o ${MINIFI_DOCKER_OPTIONS_STR}
+ -d centos
+ -c BUILD_NUMBER=${BUILD_NUMBER}
+ -c DOCKER_CCACHE_DUMP_LOCATION=${DOCKER_CCACHE_DUMP_LOCATION}
+ -c DOCKER_SKIP_TESTS=OFF
+ WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/)
+
add_custom_target(
fedora
COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerBuild.sh
diff --git a/docker/DockerBuild.sh b/docker/DockerBuild.sh
index 0f53f7f77..14c5efd11 100755
--- a/docker/DockerBuild.sh
+++ b/docker/DockerBuild.sh
@@ -160,10 +160,13 @@ BUILD_ARGS+=("--build-arg" "UID=${UID_ARG}"
"--build-arg" "DUMP_LOCATION=${DUMP_LOCATION}"
"--build-arg" "DISTRO_NAME=${DISTRO_NAME}"
"--build-arg" "DOCKER_SKIP_TESTS=${DOCKER_SKIP_TESTS}")
+if [ -n "${DUMP_LOCATION}" ]; then
+ BUILD_ARGS+=("--build-arg" "DOCKER_MAKE_TARGET=package")
+fi
if [ -n "${DISTRO_NAME}" ]; then
- echo DOCKER_BUILDKIT=0 docker build "${BUILD_ARGS[@]}" -f "${DOCKERFILE}" -t apacheminificpp:"${TAG}" ..
- DOCKER_BUILDKIT=0 docker build "${BUILD_ARGS[@]}" -f "${DOCKERFILE}" -t apacheminificpp:"${TAG}" ..
+ echo DOCKER_BUILDKIT=1 docker build "${BUILD_ARGS[@]}" -f "${DOCKERFILE}" -t apacheminificpp:"${TAG}" ..
+ DOCKER_BUILDKIT=1 docker build "${BUILD_ARGS[@]}" -f "${DOCKERFILE}" -t apacheminificpp:"${TAG}" ..
if [ -n "${DOCKER_CCACHE_DUMP_LOCATION}" ]; then
dump_ccache "apacheminificpp:${TAG}" "${DOCKER_CCACHE_DUMP_LOCATION}"
diff --git a/docker/bionic/Dockerfile b/docker/bionic/Dockerfile
index a715d2afa..41ac52461 100644
--- a/docker/bionic/Dockerfile
+++ b/docker/bionic/Dockerfile
@@ -21,6 +21,7 @@ FROM ubuntu:bionic AS build_deps
LABEL maintainer="Apache NiFi <de...@nifi.apache.org>"
ARG MINIFI_VERSION
+ARG DOCKER_MAKE_TARGET="all"
# Install the system dependencies needed for a build
@@ -50,4 +51,4 @@ RUN cd $MINIFI_BASE_DIR \
&& ./bootstrap.sh -t \
&& cd $MINIFI_BASE_DIR/build \
&& cmake -DSTATIC_BUILD= -DSKIP_TESTS=true ${MINIFI_OPTIONS} -DAWS_ENABLE_UNITY_BUILD=OFF -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" .. \
- && make -j "$(nproc)" package
+ && make -j "$(nproc)" ${DOCKER_MAKE_TARGET}
diff --git a/docker/centos/Dockerfile b/docker/centos/Dockerfile
index be82d8571..f0e55107b 100644
--- a/docker/centos/Dockerfile
+++ b/docker/centos/Dockerfile
@@ -29,6 +29,7 @@ ARG GID=1000
ARG MINIFI_OPTIONS=""
ARG CMAKE_BUILD_TYPE=Release
ARG DOCKER_SKIP_TESTS=ON
+ARG DOCKER_MAKE_TARGET="all"
# Install the system dependencies needed for a build
@@ -36,13 +37,20 @@ ENV MINIFI_BASE_DIR /opt/minifi
ENV MINIFI_HOME $MINIFI_BASE_DIR/nifi-minifi-cpp-$MINIFI_VERSION
ENV USER minificpp
-# gpsd-devel is in EPEL
-RUN yum -y install epel-release && yum -y install java-1.8.0-openjdk java-1.8.0-openjdk-devel python36-devel sudo git which maven make libarchive boost-devel lua-devel libusbx-devel libpng-devel \
- gpsd-devel libpcap-devel ccache ca-certificates
-
RUN mkdir -p $MINIFI_BASE_DIR
COPY . ${MINIFI_BASE_DIR}
+# gpsd-devel and ccache are in EPEL
+RUN yum -y install epel-release && yum -y install sudo git which make libarchive ccache ca-certificates && \
+ if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_GPS=ON"; then yum -y install gpsd-devel; fi && \
+ if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_JNI=ON"; then yum -y install java-1.8.0-openjdk maven; fi && \
+ if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_PCAP=ON"; then yum -y install libpcap-devel; fi && \
+ if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_USB_CAMERA=ON"; then yum -y install libpng-devel libusbx-devel; fi && \
+ if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_OPENCV=ON" || echo "$MINIFI_OPTIONS" | grep -q "ENABLE_BUSTACHE=ON"; then yum -y install boost-devel; fi && \
+ if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_SCRIPTING=ON" && echo "$MINIFI_OPTIONS" | grep -q "ENABLE_LUA_SCRIPTING=ON"; then yum -y install lua-devel; fi && \
+ if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_SCRIPTING=ON" && echo "$MINIFI_OPTIONS" | grep -q "DISABLE_PYTHON_SCRIPTING=OFF"; then yum -y install python36-devel; fi && \
+ if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_SFTP=ON" && [ "${DOCKER_SKIP_TESTS}" == "OFF" ]; then yum -y install java-1.8.0-openjdk maven; fi
+
RUN cd $MINIFI_BASE_DIR && \
./bootstrap.sh -t && \
ln -s /usr/bin/ccache /usr/lib64/ccache/c++
@@ -60,4 +68,4 @@ RUN cd $MINIFI_BASE_DIR && \
export PATH=/usr/lib64/ccache${PATH:+:${PATH}} && \
export CCACHE_DIR=${MINIFI_BASE_DIR}/.ccache && \
cmake3 -DSTATIC_BUILD= -DSKIP_TESTS=${DOCKER_SKIP_TESTS} ${MINIFI_OPTIONS} -DAWS_ENABLE_UNITY_BUILD=OFF -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" .. && \
- make -j "$(nproc)" package
+ make -j "$(nproc)" ${DOCKER_MAKE_TARGET}
diff --git a/docker/fedora/Dockerfile b/docker/fedora/Dockerfile
index b6155d034..1f34ba776 100644
--- a/docker/fedora/Dockerfile
+++ b/docker/fedora/Dockerfile
@@ -21,6 +21,7 @@ FROM fedora:34 AS build_deps
LABEL maintainer="Apache NiFi <de...@nifi.apache.org>"
ARG MINIFI_VERSION
+ARG DOCKER_MAKE_TARGET="all"
# Install the system dependencies needed for a build
@@ -51,5 +52,5 @@ RUN cd $MINIFI_BASE_DIR \
&& mkdir build \
&& cd build \
&& cmake3 -DSTATIC_BUILD= -DSKIP_TESTS=true ${MINIFI_OPTIONS} -DAWS_ENABLE_UNITY_BUILD=OFF -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" .. \
- && make -j "$(nproc)" package
+ && make -j "$(nproc)" ${DOCKER_MAKE_TARGET}
diff --git a/docker/focal/Dockerfile b/docker/focal/Dockerfile
index 77b3ef74d..564c74154 100644
--- a/docker/focal/Dockerfile
+++ b/docker/focal/Dockerfile
@@ -21,6 +21,7 @@ FROM ubuntu:focal AS build_deps
LABEL maintainer="Apache NiFi <de...@nifi.apache.org>"
ARG MINIFI_VERSION
+ARG DOCKER_MAKE_TARGET="all"
# Install the system dependencies needed for a build
@@ -50,4 +51,4 @@ RUN cd $MINIFI_BASE_DIR \
&& ./bootstrap.sh -t \
&& cd $MINIFI_BASE_DIR/build \
&& cmake -DSTATIC_BUILD= -DSKIP_TESTS=true ${MINIFI_OPTIONS} -DAWS_ENABLE_UNITY_BUILD=OFF -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" .. \
- && make -j "$(nproc)" package
+ && make -j "$(nproc)" ${DOCKER_MAKE_TARGET}