You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by sz...@apache.org on 2023/04/03 16:00:31 UTC
[nifi-minifi-cpp] 01/03: MINIFICPP-1716 Recover core dumps from CI up to 1G
This is an automated email from the ASF dual-hosted git repository.
szaszm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 31e6e6b6a9fc53ca011776aea30892a5290e1d3d
Author: Gabor Gyimesi <ga...@gmail.com>
AuthorDate: Mon Apr 3 17:43:02 2023 +0200
MINIFICPP-1716 Recover core dumps from CI up to 1G
- Keep RelWithDebInfo in clang build only
Closes #1511
Signed-off-by: Marton Szasz <sz...@apache.org>
---
.github/workflows/ci.yml | 91 ++++++++++++++++++++++++++++++++++++++++++++----
cmake/DockerConfig.cmake | 6 ++++
docker/DockerBuild.sh | 6 +++-
3 files changed, 96 insertions(+), 7 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 402871f95..185639af3 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -39,9 +39,27 @@ jobs:
# CPPFLAGS are not recognized by cmake, so we have to force them to CFLAGS and CXXFLAGS to have flex 2.6 working
./bootstrap.sh -e -t && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DCI_BUILD=ON -DCMAKE_C_FLAGS="${CPPFLAGS} ${CFLAGS}" -DCMAKE_CXX_FLAGS="${CPPFLAGS} ${CXXFLAGS}" -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_SQL=ON -DUSE_REAL_ODBC_TEST_DRIVER=ON -DENABLE_AZURE=ON -DENABLE_GCP=ON -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_RULE_MESSAGES=OFF -DSTRICT_GSL_CHECKS=AUDIT -DFAIL_ON_WARNINGS=ON .. && cmake --build . --parallel 4
- name: test
- run: cd build && make test ARGS="--timeout 300 -j4 --output-on-failure"
+ id: test
+ run: |
+ # Set core file size limit to 1GiB
+ ulimit -c 1048576
+ cd build && make test ARGS="--timeout 300 -j4 --output-on-failure"
- name: linter
run: cd build && make -j4 linter
+ - name: check-cores
+ if: ${{ failure() && steps.test.conclusion == 'failure' }}
+ run: |
+ if [ "$(ls -A /cores)" ]; then echo "CORES_EXIST=true" >> $GITHUB_ENV; fi
+ - uses: actions/upload-artifact@v3.1.2
+ if: ${{ failure() && env.CORES_EXIST == 'true' }}
+ with:
+ name: macos-coredumps
+ path: /cores
+ - uses: actions/upload-artifact@v3.1.2
+ if: ${{ failure() && env.CORES_EXIST == 'true' }}
+ with:
+ name: macos-binaries
+ path: build/bin
windows_VS2019:
name: "windows-2019"
runs-on: windows-2019
@@ -107,7 +125,25 @@ jobs:
-DENABLE_AZURE=OFF -DENABLE_SPLUNK=OFF -DENABLE_GCP=OFF -DENABLE_PROCFS=OFF -DENABLE_BUSTACHE=ON -DENABLE_PCAP=ON -DENABLE_JNI=ON -DENABLE_SFTP=ON ..
make -j$(nproc) VERBOSE=1
- name: test
- run: cd build && make test ARGS="--timeout 300 -j2 --output-on-failure"
+ id: test
+ run: |
+ # Set core file size limit to 1GiB
+ ulimit -c 1048576
+ cd build && make test ARGS="--timeout 300 -j8 --output-on-failure"
+ - name: check-cores
+ if: ${{ failure() && steps.test.conclusion == 'failure' }}
+ run: |
+ if [ "$(ls -A /var/lib/apport/coredump/)" ]; then echo "CORES_EXIST=true" >> $GITHUB_ENV; fi
+ - uses: actions/upload-artifact@v3.1.2
+ if: ${{ failure() && env.CORES_EXIST == 'true' }}
+ with:
+ name: ubuntu-coredumps
+ path: /var/lib/apport/coredump/
+ - uses: actions/upload-artifact@v3.1.2
+ if: ${{ failure() && env.CORES_EXIST == 'true' }}
+ with:
+ name: ubuntu-binaries
+ path: build/bin
ubuntu_20_04_clang:
name: "ubuntu-20.04-clang"
runs-on: ubuntu-20.04
@@ -142,14 +178,18 @@ jobs:
export CXX=clang++-14
export CXXFLAGS="${CXXFLAGS} -stdlib=libc++"
export LDFLAGS="${LDFLAGS} -stdlib=libc++"
- cmake -DUSE_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCI_BUILD=ON -DSTRICT_GSL_CHECKS=AUDIT -DFAIL_ON_WARNINGS=ON -DENABLE_AWS=ON -DENABLE_AZURE=ON -DENABLE_BUSTACHE=ON -DENABLE_COAP=ON \
+ cmake -DUSE_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCI_BUILD=ON -DSTRICT_GSL_CHECKS=AUDIT -DFAIL_ON_WARNINGS=ON -DENABLE_AWS=ON -DENABLE_AZURE=ON -DENABLE_BUSTACHE=ON -DENABLE_COAP=ON \
-DENABLE_ENCRYPT_CONFIG=ON -DENABLE_GPS=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_MQTT=ON -DENABLE_NANOFI=ON -DENABLE_OPC=ON -DENABLE_OPENCV=ON \
-DENABLE_OPENWSMAN=ON -DENABLE_OPS=ON -DENABLE_PCAP=ON -DENABLE_SENSORS=ON -DENABLE_SQL=ON -DENABLE_SYSTEMD=ON -DENABLE_TENSORFLOW=OFF \
-DENABLE_USB_CAMERA=ON -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_GCP=ON -DENABLE_PROCFS=ON -DENABLE_PROMETHEUS=ON -DENABLE_ELASTICSEARCH=ON \
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON ..
cmake --build . --parallel $(nproc)
- name: test
- run: cd build && make test ARGS="--timeout 300 -j8 --output-on-failure"
+ id: test
+ run: |
+ # Set core file size limit to 1GiB
+ ulimit -c 1048576
+ cd build && make test ARGS="--timeout 300 -j8 --output-on-failure"
- name: linter
run: cd build && make -j$(nproc) linter
- name: shellcheck
@@ -173,6 +213,20 @@ jobs:
sed -i -e 's/\/usr\/lib\/ccache\/clang-14/\/lib\/llvm-14\/bin\/clang/g' build/compile_commands.json
./run_clang_tidy.sh "${FILES}"
+ - name: check-cores
+ if: ${{ failure() && steps.test.conclusion == 'failure' }}
+ run: |
+ if [ "$(ls -A /var/lib/apport/coredump/)" ]; then echo "CORES_EXIST=true" >> $GITHUB_ENV; fi
+ - uses: actions/upload-artifact@v3.1.2
+ if: ${{ failure() && env.CORES_EXIST == 'true' }}
+ with:
+ name: ubuntu-clang-coredumps
+ path: /var/lib/apport/coredump/
+ - uses: actions/upload-artifact@v3.1.2
+ if: ${{ failure() && env.CORES_EXIST == 'true' }}
+ with:
+ name: ubuntu-clang-binaries
+ path: build/bin
centos:
name: "centos"
runs-on: ubuntu-20.04
@@ -196,13 +250,38 @@ jobs:
- id: build
run: |
if [ -d ~/.ccache ]; then mv ~/.ccache .; fi
- mkdir build && cd build && cmake -DUSE_SHARED_LIBS=ON -DCI_BUILD=ON -DSTRICT_GSL_CHECKS=AUDIT -DFAIL_ON_WARNINGS=ON -DENABLE_AWS=ON -DENABLE_AZURE=ON \
+ # centos build can run out of the github runners' disk space if built with RelWithDebInfo so we keep the Release build here
+ mkdir build && cd build && cmake -DUSE_SHARED_LIBS=ON -DCI_BUILD=ON -DCMAKE_BUILD_TYPE=Release -DSTRICT_GSL_CHECKS=AUDIT -DFAIL_ON_WARNINGS=ON -DENABLE_AWS=ON -DENABLE_AZURE=ON \
-DENABLE_COAP=ON -DENABLE_ENCRYPT_CONFIG=ON -DENABLE_GPS=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_MQTT=ON -DENABLE_NANOFI=ON -DENABLE_OPC=ON \
-DENABLE_OPENCV=ON -DENABLE_OPENWSMAN=ON -DENABLE_OPS=ON -DENABLE_SENSORS=ON -DENABLE_SQL=ON -DENABLE_SYSTEMD=ON \
-DENABLE_USB_CAMERA=ON -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_GCP=ON -DENABLE_PROCFS=ON -DENABLE_PROMETHEUS=ON \
-DENABLE_ELASTICSEARCH=ON -DDOCKER_SKIP_TESTS=OFF -DDOCKER_BUILD_ONLY=ON -DDOCKER_CCACHE_DUMP_LOCATION=$HOME/.ccache .. && make centos-test
- id: test
- run: docker run --rm apacheminificpp:$(docker images | grep apacheminificpp | grep centos | awk '{print $2}' | head -1) bash -c 'cd /opt/minifi/build && make test ARGS="--timeout 300 -j8 --output-on-failure"'
+ run: |
+ # Set core file size limit to 1GiB
+ ulimit -c 1048576
+ echo '/cores/core.%e.%p' | sudo tee /proc/sys/kernel/core_pattern
+ sudo mkdir /cores
+ sudo chmod 777 /cores
+ docker run --name centos-test --init --ulimit core=1048576 --mount type=bind,source=/cores,target=/cores apacheminificpp:$(docker images | grep apacheminificpp | grep centos | awk '{print $2}') bash -c 'cd /opt/minifi/build && make test ARGS="--timeout 300 -j8 --output-on-failure"'
+ - name: check-cores
+ if: ${{ failure() && steps.test.conclusion == 'failure' }}
+ run: |
+ if [ "$(ls -A /cores)" ]; then
+ echo "CORES_EXIST=true" >> $GITHUB_ENV;
+ docker cp centos-test:/opt/minifi/build/bin /tmp
+ sudo chmod -R 777 /cores
+ fi
+ - uses: actions/upload-artifact@v3.1.2
+ if: ${{ failure() && env.CORES_EXIST == 'true' }}
+ with:
+ name: centos-crashes
+ path: /cores
+ - uses: actions/upload-artifact@v3.1.2
+ if: ${{ failure() && env.CORES_EXIST == 'true' }}
+ with:
+ name: centos-binaries
+ path: /tmp/bin
docker_integration_tests:
name: "Docker integration tests"
runs-on: ubuntu-20.04
diff --git a/cmake/DockerConfig.cmake b/cmake/DockerConfig.cmake
index cb2a1df18..1b294b824 100644
--- a/cmake/DockerConfig.cmake
+++ b/cmake/DockerConfig.cmake
@@ -26,6 +26,7 @@ add_custom_target(
-c DOCKER_BASE_IMAGE=${DOCKER_BASE_IMAGE}
-c DOCKER_CCACHE_DUMP_LOCATION=${DOCKER_CCACHE_DUMP_LOCATION}
-c DOCKER_SKIP_TESTS=${DOCKER_SKIP_TESTS}
+ -c CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-c BUILD_NUMBER=${BUILD_NUMBER}
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/)
@@ -47,6 +48,7 @@ add_custom_target(
-c DOCKER_BASE_IMAGE=${DOCKER_BASE_IMAGE}
-c DOCKER_SKIP_TESTS=${DOCKER_SKIP_TESTS}
-c BUILD_NUMBER=${BUILD_NUMBER}
+ -c CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/)
add_custom_target(
@@ -61,6 +63,7 @@ add_custom_target(
-c BUILD_NUMBER=${BUILD_NUMBER}
-c DOCKER_CCACHE_DUMP_LOCATION=${DOCKER_CCACHE_DUMP_LOCATION}
-c DOCKER_SKIP_TESTS=${DOCKER_SKIP_TESTS}
+ -c CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/)
add_custom_target(
@@ -86,6 +89,7 @@ add_custom_target(
-l ${CMAKE_BINARY_DIR}
-d fedora
-c BUILD_NUMBER=${BUILD_NUMBER}
+ -c CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/)
add_custom_target(
@@ -98,6 +102,7 @@ add_custom_target(
-l ${CMAKE_BINARY_DIR}
-d bionic
-c BUILD_NUMBER=${BUILD_NUMBER}
+ -c CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/)
add_custom_target(
@@ -110,6 +115,7 @@ add_custom_target(
-l ${CMAKE_BINARY_DIR}
-d focal
-c BUILD_NUMBER=${BUILD_NUMBER}
+ -c CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/)
if (EXISTS ${CMAKE_SOURCE_DIR}/docker/test/integration/features)
diff --git a/docker/DockerBuild.sh b/docker/DockerBuild.sh
index 14c5efd11..4ac0c4050 100755
--- a/docker/DockerBuild.sh
+++ b/docker/DockerBuild.sh
@@ -30,6 +30,7 @@ DISTRO_NAME=
BUILD_NUMBER=
DOCKER_CCACHE_DUMP_LOCATION=
DOCKER_SKIP_TESTS=ON
+CMAKE_BUILD_TYPE=Release
function usage {
echo "Usage: ./DockerBuild.sh -v <MINIFI_VERSION> [additional options]"
@@ -100,6 +101,8 @@ while [[ $# -gt 0 ]]; do
DOCKER_CCACHE_DUMP_LOCATION="${ARR[1]}"
elif [ "${ARR[0]}" == "DOCKER_SKIP_TESTS" ]; then
DOCKER_SKIP_TESTS="${ARR[1]}"
+ elif [ "${ARR[0]}" == "CMAKE_BUILD_TYPE" ]; then
+ CMAKE_BUILD_TYPE="${ARR[1]}"
else
BUILD_ARGS+=("--build-arg" "${ARR[0]}=${ARR[1]}")
fi
@@ -159,7 +162,8 @@ BUILD_ARGS+=("--build-arg" "UID=${UID_ARG}"
"--build-arg" "MINIFI_VERSION=${MINIFI_VERSION}"
"--build-arg" "DUMP_LOCATION=${DUMP_LOCATION}"
"--build-arg" "DISTRO_NAME=${DISTRO_NAME}"
- "--build-arg" "DOCKER_SKIP_TESTS=${DOCKER_SKIP_TESTS}")
+ "--build-arg" "DOCKER_SKIP_TESTS=${DOCKER_SKIP_TESTS}"
+ "--build-arg" "CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}")
if [ -n "${DUMP_LOCATION}" ]; then
BUILD_ARGS+=("--build-arg" "DOCKER_MAKE_TARGET=package")
fi