You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by fg...@apache.org on 2023/06/13 12:14:12 UTC
[nifi-minifi-cpp] 01/04: MINIFICPP-1641 Parallelization of docker tests
This is an automated email from the ASF dual-hosted git repository.
fgerlits pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit f4878bb707b51e8ee365d6ce4f3d67eea6b11f18
Author: Martin Zink <ma...@apache.org>
AuthorDate: Thu May 4 09:39:53 2023 +0200
MINIFICPP-1641 Parallelization of docker tests
Signed-off-by: Ferenc Gerlits <fg...@gmail.com>
This closes #1574
---
.github/workflows/ci.yml | 110 ++--------
.github/workflows/verify-python-compatibility.yml | 128 ++++--------
CMakeLists.txt | 2 +-
CMakeSettings.json | 4 +-
Windows.md | 2 +-
cmake/DockerConfig.cmake | 57 +++---
cmake/MiNiFiOptions.cmake | 3 +-
cmake/VerifyPythonCompatibility.cmake | 29 +--
docker/DockerVerify.sh | 57 +++---
docker/requirements.txt | 4 +-
docker/test/integration/cluster/ContainerStore.py | 224 +++++++++++++++++----
.../test/integration/cluster/DockerCommunicator.py | 5 +-
.../test/integration/cluster/DockerTestCluster.py | 32 ++-
.../cluster/DockerTestDirectoryBindings.py | 93 +++++----
.../FeatureContext.py} | 11 +-
docker/test/integration/cluster/ImageStore.py | 41 ----
.../cluster/checkers/ElasticSearchChecker.py | 8 +-
.../containers/AzureStorageServerContainer.py | 4 +-
.../integration/cluster/containers/Container.py | 4 +-
.../cluster/containers/ElasticsearchContainer.py | 68 ++++++-
.../cluster/containers/FakeGcsServerContainer.py | 7 +-
.../cluster/containers/FlowContainer.py | 10 +-
.../cluster/containers/HttpProxyContainer.py | 5 +-
.../cluster/containers/KafkaBrokerContainer.py | 60 +++++-
.../containers/MinifiAsPodInKubernetesCluster.py | 11 +-
.../cluster/containers/MinifiC2ServerContainer.py | 72 ++++++-
.../cluster/containers/MinifiContainer.py | 36 ++--
.../cluster/containers/MqttBrokerContainer.py | 5 +-
.../cluster/containers/NifiContainer.py | 4 +-
.../cluster/containers/OPCUAServerContainer.py | 5 +-
.../cluster/containers/OpensearchContainer.py | 50 ++++-
.../containers/PostgreSQLServerContainer.py | 6 +-
.../cluster/containers/PrometheusContainer.py | 29 ++-
.../cluster/containers/S3ServerContainer.py | 5 +-
.../cluster/containers/SplunkContainer.py | 4 +-
.../cluster/containers/SyslogTcpClientContainer.py | 7 +-
.../cluster/containers/SyslogUdpClientContainer.py | 7 +-
.../cluster/containers/TcpClientContainer.py | 10 +-
.../cluster/containers/ZookeeperContainer.py | 8 +-
docker/test/integration/features/CMakeLists.txt | 117 -----------
.../MiNiFi_integration_test_driver.py | 67 ++++--
docker/test/integration/{ => features}/README.md | 0
.../features/attributes_to_json.feature | 1 +
.../integration/features/azure_storage.feature | 1 +
.../features/core_functionality.feature | 8 +-
.../features/defragtextflowfiles.feature | 1 +
.../integration/features/elasticsearch.feature | 1 +
.../test/integration/{ => features}/environment.py | 61 +++---
.../features/file_system_operations.feature | 16 ++
.../features/google_cloud_storage.feature | 1 +
.../test/integration/features/hashcontent.feature | 1 +
docker/test/integration/features/http.feature | 25 +--
docker/test/integration/features/https.feature | 3 +-
docker/test/integration/features/kafka.feature | 97 +++++----
.../features/kubernetes_logging.feature | 1 +
.../features/kubernetes_metrics.feature | 1 +
.../test/integration/features/lua_script.feature | 1 +
.../integration/features/minifi_c2_server.feature | 1 +
.../integration/features/minifi_controller.feature | 1 +
docker/test/integration/features/mqtt.feature | 1 +
.../integration/features/network_listener.feature | 1 +
docker/test/integration/features/opcua.feature | 133 ++++++------
.../test/integration/features/opensearch.feature | 7 +-
.../test/integration/features/prometheus.feature | 1 +
docker/test/integration/features/python.feature | 1 +
.../integration/features/python_script.feature | 1 +
.../features/python_with_modules.feature | 2 +
.../test/integration/features/replace_text.feature | 1 +
docker/test/integration/features/routetext.feature | 1 +
docker/test/integration/features/s2s.feature | 7 +-
docker/test/integration/features/s3.feature | 49 ++---
docker/test/integration/features/splunk.feature | 9 +-
docker/test/integration/features/sql.feature | 1 +
.../test/integration/{ => features}/steps/steps.py | 193 ++++++++++--------
.../integration/features/syslog_listener.feature | 1 +
docker/test/integration/{ => features}/utils.py | 0
docker/test/integration/minifi/core/Processor.py | 4 +
.../minifi/processors/AddPythonAttribute.py | 4 +-
.../minifi/processors/AttributesToJSON.py | 4 +-
.../processors/AzureBlobStorageProcessorBase.py | 8 +-
.../processors/CollectKubernetesPodMetrics.py | 5 +-
.../integration/minifi/processors/ConsumeKafka.py | 7 +-
.../integration/minifi/processors/ConsumeMQTT.py | 7 +-
.../minifi/processors/CountingProcessor.py | 6 +-
.../minifi/processors/DefragmentText.py | 5 +-
.../minifi/processors/DeleteAzureBlobStorage.py | 5 +-
.../minifi/processors/DeleteGCSObject.py | 8 +-
.../minifi/processors/DeleteS3Object.py | 6 +-
.../minifi/processors/ExecutePythonProcessor.py | 5 +-
.../integration/minifi/processors/ExecuteSQL.py | 5 +-
.../integration/minifi/processors/ExecuteScript.py | 5 +-
.../minifi/processors/FetchAzureBlobStorage.py | 5 +-
.../integration/minifi/processors/FetchFile.py | 5 +-
.../minifi/processors/FetchGCSObject.py | 8 +-
.../minifi/processors/FetchOPCProcessor.py | 5 +-
.../integration/minifi/processors/FetchS3Object.py | 6 +-
.../processors/GaussianDistributionWithNumpy.py | 6 +-
.../minifi/processors/GenerateFlowFile.py | 5 +-
.../test/integration/minifi/processors/GetFile.py | 5 +-
.../integration/minifi/processors/HashContent.py | 5 +-
.../integration/minifi/processors/InvokeHTTP.py | 4 +-
.../minifi/processors/ListAzureBlobStorage.py | 9 +-
.../test/integration/minifi/processors/ListFile.py | 5 +-
.../integration/minifi/processors/ListGCSBucket.py | 8 +-
.../test/integration/minifi/processors/ListS3.py | 29 +--
.../integration/minifi/processors/ListenHTTP.py | 5 +-
.../integration/minifi/processors/ListenSyslog.py | 5 +-
.../integration/minifi/processors/ListenTCP.py | 5 +-
.../integration/minifi/processors/LogAttribute.py | 5 +-
.../minifi/processors/LogOnDestructionProcessor.py | 5 +-
.../minifi/processors/MoveContentToJson.py | 6 +-
.../minifi/processors/PostElasticsearch.py | 7 +-
.../integration/minifi/processors/PublishKafka.py | 7 +-
.../integration/minifi/processors/PublishMQTT.py | 7 +-
.../minifi/processors/PutAzureBlobStorage.py | 7 +-
.../test/integration/minifi/processors/PutFile.py | 5 +-
.../integration/minifi/processors/PutGCSObject.py | 8 +-
.../minifi/processors/PutOPCProcessor.py | 5 +-
.../integration/minifi/processors/PutS3Object.py | 6 +-
.../test/integration/minifi/processors/PutSQL.py | 5 +-
.../integration/minifi/processors/PutSplunkHTTP.py | 5 +-
.../minifi/processors/QueryDatabaseTable.py | 5 +-
.../minifi/processors/QuerySplunkIndexingStatus.py | 6 +-
.../minifi/processors/RemoveFlowFile.py | 6 +-
.../integration/minifi/processors/ReplaceText.py | 5 +-
.../minifi/processors/RouteOnAttribute.py | 5 +-
.../integration/minifi/processors/RouteText.py | 5 +-
.../test/integration/minifi/processors/TailFile.py | 5 +-
.../minifi/processors/UpdateAttribute.py | 5 +-
.../integration/resources/elasticsearch/Dockerfile | 7 -
.../resources/elasticsearch/certs/elastic_http.crt | 19 --
.../resources/elasticsearch/certs/elastic_http.key | 28 ---
.../elasticsearch/certs/elastic_transport.crt | 19 --
.../elasticsearch/certs/elastic_transport.key | 28 ---
.../elasticsearch/certs/minifi_client.crt | 19 --
.../elasticsearch/certs/minifi_client.key | 28 ---
.../resources/elasticsearch/certs/root_ca.crt | 19 --
.../resources/kafka_broker/conf/certs/ca-cert | 21 --
.../resources/kafka_broker/conf/certs/ca-cert.key | 30 ---
.../resources/kafka_broker/conf/certs/ca-cert.srl | 1 -
.../kafka_broker/conf/certs/client_cert.key | 28 ---
.../kafka_broker/conf/certs/client_cert.pem | 19 --
.../kafka_broker/conf/certs/server_cert.key | 28 ---
.../kafka_broker/conf/certs/server_cert.pem | 19 --
.../kafka_broker/conf/certs/server_keystore.jks | Bin 2643 -> 0 bytes
.../kafka_broker/conf/certs/server_truststore.jks | Bin 2135 -> 0 bytes
.../resources/kafka_broker/conf/server.properties | 5 +-
.../resources/minifi-c2-server-ssl/Dockerfile | 7 -
.../minifi-c2-server-ssl/authorities.yaml | 2 -
.../resources/minifi-c2-server-ssl/c2.properties | 10 -
.../certs/minifi-c2-server-keystore.p12 | Bin 2544 -> 0 bytes
.../certs/minifi-c2-server-truststore.p12 | Bin 2978 -> 0 bytes
.../certs/minifi-c2-server.crt | 20 --
.../certs/minifi-c2-server.key | 28 ---
.../minifi-c2-server-ssl/certs/minifi-cpp-flow.crt | 20 --
.../minifi-c2-server-ssl/certs/minifi-cpp-flow.key | 28 ---
.../minifi-c2-server-ssl/certs/root-ca.key | 30 ---
.../minifi-c2-server-ssl/certs/root-ca.pem | 19 --
.../resources/minifi-c2-server/Dockerfile | 2 -
.../authorizations.yaml | 0
.../resources/minifi-c2-server/c2.properties | 10 +
.../config.yml => minifi-c2-server/config-ssl.yml} | 6 +-
.../integration/resources/opensearch/Dockerfile | 5 -
.../resources/opensearch/certs/admin-key.pem | 28 ---
.../resources/opensearch/certs/admin.pem | 19 --
.../resources/opensearch/certs/root-ca.pem | 21 --
.../integration/resources/prometheus/Dockerfile | 2 -
.../resources/prometheus/conf/prometheus.yml | 7 -
.../test/integration/ssl_utils/SSL_cert_utils.py | 27 ++-
win_build_vs.bat | 2 +-
170 files changed, 1370 insertions(+), 1606 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d87303085..11b9c618d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1,7 +1,7 @@
name: "MiNiFi-CPP CI"
on: [push, pull_request, workflow_dispatch]
env:
- DOCKER_CMAKE_FLAGS: -DUSE_SHARED_LIBS= -DSTRICT_GSL_CHECKS=AUDIT -DCI_BUILD=ON -DDISABLE_JEMALLOC=ON -DENABLE_AWS=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_MQTT=ON -DENABLE_AZURE=ON -DENABLE_SQL=ON \
+ DOCKER_CMAKE_FLAGS: -DDOCKER_VERIFY_THREAD=3 -DUSE_SHARED_LIBS= -DSTRICT_GSL_CHECKS=AUDIT -DCI_BUILD=ON -DDISABLE_JEMALLOC=ON -DENABLE_AWS=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_MQTT=ON -DENABLE_AZURE=ON -DENABLE_SQL=ON \
-DENABLE_SPLUNK=ON -DENABLE_GCP=ON -DENABLE_OPC=ON -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_TEST_PROCESSORS=ON -DENABLE_PROMETHEUS=ON \
-DENABLE_ELASTICSEARCH=OFF -DDOCKER_BUILD_ONLY=ON -DDOCKER_CCACHE_DUMP_LOCATION=$HOME/.ccache
SCCACHE_GHA_ENABLE: true
@@ -329,7 +329,7 @@ jobs:
name: minifi_docker
path: build/minifi_docker.tar
docker_tests_q1:
- name: "Docker integration tests 1/4"
+ name: "Docker integration tests"
needs: docker_build
runs-on: ubuntu-20.04
timeout-minutes: 180
@@ -357,94 +357,22 @@ jobs:
sudo apt install -y python3-virtualenv
- id: test
name: Docker Verify
- run: cd build && make docker-verify-q1
- docker_tests_q2:
- name: "Docker integration tests 2/4"
- needs: docker_build
- runs-on: ubuntu-20.04
- timeout-minutes: 180
- steps:
- - id: checkout
- uses: actions/checkout@v3
- - id: run_cmake
- name: Run CMake
- run: |
- mkdir build
- cd build
- cmake ${DOCKER_CMAKE_FLAGS} ..
- - name: Download artifact
- uses: actions/download-artifact@v3
- with:
- name: minifi_docker
- path: build
- - name: Load Docker image
- run: |
- docker load --input ./build/minifi_docker.tar
- - id: install_deps
- name: Install dependencies for Docker Verify
- run: |
- sudo apt update
- sudo apt install -y python3-virtualenv
- - id: test
- name: Docker Verify
- run: cd build && make docker-verify-q2
- docker_tests_q3:
- name: "Docker integration tests 3/4"
- needs: docker_build
- runs-on: ubuntu-20.04
- timeout-minutes: 180
- steps:
- - id: checkout
- uses: actions/checkout@v3
- - id: run_cmake
- name: Run CMake
- run: |
- mkdir build
- cd build
- cmake ${DOCKER_CMAKE_FLAGS} ..
- - name: Download artifact
- uses: actions/download-artifact@v3
+ working-directory: ./build
+ run: make docker-verify
+ - name: Test Reporter
+ if: always()
+ uses: phoenix-actions/test-reporting@v12
with:
- name: minifi_docker
- path: build
- - name: Load Docker image
- run: |
- docker load --input ./build/minifi_docker.tar
- - id: install_deps
- name: Install dependencies for Docker Verify
- run: |
- sudo apt update
- sudo apt install -y python3-virtualenv
- - id: test
- name: Docker Verify
- run: cd build && make docker-verify-q3
- docker_tests_q4:
- name: "Docker integration tests 4/4"
- needs: docker_build
- runs-on: ubuntu-20.04
- timeout-minutes: 180
- steps:
- - id: checkout
- uses: actions/checkout@v3
- - id: run_cmake
- name: Run CMake
- run: |
- mkdir build
- cd build
- cmake ${DOCKER_CMAKE_FLAGS} ..
- - name: Download artifact
- uses: actions/download-artifact@v3
+ name: Docker integration tests
+ path: build/behavex_output/behave/*.xml
+ reporter: java-junit
+ output-to: 'step-summary'
+ only-summary: 'true'
+ list-tests: 'failed'
+ list-suites: 'failed'
+ - name: Upload artifact
+ if: failure()
+ uses: actions/upload-artifact@v3
with:
- name: minifi_docker
- path: build
- - name: Load Docker image
- run: |
- docker load --input ./build/minifi_docker.tar
- - id: install_deps
- name: Install dependencies for Docker Verify
- run: |
- sudo apt update
- sudo apt install -y python3-virtualenv
- - id: test
- name: Docker Verify
- run: cd build && make docker-verify-q4
+ name: behavex_output
+ path: build/behavex_output
diff --git a/.github/workflows/verify-python-compatibility.yml b/.github/workflows/verify-python-compatibility.yml
index 6cd411532..263b77989 100644
--- a/.github/workflows/verify-python-compatibility.yml
+++ b/.github/workflows/verify-python-compatibility.yml
@@ -30,11 +30,35 @@ jobs:
name: minifi-tar
path: build/nifi-minifi-cpp-*-bin-centos.tar.gz
if-no-files-found: error
- rocky-linux8:
- name: "Python tests on Rocky Linux 8"
+
+ docker-tests:
+ name: "${{ matrix.platform.name }}"
runs-on: ubuntu-20.04
timeout-minutes: 180
needs: centos-build
+ strategy:
+ matrix:
+ platform: [
+ { name: "Rocky Linux 8",
+ build_cmd: "make rocky8_docker_from_centos_build",
+ verify_cmd: "make docker-verify-python-rocky8" },
+ { name: "Rocky Linux 9",
+ build_cmd: "make rocky9_docker_from_centos_build",
+ verify_cmd: "make docker-verify-python-rocky9" },
+ { name: "Patchelf-ed Ubuntu 22.04",
+ build_cmd: "make patched_jammy_docker_from_centos_build",
+ verify_cmd: "make docker-verify-python-patched_jammy" },
+ { name: "Patchelf-ed Debian 11",
+ build_cmd: "make patched_bullseye_docker_from_centos_build",
+ verify_cmd: "make docker-verify-python-patched_bullseye" },
+ { name: "Ubuntu 22.04 with Anaconda",
+ build_cmd: "make jammy_docker_from_centos_build && make conda_jammy_docker_from_centos_build",
+ verify_cmd: "make docker-verify-python-conda_jammy" },
+ { name: "Rocky Linux 9 in virtualenv",
+ build_cmd: "make rocky9_docker_from_centos_build && make venv_rocky9_docker_from_centos_build",
+ verify_cmd: "make docker-verify-python-venv_rocky9" }
+ ]
+
steps:
- id: checkout
uses: actions/checkout@v3
@@ -47,94 +71,14 @@ jobs:
sudo apt update
sudo apt install -y python3-virtualenv
- run: cd build && cmake -DENABLE_PYTHON_SCRIPTING=ON -DDOCKER_BUILD_ONLY=ON .. \
- && make rocky8_docker_from_centos_build && make docker-verify-python-rocky8
- rocky-linux9:
- name: "Python tests on Rocky Linux 9"
- runs-on: ubuntu-20.04
- timeout-minutes: 180
- needs: centos-build
- steps:
- - id: checkout
- uses: actions/checkout@v3
- - uses: actions/download-artifact@v3
- with:
- name: minifi-tar
- path: build
- - id: install_deps
- run: |
- sudo apt update
- sudo apt install -y python3-virtualenv
- - run: cd build && cmake -DENABLE_PYTHON_SCRIPTING=ON -DDOCKER_BUILD_ONLY=ON .. \
- && make rocky9_docker_from_centos_build && make docker-verify-python-rocky9
- patched-jammy:
- name: "Python tests on patchelf-ed Ubuntu 22.04"
- runs-on: ubuntu-20.04
- timeout-minutes: 180
- needs: centos-build
- steps:
- - id: checkout
- uses: actions/checkout@v3
- - uses: actions/download-artifact@v3
- with:
- name: minifi-tar
- path: build
- - id: install_deps
- run: |
- sudo apt update
- sudo apt install -y python3-virtualenv
- - run: cd build && cmake -DENABLE_PYTHON_SCRIPTING=ON -DDOCKER_BUILD_ONLY=ON .. \
- && make patched_jammy_docker_from_centos_build && make docker-verify-python-patched_jammy
- patched-bullseye:
- name: "Python tests on patchelf-ed Debian Bullseye"
- runs-on: ubuntu-20.04
- timeout-minutes: 180
- needs: centos-build
- steps:
- - id: checkout
- uses: actions/checkout@v3
- - uses: actions/download-artifact@v3
+ && ${{ matrix.platform.build_cmd }} && ${{ matrix.platform.verify_cmd }}
+ - name: Test Reporter
+ if: always()
+ uses: phoenix-actions/test-reporting@v12
with:
- name: minifi-tar
- path: build
- - id: install_deps
- run: |
- sudo apt update
- sudo apt install -y python3-virtualenv
- - run: cd build && cmake -DENABLE_PYTHON_SCRIPTING=ON -DDOCKER_BUILD_ONLY=ON .. \
- && make patched_bullseye_docker_from_centos_build && make docker-verify-python-patched_bullseye
- conda_jammy:
- name: "Python tests on Ubuntu 22.04 with anaconda"
- runs-on: ubuntu-20.04
- timeout-minutes: 180
- needs: centos-build
- steps:
- - id: checkout
- uses: actions/checkout@v3
- - uses: actions/download-artifact@v3
- with:
- name: minifi-tar
- path: build
- - id: install_deps
- run: |
- sudo apt update
- sudo apt install -y python3-virtualenv
- - run: cd build && cmake -DENABLE_PYTHON_SCRIPTING=ON -DDOCKER_BUILD_ONLY=ON .. \
- && make jammy_docker_from_centos_build && make conda_jammy_docker_from_centos_build && make docker-verify-python-conda_jammy
- venv_rock9:
- name: "Python tests on Rocky Linux 9 with virtual environment"
- runs-on: ubuntu-20.04
- timeout-minutes: 180
- needs: centos-build
- steps:
- - id: checkout
- uses: actions/checkout@v3
- - uses: actions/download-artifact@v3
- with:
- name: minifi-tar
- path: build
- - id: install_deps
- run: |
- sudo apt update
- sudo apt install -y python3-virtualenv
- - run: cd build && cmake -DENABLE_PYTHON_SCRIPTING=ON -DDOCKER_BUILD_ONLY=ON .. \
- && make rocky9_docker_from_centos_build && make venv_rocky9_docker_from_centos_build && make docker-verify-python-venv_rocky9
+ name: "${{ matrix.platform.name }}"
+ path: build/behavex_output/behave/*.xml
+ reporter: java-junit
+ output-to: 'step-summary'
+ list-tests: 'failed'
+ list-suites: 'failed'
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2e76bc7f7..e9976e3dd 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -378,7 +378,7 @@ if (ENABLE_ENCRYPT_CONFIG)
add_subdirectory(encrypt-config)
endif()
-if (NOT DISABLE_CURL AND NOT DISABLE_CONTROLLER)
+if (NOT DISABLE_CURL AND ENABLE_CONTROLLER)
add_subdirectory(controller)
endif()
diff --git a/CMakeSettings.json b/CMakeSettings.json
index 7e4456ad1..68c3cb4a8 100644
--- a/CMakeSettings.json
+++ b/CMakeSettings.json
@@ -48,8 +48,8 @@
"value": "OFF"
},
{
- "name": "DISABLE_CONTROLLER",
- "value": "ON"
+ "name": "ENABLE_CONTROLLER",
+ "value": "OFF"
},
{
"name": "BUILD_ROCKSDB",
diff --git a/Windows.md b/Windows.md
index 67f73b2e6..f215cc083 100644
--- a/Windows.md
+++ b/Windows.md
@@ -113,7 +113,7 @@ A basic working CMake configuration can be inferred from the `win_build_vs.bat`.
```
mkdir build
cd build
-cmake -G "Visual Studio 16 2019" -A x64 -DINSTALLER_MERGE_MODULES=OFF -DTEST_CUSTOM_WEL_PROVIDER=OFF -DENABLE_SQL=OFF -DUSE_REAL_ODBC_TEST_DRIVER=OFF -DCMAKE_BUILD_TYPE_INIT=Release -DCMAKE_BUILD_TYPE=Release -DWIN32=WIN32 -DENABLE_LIBRDKAFKA=OFF -DENABLE_JNI=OFF -DOPENSSL_OFF=OFF -DENABLE_COAP=OFF -DENABLE_AWS=OFF -DENABLE_PDH= -DENABLE_AZURE=OFF -DENABLE_SFTP=OFF -DENABLE_SPLUNK= -DENABLE_GCP= -DENABLE_NANOFI=OFF -DENABLE_OPENCV=OFF -DENABLE_PROMETHEUS=OFF -DENABLE_ELASTICSEARCH= -DUSE [...]
+cmake -G "Visual Studio 16 2019" -A x64 -DINSTALLER_MERGE_MODULES=OFF -DTEST_CUSTOM_WEL_PROVIDER=OFF -DENABLE_SQL=OFF -DUSE_REAL_ODBC_TEST_DRIVER=OFF -DCMAKE_BUILD_TYPE_INIT=Release -DCMAKE_BUILD_TYPE=Release -DWIN32=WIN32 -DENABLE_LIBRDKAFKA=OFF -DENABLE_JNI=OFF -DOPENSSL_OFF=OFF -DENABLE_COAP=OFF -DENABLE_AWS=OFF -DENABLE_PDH= -DENABLE_AZURE=OFF -DENABLE_SFTP=OFF -DENABLE_SPLUNK= -DENABLE_GCP= -DENABLE_NANOFI=OFF -DENABLE_OPENCV=OFF -DENABLE_PROMETHEUS=OFF -DENABLE_ELASTICSEARCH= -DUSE [...]
msbuild /m nifi-minifi-cpp.sln /property:Configuration=Release /property:Platform=x64
copy minifi_main\Release\minifi.exe minifi_main\
cpack
diff --git a/cmake/DockerConfig.cmake b/cmake/DockerConfig.cmake
index c70539106..7137cbcac 100644
--- a/cmake/DockerConfig.cmake
+++ b/cmake/DockerConfig.cmake
@@ -15,13 +15,18 @@
# specific language governing permissions and limitations
# under the License.
+set(PROJECT_VERSION_STR ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH})
+include(ProcessorCount)
+ProcessorCount(PROCESSOR_COUNT)
+set(DOCKER_VERIFY_THREADS "${PROCESSOR_COUNT}" CACHE STRING "Number of threads that docker-verify can utilize")
+
# Create a custom build target called "docker" that will invoke DockerBuild.sh and create the NiFi-MiNiFi-CPP Docker image
add_custom_target(
docker
COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerBuild.sh
-u 1000
-g 1000
- -v ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}
+ -v ${PROJECT_VERSION_STR}
-o ${MINIFI_DOCKER_OPTIONS_STR}
-c DOCKER_BASE_IMAGE=${DOCKER_BASE_IMAGE}
-c DOCKER_CCACHE_DUMP_LOCATION=${DOCKER_CCACHE_DUMP_LOCATION}
@@ -40,11 +45,11 @@ add_custom_target(
-u 1000
-g 1000
-p minimal
- -v ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}
+ -v ${PROJECT_VERSION_STR}
-o \"-DENABLE_LIBRDKAFKA=ON
-DENABLE_AWS=ON
-DENABLE_AZURE=ON
- -DDISABLE_CONTROLLER=ON
+ -DENABLE_CONTROLLER=OFF
-DENABLE_PROMETHEUS=ON
-DENABLE_MQTT=OFF
-DENABLE_ELASTICSEARCH=OFF
@@ -66,7 +71,7 @@ add_custom_target(
COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerBuild.sh
-u 1000
-g 1000
- -v ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}
+ -v ${PROJECT_VERSION_STR}
-o ${MINIFI_DOCKER_OPTIONS_STR}
-l ${CMAKE_BINARY_DIR}
-d centos
@@ -84,7 +89,7 @@ add_custom_target(
COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerBuild.sh
-u 1000
-g 1000
- -v ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}
+ -v ${PROJECT_VERSION_STR}
-o ${MINIFI_DOCKER_OPTIONS_STR}
-d centos
-c BUILD_NUMBER=${BUILD_NUMBER}
@@ -100,7 +105,7 @@ add_custom_target(
COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerBuild.sh
-u 1000
-g 1000
- -v ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}
+ -v ${PROJECT_VERSION_STR}
-o ${MINIFI_DOCKER_OPTIONS_STR}
-l ${CMAKE_BINARY_DIR}
-d fedora
@@ -116,7 +121,7 @@ add_custom_target(
COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerBuild.sh
-u 1000
-g 1000
- -v ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}
+ -v ${PROJECT_VERSION_STR}
-o ${MINIFI_DOCKER_OPTIONS_STR}
-l ${CMAKE_BINARY_DIR}
-d bionic
@@ -132,7 +137,7 @@ add_custom_target(
COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerBuild.sh
-u 1000
-g 1000
- -v ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}
+ -v ${PROJECT_VERSION_STR}
-o ${MINIFI_DOCKER_OPTIONS_STR}
-l ${CMAKE_BINARY_DIR}
-d focal
@@ -148,7 +153,7 @@ add_custom_target(
COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerBuild.sh
-u 1000
-g 1000
- -v ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}
+ -v ${PROJECT_VERSION_STR}
-o ${MINIFI_DOCKER_OPTIONS_STR}
-l ${CMAKE_BINARY_DIR}
-d rockylinux
@@ -162,31 +167,21 @@ add_custom_target(
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/)
if (EXISTS ${CMAKE_SOURCE_DIR}/docker/test/integration/features)
- add_subdirectory(${CMAKE_SOURCE_DIR}/docker/test/integration/features)
-
- add_custom_target(
- docker-verify-all
- COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerVerify.sh --enable_test_processors ON ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH} ${ALL_BEHAVE_TESTS})
+ set(ENABLED_TAGS "CORE")
+ foreach(MINIFI_OPTION ${MINIFI_OPTIONS})
+ string(FIND ${MINIFI_OPTION} "ENABLE" my_index)
+ if(my_index EQUAL -1)
+ continue()
+ elseif(${${MINIFI_OPTION}})
+ set(ENABLED_TAGS "${ENABLED_TAGS},${MINIFI_OPTION}")
+ endif()
+ endforeach()
+
+ set(DISABLED_TAGS "SKIP_CI,NEEDS_NUMPY")
add_custom_target(
docker-verify
- COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerVerify.sh --enable_test_processors ${ENABLE_TEST_PROCESSORS} ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH} ${ENABLED_BEHAVE_TESTS})
-
- add_custom_target(
- docker-verify-q1
- COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerVerify.sh --enable_test_processors ${ENABLE_TEST_PROCESSORS} ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH} ${ENABLED_BEHAVE_TESTS_FIRST_QUADRANT})
-
- add_custom_target(
- docker-verify-q2
- COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerVerify.sh --enable_test_processors ${ENABLE_TEST_PROCESSORS} ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH} ${ENABLED_BEHAVE_TESTS_SECOND_QUADRANT})
-
- add_custom_target(
- docker-verify-q3
- COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerVerify.sh --enable_test_processors ${ENABLE_TEST_PROCESSORS} ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH} ${ENABLED_BEHAVE_TESTS_THIRD_QUADRANT})
-
- add_custom_target(
- docker-verify-q4
- COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerVerify.sh --enable_test_processors ${ENABLE_TEST_PROCESSORS} ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH} ${ENABLED_BEHAVE_TESTS_LAST_QUADRANT})
+ COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerVerify.sh ${PROJECT_VERSION_STR} ${ENABLED_TAGS} --tags_to_exclude=${DISABLED_TAGS} --parallel_processes=${DOCKER_VERIFY_THREADS})
endif()
include(VerifyPythonCompatibility)
diff --git a/cmake/MiNiFiOptions.cmake b/cmake/MiNiFiOptions.cmake
index 9600f34b3..33dd6154a 100644
--- a/cmake/MiNiFiOptions.cmake
+++ b/cmake/MiNiFiOptions.cmake
@@ -121,8 +121,7 @@ add_minifi_option(ENABLE_KUBERNETES "Enables the Kubernetes extensions." ON)
add_minifi_option(ENABLE_TEST_PROCESSORS "Enables test processors" OFF)
add_minifi_option(ENABLE_PROMETHEUS "Enables Prometheus support." ON)
add_minifi_option(DISABLE_JEMALLOC "Disables jemalloc." OFF)
-add_minifi_option(DISABLE_CURL "Disables curl." OFF)
-add_minifi_option(DISABLE_CONTROLLER "Disables build of MiNiFi controller binary." OFF)
+add_minifi_dependent_option(ENABLE_CONTROLLER "Enables the build of MiNiFi controller binary." ON "NOT DISABLE_CURL" OFF)
set_minifi_cache_variable(CUSTOM_MALLOC OFF "Overwrite malloc implementation.")
set_property(CACHE CUSTOM_MALLOC PROPERTY STRINGS "jemalloc" "mimalloc" "rpmalloc" OFF)
diff --git a/cmake/VerifyPythonCompatibility.cmake b/cmake/VerifyPythonCompatibility.cmake
index 02e7f31e1..16667e0fc 100644
--- a/cmake/VerifyPythonCompatibility.cmake
+++ b/cmake/VerifyPythonCompatibility.cmake
@@ -34,10 +34,16 @@ function(ADD_DOCKER_TARGET_FROM_CENTOS BASE_IMAGE TAG_PREFIX INSTALL_PACKAGE_CMD
${CMAKE_BINARY_DIR})
endfunction()
-function(ADD_DOCKER_VERIFY TAG_PREFIX)
- add_custom_target(
- docker-verify-python-${TAG_PREFIX}
- COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerVerify.sh --image-tag-prefix ${TAG_PREFIX} ${MINIFI_VERSION_STR} ${PYTHON_TESTS})
+function(ADD_DOCKER_VERIFY_PYTHON TAG_PREFIX HAS_MODULES)
+ if (HAS_MODULES)
+ add_custom_target(
+ docker-verify-python-${TAG_PREFIX}
+ COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerVerify.sh --image-tag-prefix ${TAG_PREFIX} ${MINIFI_VERSION_STR} ENABLE_PYTHON_SCRIPTING)
+ else()
+ add_custom_target(
+ docker-verify-python-${TAG_PREFIX}
+ COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerVerify.sh --image-tag-prefix ${TAG_PREFIX} ${MINIFI_VERSION_STR} ENABLE_PYTHON_SCRIPTING --tags_to_exclude NEEDS_NUMPY)
+ endif()
endfunction()
function(ADD_CONDA_TO_DOCKER TAG_PREFIX)
@@ -70,13 +76,10 @@ ADD_CONDA_TO_DOCKER(jammy)
ADD_VENV_TO_DOCKER(rocky9)
if (EXISTS ${CMAKE_SOURCE_DIR}/docker/test/integration/features)
- set(PYTHON_TESTS "${PYTHON_TESTS};${CMAKE_SOURCE_DIR}/docker/test/integration/features/python.feature")
- set(PYTHON_TESTS "${PYTHON_TESTS};${CMAKE_SOURCE_DIR}/docker/test/integration/features/python_script.feature")
- ADD_DOCKER_VERIFY(rocky8)
- ADD_DOCKER_VERIFY(rocky9)
- ADD_DOCKER_VERIFY(patched_jammy)
- ADD_DOCKER_VERIFY(patched_bullseye)
- set(PYTHON_TESTS "${PYTHON_TESTS};${CMAKE_SOURCE_DIR}/docker/test/integration/features/python_with_modules.feature")
- ADD_DOCKER_VERIFY(conda_jammy)
- ADD_DOCKER_VERIFY(venv_rocky9)
+ ADD_DOCKER_VERIFY_PYTHON(rocky8 FALSE)
+ ADD_DOCKER_VERIFY_PYTHON(rocky9 FALSE)
+ ADD_DOCKER_VERIFY_PYTHON(patched_jammy FALSE)
+ ADD_DOCKER_VERIFY_PYTHON(patched_bullseye FALSE)
+ ADD_DOCKER_VERIFY_PYTHON(conda_jammy TRUE)
+ ADD_DOCKER_VERIFY_PYTHON(venv_rocky9 TRUE)
endif()
diff --git a/docker/DockerVerify.sh b/docker/DockerVerify.sh
index 49ea8fd6c..a53d9c85a 100755
--- a/docker/DockerVerify.sh
+++ b/docker/DockerVerify.sh
@@ -26,18 +26,19 @@ die()
}
_positionals=()
-_arg_feature_path=('' )
_arg_image_tag_prefix=
-_enable_test_processors=OFF
-
+_arg_tags_to_exclude=
+_arg_parallel_processes=3
print_help()
{
printf '%s\n' "Runs the provided behave tests in a containerized environment"
- printf 'Usage: %s [--image-tag-prefix <arg>] [--enable_test_processors <ON|OFF>] [-h|--help] <minifi_version> <feature_path-1> [<feature_path-2>] ... [<feature_path-n>] ...\n' "$0"
+ printf 'Usage: %s [--image-tag-prefix <arg>] [-h|--help] <minifi_version> <feature_path-1> [<feature_path-2>] ... [<feature_path-n>] ...\n' "$0"
printf '\t%s\n' "<minifi_version>: the version of minifi"
- printf '\t%s\n' "<feature_path>: feature files to run"
+ printf '\t%s\n' "<tags_to_run>: include these tags"
+ printf '\t%s\n' "--tags_to_exclude: optional tags that should be skipped (no default)"
printf '\t%s\n' "--image-tag-prefix: optional prefix to the docker tag (no default)"
+ printf '\t%s\n' "--parallel_processes: optional argument that specifies the number of parallel processes that can be executed simultaneously. (default: 3)"
printf '\t%s\n' "-h, --help: Prints help"
}
@@ -57,13 +58,21 @@ parse_commandline()
--image-tag-prefix=*)
_arg_image_tag_prefix="${_key##--image-tag-prefix=}"
;;
- --enable_test_processors)
+ --tags_to_exclude)
+ test $# -lt 2 && die "Missing value for the optional argument '$_key'." 1
+ _arg_tags_to_exclude="$2"
+ shift
+ ;;
+ --tags_to_exclude=*)
+ _arg_tags_to_exclude="${_key##--tags_to_exclude=}"
+ ;;
+ --parallel_processes)
test $# -lt 2 && die "Missing value for the optional argument '$_key'." 1
- _enable_test_processors="$2"
+ _arg_parallel_processes="$2"
shift
;;
- --enable_test_processors=*)
- _enable_test_processors="${_key##--enable_test_processors=}"
+ --parallel_processes=*)
+ _arg_parallel_processes="${_key##--parallel_processes=}"
;;
-h|--help)
print_help
@@ -86,7 +95,7 @@ parse_commandline()
handle_passed_args_count()
{
- local _required_args_string="'minifi_version' and 'feature_path'"
+ local _required_args_string="'minifi_version' and 'tags_to_run'"
test "${_positionals_count}" -ge 2 || _PRINT_HELP=yes die "FATAL ERROR: Not enough positional arguments - we require at least 2 (namely: $_required_args_string), but got only ${_positionals_count}." 1
}
@@ -94,12 +103,7 @@ handle_passed_args_count()
assign_positional_args()
{
local _positional_name _shift_for=$1
- _positional_names="_arg_minifi_version _arg_feature_path "
- _our_args=$((${#_positionals[@]} - 2))
- for ((ii = 0; ii < _our_args; ii++))
- do
- _positional_names="$_positional_names _arg_feature_path[$((ii + 1))]"
- done
+ _positional_names="_arg_minifi_version _arg_tags_to_run "
shift "$_shift_for"
for _positional_name in ${_positional_names}
@@ -165,16 +169,19 @@ TEST_DIRECTORY="${docker_dir}/test/integration"
export TEST_DIRECTORY
# Add --no-logcapture to see logs interleaved with the test output
-BEHAVE_OPTS=(-f pretty --logging-level INFO --logging-clear-handlers)
-if [[ "${_enable_test_processors}" == "ON" ]]; then
- BEHAVE_OPTS+=(--define "test_processors=ON")
-else
- BEHAVE_OPTS+=(--define "test_processors=OFF")
+BEHAVE_OPTS=(--logging-level INFO --parallel-processes "${_arg_parallel_processes}" --parallel-scheme feature -o "${PWD}/behavex_output" -t "${_arg_tags_to_run}")
+if ! test -z "${_arg_tags_to_exclude}"
+then
+ IFS=','
+ read -ra splits <<< "${_arg_tags_to_exclude}"
+ for split in "${splits[@]}"
+ do
+ BEHAVE_OPTS=("${BEHAVE_OPTS[@]}" -t "~${split}")
+ done
fi
-# Specify feature or scenario to run a specific test e.g.:
-# behave "${BEHAVE_OPTS[@]}" "features/file_system_operations.feature"
-# behave "${BEHAVE_OPTS[@]}" "features/file_system_operations.feature" -n "Get and put operations run in a simple flow"
+echo "${BEHAVE_OPTS[@]}"
+
cd "${docker_dir}/test/integration"
exec
- behave "${BEHAVE_OPTS[@]}" "${_arg_feature_path[@]}"
+ behavex "${BEHAVE_OPTS[@]}"
diff --git a/docker/requirements.txt b/docker/requirements.txt
index 8e49046c9..715077678 100644
--- a/docker/requirements.txt
+++ b/docker/requirements.txt
@@ -1,4 +1,6 @@
-behave==1.2.6
+pyjks==20.0.0
+shortuuid==1.0.11
+behavex==2.0.1
docker==5.0.0
kafka-python==2.0.2
confluent-kafka==1.7.0
diff --git a/docker/test/integration/cluster/ContainerStore.py b/docker/test/integration/cluster/ContainerStore.py
index 7a511b7c5..2f9d29aef 100644
--- a/docker/test/integration/cluster/ContainerStore.py
+++ b/docker/test/integration/cluster/ContainerStore.py
@@ -12,8 +12,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-import uuid
import logging
+import shortuuid
from .containers.MinifiContainer import MinifiOptions
from .containers.MinifiContainer import MinifiContainer
from .containers.NifiContainer import NifiContainer
@@ -35,10 +35,12 @@ from .containers.MinifiAsPodInKubernetesCluster import MinifiAsPodInKubernetesCl
from .containers.TcpClientContainer import TcpClientContainer
from .containers.PrometheusContainer import PrometheusContainer
from .containers.MinifiC2ServerContainer import MinifiC2ServerContainer
+from .FeatureContext import FeatureContext
class ContainerStore:
- def __init__(self, network, image_store, kubernetes_proxy):
+ def __init__(self, network, image_store, kubernetes_proxy, feature_id):
+ self.feature_id = feature_id
self.minifi_options = MinifiOptions()
self.containers = {}
self.data_directories = {}
@@ -46,6 +48,11 @@ class ContainerStore:
self.image_store = image_store
self.kubernetes_proxy = kubernetes_proxy
+ def get_container_name_with_postfix(self, container_name: str):
+ if not container_name.endswith(self.feature_id):
+ return container_name + "-" + self.feature_id
+ return container_name
+
def cleanup(self):
for container in self.containers.values():
container.cleanup()
@@ -61,82 +68,228 @@ class ContainerStore:
for container in self.containers.values():
container.vols = self.vols
- def acquire_container(self, name, engine='minifi-cpp', command=None):
- if name is not None and name in self.containers:
- return self.containers[name]
+ def acquire_container(self, context, container_name: str, engine='minifi-cpp', command=None):
+ container_name = self.get_container_name_with_postfix(container_name)
+ if container_name is not None and container_name in self.containers:
+ return self.containers[container_name]
+
+ if container_name is None and (engine == 'nifi' or engine == 'minifi-cpp'):
+ container_name = engine + '-' + shortuuid.uuid()
+ logging.info('Container name was not provided; using generated name \'%s\'', container_name)
- if name is None and (engine == 'nifi' or engine == 'minifi-cpp'):
- name = engine + '-' + str(uuid.uuid4())
- logging.info('Container name was not provided; using generated name \'%s\'', name)
+ feature_context = FeatureContext(feature_id=context.feature_id,
+ root_ca_cert=context.test.root_ca_cert,
+ root_ca_key=context.test.root_ca_key)
if engine == 'nifi':
- return self.containers.setdefault(name, NifiContainer(self.data_directories["nifi_config_dir"], name, self.vols, self.network, self.image_store, command))
+ return self.containers.setdefault(container_name,
+ NifiContainer(feature_context=feature_context,
+ config_dir=self.data_directories["nifi_config_dir"],
+ name=container_name,
+ vols=self.vols,
+ network=self.network,
+ image_store=self.image_store,
+ command=command))
elif engine == 'minifi-cpp':
- return self.containers.setdefault(name, MinifiContainer(self.data_directories["minifi_config_dir"], self.minifi_options, name, self.vols, self.network, self.image_store, command))
+ return self.containers.setdefault(container_name,
+ MinifiContainer(feature_context=feature_context,
+ config_dir=self.data_directories["minifi_config_dir"],
+ options=self.minifi_options,
+ name=container_name,
+ vols=self.vols,
+ network=self.network,
+ image_store=self.image_store,
+ command=command))
elif engine == 'kubernetes':
- return self.containers.setdefault(name, MinifiAsPodInKubernetesCluster(self.kubernetes_proxy, self.data_directories["kubernetes_config_dir"], self.minifi_options, name, self.vols, self.network, self.image_store, command))
+ return self.containers.setdefault(container_name,
+ MinifiAsPodInKubernetesCluster(feature_context=feature_context,
+ kubernetes_proxy=self.kubernetes_proxy,
+ config_dir=self.data_directories["kubernetes_config_dir"],
+ minifi_options=self.minifi_options,
+ name=container_name,
+ vols=self.vols,
+ network=self.network,
+ image_store=self.image_store,
+ command=command))
elif engine == 'kafka-broker':
- if 'zookeeper' not in self.containers:
- self.containers.setdefault('zookeeper', ZookeeperContainer('zookeeper', self.vols, self.network, self.image_store, command))
- return self.containers.setdefault(name, KafkaBrokerContainer(name, self.vols, self.network, self.image_store, command))
+ zookeeper_name = self.get_container_name_with_postfix('zookeeper')
+ if zookeeper_name not in self.containers:
+ self.containers.setdefault(zookeeper_name,
+ ZookeeperContainer(feature_context=feature_context,
+ name=zookeeper_name,
+ vols=self.vols,
+ network=self.network,
+ image_store=self.image_store,
+ command=command))
+ return self.containers.setdefault(container_name,
+ KafkaBrokerContainer(feature_context=feature_context,
+ name=container_name,
+ vols=self.vols,
+ network=self.network,
+ image_store=self.image_store,
+ command=command))
elif engine == 'http-proxy':
- return self.containers.setdefault(name, HttpProxyContainer(name, self.vols, self.network, self.image_store, command))
+ return self.containers.setdefault(container_name,
+ HttpProxyContainer(feature_context=feature_context,
+ name=container_name,
+ vols=self.vols,
+ network=self.network,
+ image_store=self.image_store,
+ command=command))
elif engine == 's3-server':
- return self.containers.setdefault(name, S3ServerContainer(name, self.vols, self.network, self.image_store, command))
+ return self.containers.setdefault(container_name,
+ S3ServerContainer(feature_context=feature_context,
+ name=container_name,
+ vols=self.vols,
+ network=self.network,
+ image_store=self.image_store,
+ command=command))
elif engine == 'azure-storage-server':
- return self.containers.setdefault(name, AzureStorageServerContainer(name, self.vols, self.network, self.image_store, command))
+ return self.containers.setdefault(container_name,
+ AzureStorageServerContainer(feature_context=feature_context,
+ name=container_name,
+ vols=self.vols,
+ network=self.network,
+ image_store=self.image_store,
+ command=command))
elif engine == 'fake-gcs-server':
- return self.containers.setdefault(name, FakeGcsServerContainer(name, self.vols, self.network, self.image_store, command))
+ return self.containers.setdefault(container_name,
+ FakeGcsServerContainer(feature_context=feature_context,
+ name=container_name,
+ vols=self.vols,
+ network=self.network,
+ image_store=self.image_store,
+ command=command))
elif engine == 'postgresql-server':
- return self.containers.setdefault(name, PostgreSQLServerContainer(name, self.vols, self.network, self.image_store, command))
+ return self.containers.setdefault(container_name,
+ PostgreSQLServerContainer(feature_context=feature_context,
+ name=container_name,
+ vols=self.vols,
+ network=self.network,
+ image_store=self.image_store,
+ command=command))
elif engine == 'mqtt-broker':
- return self.containers.setdefault(name, MqttBrokerContainer(name, self.vols, self.network, self.image_store, command))
+ return self.containers.setdefault(container_name,
+ MqttBrokerContainer(feature_context=feature_context,
+ name=container_name,
+ vols=self.vols,
+ network=self.network,
+ image_store=self.image_store,
+ command=command))
elif engine == 'opcua-server':
- return self.containers.setdefault(name, OPCUAServerContainer(name, self.vols, self.network, self.image_store, command))
+ return self.containers.setdefault(container_name,
+ OPCUAServerContainer(feature_context=feature_context,
+ name=container_name,
+ vols=self.vols,
+ network=self.network,
+ image_store=self.image_store,
+ command=command))
elif engine == 'splunk':
- return self.containers.setdefault(name, SplunkContainer(name, self.vols, self.network, self.image_store, command))
+ return self.containers.setdefault(container_name,
+ SplunkContainer(feature_context=feature_context,
+ name=container_name,
+ vols=self.vols,
+ network=self.network,
+ image_store=self.image_store,
+ command=command))
elif engine == 'elasticsearch':
- return self.containers.setdefault(name, ElasticsearchContainer(name, self.vols, self.network, self.image_store, command))
+ return self.containers.setdefault(container_name,
+ ElasticsearchContainer(feature_context=feature_context,
+ name=container_name,
+ vols=self.vols,
+ network=self.network,
+ image_store=self.image_store,
+ command=command))
elif engine == 'opensearch':
- return self.containers.setdefault(name, OpensearchContainer(name, self.vols, self.network, self.image_store, command))
+ return self.containers.setdefault(container_name,
+ OpensearchContainer(feature_context=feature_context,
+ name=container_name,
+ vols=self.vols,
+ network=self.network,
+ image_store=self.image_store,
+ command=command))
elif engine == "syslog-udp-client":
- return self.containers.setdefault(name, SyslogUdpClientContainer(name, self.vols, self.network, self.image_store, command))
+ return self.containers.setdefault(container_name,
+ SyslogUdpClientContainer(
+ feature_context=feature_context,
+ name=container_name,
+ vols=self.vols,
+ network=self.network,
+ image_store=self.image_store,
+ command=command))
elif engine == "syslog-tcp-client":
- return self.containers.setdefault(name, SyslogTcpClientContainer(name, self.vols, self.network, self.image_store, command))
+ return self.containers.setdefault(container_name,
+ SyslogTcpClientContainer(
+ feature_context=feature_context,
+ name=container_name,
+ vols=self.vols,
+ network=self.network,
+ image_store=self.image_store,
+ command=command))
elif engine == "tcp-client":
- return self.containers.setdefault(name, TcpClientContainer(name, self.vols, self.network, self.image_store, command))
+ return self.containers.setdefault(container_name,
+ TcpClientContainer(feature_context=feature_context,
+ name=container_name,
+ vols=self.vols,
+ network=self.network,
+ image_store=self.image_store,
+ command=command))
elif engine == "prometheus":
- return self.containers.setdefault(name, PrometheusContainer(name, self.vols, self.network, self.image_store, command))
+ return self.containers.setdefault(container_name,
+ PrometheusContainer(feature_context=feature_context,
+ name=container_name,
+ vols=self.vols,
+ network=self.network,
+ image_store=self.image_store,
+ command=command))
elif engine == "minifi-c2-server":
- return self.containers.setdefault(name, MinifiC2ServerContainer(name, self.vols, self.network, self.image_store, command))
+ return self.containers.setdefault(container_name,
+ MinifiC2ServerContainer(feature_context=feature_context,
+ name=container_name,
+ vols=self.vols,
+ network=self.network,
+ image_store=self.image_store,
+ command=command,
+ ssl=False))
elif engine == "minifi-c2-server-ssl":
- return self.containers.setdefault(name, MinifiC2ServerContainer(name, self.vols, self.network, self.image_store, command, ssl=True))
+ return self.containers.setdefault(container_name,
+ MinifiC2ServerContainer(feature_context=feature_context,
+ name=container_name,
+ vols=self.vols,
+ network=self.network,
+ image_store=self.image_store,
+ command=command,
+ ssl=True))
else:
raise Exception('invalid flow engine: \'%s\'' % engine)
- def deploy_container(self, name):
- if name is None or name not in self.containers:
- raise Exception('Invalid container to deploy: \'%s\'' % name)
+ def deploy_container(self, container_name: str):
+ container_name = self.get_container_name_with_postfix(container_name)
+ if container_name is None or container_name not in self.containers:
+ raise Exception('Invalid container to deploy: \'%s\'' % container_name)
- self.containers[name].deploy()
+ self.containers[container_name].deploy()
def deploy_all(self):
for container in self.containers.values():
container.deploy()
def stop_container(self, container_name):
+ container_name = self.get_container_name_with_postfix(container_name)
if container_name not in self.containers:
logging.error('Could not stop container because it is not found: \'%s\'', container_name)
return
self.containers[container_name].stop()
def kill_container(self, container_name):
+ container_name = self.get_container_name_with_postfix(container_name)
if container_name not in self.containers:
logging.error('Could not kill container because it is not found: \'%s\'', container_name)
return
self.containers[container_name].kill()
def restart_container(self, container_name):
+ container_name = self.get_container_name_with_postfix(container_name)
if container_name not in self.containers:
logging.error('Could not restart container because it is not found: \'%s\'', container_name)
return
@@ -170,12 +323,15 @@ class ContainerStore:
self.minifi_options.enable_controller_socket = True
def get_startup_finished_log_entry(self, container_name):
+ container_name = self.get_container_name_with_postfix(container_name)
return self.containers[container_name].get_startup_finished_log_entry()
def log_source(self, container_name):
+ container_name = self.get_container_name_with_postfix(container_name)
return self.containers[container_name].log_source()
def get_app_log(self, container_name):
+ container_name = self.get_container_name_with_postfix(container_name)
return self.containers[container_name].get_app_log()
def get_container_names(self, engine=None):
diff --git a/docker/test/integration/cluster/DockerCommunicator.py b/docker/test/integration/cluster/DockerCommunicator.py
index e0fa305d0..71acdd367 100644
--- a/docker/test/integration/cluster/DockerCommunicator.py
+++ b/docker/test/integration/cluster/DockerCommunicator.py
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import docker
-import uuid
import logging
import sys
import tempfile
@@ -26,8 +25,8 @@ class DockerCommunicator:
def __init__(self):
self.client = docker.from_env()
- def create_docker_network(self):
- net_name = 'minifi_integration_test_network-' + str(uuid.uuid4())
+ def create_docker_network(self, feature_id: str):
+ net_name = 'minifi_integration_test_network-' + feature_id
logging.debug('Creating network: %s', net_name)
return self.client.networks.create(net_name)
diff --git a/docker/test/integration/cluster/DockerTestCluster.py b/docker/test/integration/cluster/DockerTestCluster.py
index 732837245..6e694712c 100644
--- a/docker/test/integration/cluster/DockerTestCluster.py
+++ b/docker/test/integration/cluster/DockerTestCluster.py
@@ -31,11 +31,11 @@ from utils import get_peak_memory_usage, get_minifi_pid, get_memory_usage, retry
class DockerTestCluster:
- def __init__(self, context):
+ def __init__(self, context, feature_id):
self.segfault = False
self.vols = {}
self.container_communicator = DockerCommunicator()
- self.container_store = ContainerStore(self.container_communicator.create_docker_network(), context.image_store, context.kubernetes_proxy)
+ self.container_store = ContainerStore(self.container_communicator.create_docker_network(feature_id), context.image_store, context.kubernetes_proxy, feature_id=feature_id)
self.aws_checker = AwsChecker(self.container_communicator)
self.azure_checker = AzureChecker(self.container_communicator)
self.elastic_search_checker = ElasticSearchChecker(self.container_communicator)
@@ -51,8 +51,8 @@ class DockerTestCluster:
def set_directory_bindings(self, volumes, data_directories):
self.container_store.set_directory_bindings(volumes, data_directories)
- def acquire_container(self, name, engine='minifi-cpp', command=None):
- return self.container_store.acquire_container(name, engine, command)
+ def acquire_container(self, context, name: str, engine: str = 'minifi-cpp', command=None):
+ return self.container_store.acquire_container(context=context, container_name=name, engine=engine, command=command)
def deploy_container(self, name):
self.container_store.deploy_container(name)
@@ -97,6 +97,7 @@ class DockerTestCluster:
self.container_store.set_controller_socket_properties_in_minifi()
def get_app_log(self, container_name):
+ container_name = self.container_store.get_container_name_with_postfix(container_name)
log_source = self.container_store.log_source(container_name)
if log_source == LogSource.FROM_DOCKER_CONTAINER:
return self.container_communicator.get_app_log_from_docker_container(container_name)
@@ -140,22 +141,27 @@ class DockerTestCluster:
logging.info(line)
def check_http_proxy_access(self, container_name, url):
+ container_name = self.container_store.get_container_name_with_postfix(container_name)
(code, output) = self.container_communicator.execute_command(container_name, ["cat", "/var/log/squid/access.log"])
- return code == 0 and url in output \
+ return code == 0 and url.lower() in output.lower() \
and ((output.count("TCP_DENIED") != 0
and output.count("TCP_MISS") >= output.count("TCP_DENIED"))
or output.count("TCP_DENIED") == 0 and "TCP_MISS" in output)
def check_s3_server_object_data(self, container_name, test_data):
+ container_name = self.container_store.get_container_name_with_postfix(container_name)
return self.aws_checker.check_s3_server_object_data(container_name, test_data)
def check_s3_server_object_metadata(self, container_name, content_type="application/octet-stream", metadata=dict()):
+ container_name = self.container_store.get_container_name_with_postfix(container_name)
return self.aws_checker.check_s3_server_object_metadata(container_name, content_type, metadata)
def is_s3_bucket_empty(self, container_name):
+ container_name = self.container_store.get_container_name_with_postfix(container_name)
return self.aws_checker.is_s3_bucket_empty(container_name)
def check_azure_storage_server_data(self, container_name, test_data):
+ container_name = self.container_store.get_container_name_with_postfix(container_name)
return self.azure_checker.check_azure_storage_server_data(container_name, test_data)
def add_test_blob(self, blob_name, content="", with_snapshot=False):
@@ -168,45 +174,58 @@ class DockerTestCluster:
return self.azure_checker.check_azure_blob_storage_is_empty(timeout_seconds)
def check_splunk_event(self, container_name, query):
+ container_name = self.container_store.get_container_name_with_postfix(container_name)
return self.splunk_checker.check_splunk_event(container_name, query)
def check_splunk_event_with_attributes(self, container_name, query, attributes):
+ container_name = self.container_store.get_container_name_with_postfix(container_name)
return self.splunk_checker.check_splunk_event_with_attributes(container_name, query, attributes)
def enable_splunk_hec_indexer(self, container_name, hec_name):
+ container_name = self.container_store.get_container_name_with_postfix(container_name)
return self.splunk_checker.enable_splunk_hec_indexer(container_name, hec_name)
def enable_splunk_hec_ssl(self, container_name, splunk_cert_pem, splunk_key_pem, root_ca_cert_pem):
+ container_name = self.container_store.get_container_name_with_postfix(container_name)
return self.splunk_checker.enable_splunk_hec_ssl(container_name, splunk_cert_pem, splunk_key_pem, root_ca_cert_pem)
def check_google_cloud_storage(self, gcs_container_name, content):
+ gcs_container_name = self.container_store.get_container_name_with_postfix(gcs_container_name)
return self.gcs_checker.check_google_cloud_storage(gcs_container_name, content)
def is_gcs_bucket_empty(self, container_name):
+ container_name = self.container_store.get_container_name_with_postfix(container_name)
return self.gcs_checker.is_gcs_bucket_empty(container_name)
def is_elasticsearch_empty(self, container_name):
+ container_name = self.container_store.get_container_name_with_postfix(container_name)
return self.elastic_search_checker.is_elasticsearch_empty(container_name)
def create_doc_elasticsearch(self, container_name, index_name, doc_id):
+ container_name = self.container_store.get_container_name_with_postfix(container_name)
return self.elastic_search_checker.create_doc_elasticsearch(container_name, index_name, doc_id)
def check_elastic_field_value(self, container_name, index_name, doc_id, field_name, field_value):
+ container_name = self.container_store.get_container_name_with_postfix(container_name)
return self.elastic_search_checker.check_elastic_field_value(container_name, index_name, doc_id, field_name, field_value)
def elastic_generate_apikey(self, elastic_container_name):
+ elastic_container_name = self.container_store.get_container_name_with_postfix(elastic_container_name)
return self.elastic_search_checker.elastic_generate_apikey(elastic_container_name)
def add_elastic_user_to_opensearch(self, container_name):
+ container_name = self.container_store.get_container_name_with_postfix(container_name)
return self.elastic_search_checker.add_elastic_user_to_opensearch(container_name)
def check_query_results(self, postgresql_container_name, query, number_of_rows, timeout_seconds):
+ postgresql_container_name = self.container_store.get_container_name_with_postfix(postgresql_container_name)
return self.postgres_checker.check_query_results(postgresql_container_name, query, number_of_rows, timeout_seconds)
def segfault_happened(self):
return self.segfault
def wait_for_kafka_consumer_to_be_registered(self, kafka_container_name):
+ kafka_container_name = self.container_store.get_container_name_with_postfix(kafka_container_name)
return self.wait_for_app_logs_regex(kafka_container_name, "Assignment received from leader.*for group docker_test_group", 60)
def wait_for_metric_class_on_prometheus(self, metric_class, timeout_seconds):
@@ -238,7 +257,8 @@ class DockerTestCluster:
return True
def wait_for_container_startup_to_finish(self, container_name):
- startup_success = self.wait_for_startup_log(container_name, 120)
+ container_name = self.container_store.get_container_name_with_postfix(container_name)
+ startup_success = self.wait_for_startup_log(container_name, 300)
if not startup_success:
logging.error("Cluster startup failed for %s", container_name)
self.log_app_output()
diff --git a/docker/test/integration/cluster/DockerTestDirectoryBindings.py b/docker/test/integration/cluster/DockerTestDirectoryBindings.py
index 259d3a44d..9946f26c6 100644
--- a/docker/test/integration/cluster/DockerTestDirectoryBindings.py
+++ b/docker/test/integration/cluster/DockerTestDirectoryBindings.py
@@ -20,15 +20,15 @@ import shutil
class DockerTestDirectoryBindings:
- def __init__(self, test_id):
+ def __init__(self, feature_id: str):
self.data_directories = {}
- self.test_id = test_id
+ self.feature_id = feature_id
def __del__(self):
self.delete_data_directories()
def cleanup_io(self):
- for folder in [self.data_directories[self.test_id]["input_dir"], self.data_directories[self.test_id]["output_dir"]]:
+ for folder in [self.data_directories[self.feature_id]["input_dir"], self.data_directories[self.feature_id]["output_dir"]]:
for filename in os.listdir(folder):
file_path = os.path.join(folder, filename)
if os.path.isfile(file_path) or os.path.islink(file_path):
@@ -37,62 +37,58 @@ class DockerTestDirectoryBindings:
shutil.rmtree(file_path)
def create_new_data_directories(self):
- self.data_directories[self.test_id] = {
- "input_dir": "/tmp/.nifi-test-input." + self.test_id,
- "output_dir": "/tmp/.nifi-test-output." + self.test_id,
- "resources_dir": "/tmp/.nifi-test-resources." + self.test_id,
- "minifi_config_dir": "/tmp/.nifi-test-minifi-config-dir." + self.test_id,
- "nifi_config_dir": "/tmp/.nifi-test-nifi-config-dir." + self.test_id,
- "kubernetes_temp_dir": "/tmp/.nifi-test-kubernetes-temp-dir." + self.test_id,
- "kubernetes_config_dir": "/tmp/.nifi-test-kubernetes-config-dir." + self.test_id
+ self.data_directories[self.feature_id] = {
+ "input_dir": "/tmp/.nifi-test-input." + self.feature_id,
+ "output_dir": "/tmp/.nifi-test-output." + self.feature_id,
+ "resources_dir": "/tmp/.nifi-test-resources." + self.feature_id,
+ "minifi_config_dir": "/tmp/.nifi-test-minifi-config-dir." + self.feature_id,
+ "nifi_config_dir": "/tmp/.nifi-test-nifi-config-dir." + self.feature_id,
+ "kubernetes_temp_dir": "/tmp/.nifi-test-kubernetes-temp-dir." + self.feature_id,
+ "kubernetes_config_dir": "/tmp/.nifi-test-kubernetes-config-dir." + self.feature_id
}
- [self.create_directory(directory) for directory in self.data_directories[self.test_id].values()]
+ [self.create_directory(directory) for directory in self.data_directories[self.feature_id].values()]
# Add resources
test_dir = os.environ['TEST_DIRECTORY'] # Based on DockerVerify.sh
- shutil.copytree(test_dir + "/resources/kafka_broker/conf/certs", self.data_directories[self.test_id]["resources_dir"] + "/certs")
- shutil.copytree(test_dir + "/resources/python", self.data_directories[self.test_id]["resources_dir"] + "/python")
- shutil.copytree(test_dir + "/resources/opcua", self.data_directories[self.test_id]["resources_dir"] + "/opcua")
- shutil.copytree(test_dir + "/resources/lua", self.data_directories[self.test_id]["resources_dir"] + "/lua")
- shutil.copytree(test_dir + "/resources/elasticsearch/certs", self.data_directories[self.test_id]["resources_dir"] + "/elasticsearch")
- shutil.copytree(test_dir + "/resources/opensearch/certs", self.data_directories[self.test_id]["resources_dir"] + "/opensearch")
- shutil.copytree(test_dir + "/resources/minifi-c2-server-ssl/certs", self.data_directories[self.test_id]["resources_dir"] + "/minifi-c2-server-ssl")
- shutil.copytree(test_dir + "/resources/minifi-controller", self.data_directories[self.test_id]["resources_dir"] + "/minifi-controller")
- shutil.copytree(test_dir + "/resources/minifi", self.data_directories[self.test_id]["minifi_config_dir"], dirs_exist_ok=True)
-
- def get_data_directories(self, test_id):
- return self.data_directories[test_id]
-
- def docker_path_to_local_path(self, test_id, docker_path):
+ shutil.copytree(test_dir + "/resources/python", self.data_directories[self.feature_id]["resources_dir"] + "/python")
+ shutil.copytree(test_dir + "/resources/opcua", self.data_directories[self.feature_id]["resources_dir"] + "/opcua")
+ shutil.copytree(test_dir + "/resources/lua", self.data_directories[self.feature_id]["resources_dir"] + "/lua")
+ shutil.copytree(test_dir + "/resources/minifi", self.data_directories[self.feature_id]["minifi_config_dir"], dirs_exist_ok=True)
+ shutil.copytree(test_dir + "/resources/minifi-controller", self.data_directories[self.feature_id]["resources_dir"] + "/minifi-controller")
+
+ def get_data_directories(self, feature_id):
+ return self.data_directories[feature_id]
+
+ def docker_path_to_local_path(self, feature_id, docker_path):
# Docker paths are currently hard-coded
if docker_path == "/tmp/input":
- return self.data_directories[test_id]["input_dir"]
+ return self.data_directories[feature_id]["input_dir"]
if docker_path == "/tmp/output":
- return self.data_directories[test_id]["output_dir"]
+ return self.data_directories[feature_id]["output_dir"]
if docker_path == "/tmp/resources":
- return self.data_directories[test_id]["resources_dir"]
+ return self.data_directories[feature_id]["resources_dir"]
# Might be worth reworking these
if docker_path == "/tmp/output/success":
- self.create_directory(self.data_directories[test_id]["output_dir"] + "/success")
- return self.data_directories[test_id]["output_dir"] + "/success"
+ self.create_directory(self.data_directories[feature_id]["output_dir"] + "/success")
+ return self.data_directories[feature_id]["output_dir"] + "/success"
if docker_path == "/tmp/output/failure":
- self.create_directory(self.data_directories[test_id]["output_dir"] + "/failure")
- return self.data_directories[test_id]["output_dir"] + "/failure"
+ self.create_directory(self.data_directories[feature_id]["output_dir"] + "/failure")
+ return self.data_directories[feature_id]["output_dir"] + "/failure"
raise Exception("Docker directory \"%s\" has no preset bindings." % docker_path)
- def get_directory_bindings(self, test_id):
+ def get_directory_bindings(self, feature_id):
"""
Performs a standard container flow deployment with the addition
of volumes supporting test input/output directories.
"""
vols = {}
- vols[self.data_directories[test_id]["input_dir"]] = {"bind": "/tmp/input", "mode": "rw"}
- vols[self.data_directories[test_id]["output_dir"]] = {"bind": "/tmp/output", "mode": "rw"}
- vols[self.data_directories[test_id]["resources_dir"]] = {"bind": "/tmp/resources", "mode": "rw"}
- vols[self.data_directories[test_id]["minifi_config_dir"]] = {"bind": "/tmp/minifi_config", "mode": "rw"}
- vols[self.data_directories[test_id]["nifi_config_dir"]] = {"bind": "/tmp/nifi_config", "mode": "rw"}
- vols[self.data_directories[test_id]["kubernetes_config_dir"]] = {"bind": "/tmp/kubernetes_config", "mode": "rw"}
+ vols[self.data_directories[feature_id]["input_dir"]] = {"bind": "/tmp/input", "mode": "rw"}
+ vols[self.data_directories[feature_id]["output_dir"]] = {"bind": "/tmp/output", "mode": "rw"}
+ vols[self.data_directories[feature_id]["resources_dir"]] = {"bind": "/tmp/resources", "mode": "rw"}
+ vols[self.data_directories[feature_id]["minifi_config_dir"]] = {"bind": "/tmp/minifi_config", "mode": "rw"}
+ vols[self.data_directories[feature_id]["nifi_config_dir"]] = {"bind": "/tmp/nifi_config", "mode": "rw"}
+ vols[self.data_directories[feature_id]["kubernetes_config_dir"]] = {"bind": "/tmp/kubernetes_config", "mode": "rw"}
return vols
@staticmethod
@@ -116,28 +112,29 @@ class DockerTestDirectoryBindings:
@staticmethod
def put_file_contents(file_abs_path, contents):
logging.info('Writing %d bytes of content to file: %s', len(contents), file_abs_path)
+ os.makedirs(os.path.dirname(file_abs_path), exist_ok=True)
with open(file_abs_path, 'wb') as test_input_file:
test_input_file.write(contents)
os.chmod(file_abs_path, 0o0777)
- def put_test_resource(self, test_id, file_name, contents):
+ def put_test_resource(self, feature_id, file_name, contents):
"""
Creates a resource file in the test resource dir and writes
the given content to it.
"""
- file_abs_path = os.path.join(self.data_directories[test_id]["resources_dir"], file_name)
+ file_abs_path = os.path.join(self.data_directories[feature_id]["resources_dir"], file_name)
self.put_file_contents(file_abs_path, contents)
- def put_test_input(self, test_id, file_name, contents):
- file_abs_path = os.path.join(self.data_directories[test_id]["input_dir"], file_name)
+ def put_test_input(self, feature_id, file_name, contents):
+ file_abs_path = os.path.join(self.data_directories[feature_id]["input_dir"], file_name)
self.put_file_contents(file_abs_path, contents)
- def put_file_to_docker_path(self, test_id, path, file_name, contents):
- file_abs_path = os.path.join(self.docker_path_to_local_path(test_id, path), file_name)
+ def put_file_to_docker_path(self, feature_id, path, file_name, contents):
+ file_abs_path = os.path.join(self.docker_path_to_local_path(feature_id, path), file_name)
self.put_file_contents(file_abs_path, contents)
- def rm_out_child(self, test_id, dir):
- child = os.path.join(self.data_directories[test_id]["output_dir"], dir)
+ def rm_out_child(self, feature_id, dir):
+ child = os.path.join(self.data_directories[feature_id]["output_dir"], dir)
logging.info('Removing %s from output folder', child)
shutil.rmtree(child)
diff --git a/docker/test/integration/minifi/processors/AttributesToJSON.py b/docker/test/integration/cluster/FeatureContext.py
similarity index 80%
copy from docker/test/integration/minifi/processors/AttributesToJSON.py
copy to docker/test/integration/cluster/FeatureContext.py
index a98fbf765..febb524d5 100644
--- a/docker/test/integration/minifi/processors/AttributesToJSON.py
+++ b/docker/test/integration/cluster/FeatureContext.py
@@ -14,9 +14,8 @@
# limitations under the License.
-from ..core.Processor import Processor
-
-
-class AttributesToJSON(Processor):
- def __init__(self):
- super(AttributesToJSON, self).__init__('AttributesToJSON')
+class FeatureContext:
+ def __init__(self, feature_id, root_ca_cert, root_ca_key):
+ self.id = feature_id
+ self.root_ca_cert = root_ca_cert
+ self.root_ca_key = root_ca_key
diff --git a/docker/test/integration/cluster/ImageStore.py b/docker/test/integration/cluster/ImageStore.py
index ade1d401a..a1da6f2ee 100644
--- a/docker/test/integration/cluster/ImageStore.py
+++ b/docker/test/integration/cluster/ImageStore.py
@@ -45,24 +45,10 @@ class ImageStore:
image = self.__build_http_proxy_image()
elif container_engine == "postgresql-server":
image = self.__build_postgresql_server_image()
- elif container_engine == "kafka-broker":
- image = self.__build_kafka_broker_image()
elif container_engine == "mqtt-broker":
image = self.__build_mqtt_broker_image()
elif container_engine == "splunk":
image = self.__build_splunk_image()
- elif container_engine == "tcp-client":
- image = self.__build_tcp_client_image()
- elif container_engine == "prometheus":
- image = self.__build_prometheus_image()
- elif container_engine == "elasticsearch":
- image = self.__build_elasticsearch_image()
- elif container_engine == "opensearch":
- image = self.__build_opensearch_image()
- elif container_engine == "minifi-c2-server":
- image = self.__build_minifi_c2_image()
- elif container_engine == "minifi-c2-server-ssl":
- image = self.__build_minifi_c2_ssl_image()
else:
raise Exception("There is no associated image for " + container_engine)
@@ -136,9 +122,6 @@ class ImageStore:
""".format(base_image='postgres:13.2'))
return self.__build_image(dockerfile)
- def __build_kafka_broker_image(self):
- return self.__build_image_by_path(self.test_dir + "/resources/kafka_broker", 'minifi-kafka')
-
def __build_mqtt_broker_image(self):
dockerfile = dedent("""\
FROM {base_image}
@@ -151,30 +134,6 @@ class ImageStore:
def __build_splunk_image(self):
return self.__build_image_by_path(self.test_dir + "/resources/splunk-hec", 'minifi-splunk')
- def __build_tcp_client_image(self):
- dockerfile = dedent("""\
- FROM {base_image}
- RUN apk add netcat-openbsd
- CMD ["/bin/sh", "-c", "echo TCP client container started; while true; do echo test_tcp_message | nc minifi-cpp-flow 10254; sleep 1; done"]
- """.format(base_image='alpine:3.13'))
-
- return self.__build_image(dockerfile)
-
- def __build_prometheus_image(self):
- return self.__build_image_by_path(self.test_dir + "/resources/prometheus", 'minifi-prometheus')
-
- def __build_elasticsearch_image(self):
- return self.__build_image_by_path(self.test_dir + "/resources/elasticsearch", 'elasticsearch')
-
- def __build_opensearch_image(self):
- return self.__build_image_by_path(self.test_dir + "/resources/opensearch", 'opensearch')
-
- def __build_minifi_c2_image(self):
- return self.__build_image_by_path(self.test_dir + "/resources/minifi-c2-server", 'minifi-c2-server')
-
- def __build_minifi_c2_ssl_image(self):
- return self.__build_image_by_path(self.test_dir + "/resources/minifi-c2-server-ssl", 'minifi-c2-server')
-
def __build_image(self, dockerfile, context_files=[]):
conf_dockerfile_buffer = BytesIO()
docker_context_buffer = BytesIO()
diff --git a/docker/test/integration/cluster/checkers/ElasticSearchChecker.py b/docker/test/integration/cluster/checkers/ElasticSearchChecker.py
index 7149545ac..34186cc4e 100644
--- a/docker/test/integration/cluster/checkers/ElasticSearchChecker.py
+++ b/docker/test/integration/cluster/checkers/ElasticSearchChecker.py
@@ -26,12 +26,12 @@ class ElasticSearchChecker:
def create_doc_elasticsearch(self, container_name, index_name, doc_id):
(code, output) = self.container_communicator.execute_command(container_name, ["/bin/bash", "-c",
"curl -u elastic:password -k -XPUT https://localhost:9200/" + index_name + "/_doc/" + doc_id + " -H Content-Type:application/json -d'{\"field1\":\"value1\"}'"])
- return code == 0 and ('"_id":"' + doc_id + '"').encode() in output
+ return code == 0 and ('"_id":"' + doc_id + '"') in output
def check_elastic_field_value(self, container_name, index_name, doc_id, field_name, field_value):
(code, output) = self.container_communicator.execute_command(container_name, ["/bin/bash", "-c",
"curl -u elastic:password -k -XGET https://localhost:9200/" + index_name + "/_doc/" + doc_id])
- return code == 0 and (field_name + '":"' + field_value).encode() in output
+ return code == 0 and (field_name + '":"' + field_value) in output
def elastic_generate_apikey(self, elastic_container_name):
(_, output) = self.container_communicator.execute_command(elastic_container_name, ["/bin/bash", "-c",
@@ -42,5 +42,5 @@ class ElasticSearchChecker:
def add_elastic_user_to_opensearch(self, container_name):
(code, output) = self.container_communicator.execute_command(container_name, ["/bin/bash", "-c",
- 'curl -u admin:admin -k -XPUT https://opensearch:9200/_plugins/_security/api/internalusers/elastic -H Content-Type:application/json -d\'{"password":"password","backend_roles":["admin"]}\''])
- return code == 0 and '"status":"CREATED"'.encode() in output
+ 'curl -u admin:admin -k -XPUT https://{hostname}:9200/_plugins/_security/api/internalusers/elastic -H Content-Type:application/json -d\'{{"password":"password","backend_roles":["admin"]}}\''.format(hostname=container_name)])
+ return code == 0 and '"status":"CREATED"' in output
diff --git a/docker/test/integration/cluster/containers/AzureStorageServerContainer.py b/docker/test/integration/cluster/containers/AzureStorageServerContainer.py
index 65014b185..716b0d339 100644
--- a/docker/test/integration/cluster/containers/AzureStorageServerContainer.py
+++ b/docker/test/integration/cluster/containers/AzureStorageServerContainer.py
@@ -19,8 +19,8 @@ from .Container import Container
class AzureStorageServerContainer(Container):
- def __init__(self, name, vols, network, image_store, command=None):
- super().__init__(name, 'azure-storage-server', vols, network, image_store, command)
+ def __init__(self, feature_context, name, vols, network, image_store, command=None):
+ super().__init__(feature_context, name, 'azure-storage-server', vols, network, image_store, command)
def get_startup_finished_log_entry(self):
return "Azurite Queue service is successfully listening at"
diff --git a/docker/test/integration/cluster/containers/Container.py b/docker/test/integration/cluster/containers/Container.py
index 1a4153e72..b4a8efe85 100644
--- a/docker/test/integration/cluster/containers/Container.py
+++ b/docker/test/integration/cluster/containers/Container.py
@@ -18,10 +18,12 @@ import docker
import logging
from ..LogSource import LogSource
+from ..FeatureContext import FeatureContext
class Container:
- def __init__(self, name, engine, vols, network, image_store, command):
+ def __init__(self, feature_context: FeatureContext, name, engine, vols, network, image_store, command):
+ self.feature_context = feature_context
self.name = name
self.engine = engine
self.vols = vols
diff --git a/docker/test/integration/cluster/containers/ElasticsearchContainer.py b/docker/test/integration/cluster/containers/ElasticsearchContainer.py
index 878603de5..35347dba1 100644
--- a/docker/test/integration/cluster/containers/ElasticsearchContainer.py
+++ b/docker/test/integration/cluster/containers/ElasticsearchContainer.py
@@ -15,12 +15,45 @@
import logging
+import tempfile
+import os
+import OpenSSL.crypto
+import docker.types
+
from .Container import Container
+from ssl_utils.SSL_cert_utils import make_server_cert, make_cert_without_extended_usage
class ElasticsearchContainer(Container):
- def __init__(self, name, vols, network, image_store, command=None):
- super().__init__(name, 'elasticsearch', vols, network, image_store, command)
+ def __init__(self, feature_context, name, vols, network, image_store, command=None):
+ super().__init__(feature_context, name, 'elasticsearch', vols, network, image_store, command)
+ http_cert, http_key = make_server_cert(f"elasticsearch-{feature_context.id}", feature_context.root_ca_cert, feature_context.root_ca_key)
+ transport_cert, transport_key = make_cert_without_extended_usage("127.0.0.1", feature_context.root_ca_cert, feature_context.root_ca_key)
+
+ self.root_ca_file = tempfile.NamedTemporaryFile(delete=False)
+ self.root_ca_file.write(OpenSSL.crypto.dump_certificate(type=OpenSSL.crypto.FILETYPE_PEM, cert=feature_context.root_ca_cert))
+ self.root_ca_file.close()
+ os.chmod(self.root_ca_file.name, 0o644)
+
+ self.http_cert_file = tempfile.NamedTemporaryFile(delete=False)
+ self.http_cert_file.write(OpenSSL.crypto.dump_certificate(type=OpenSSL.crypto.FILETYPE_PEM, cert=http_cert))
+ self.http_cert_file.close()
+ os.chmod(self.http_cert_file.name, 0o644)
+
+ self.http_key_file = tempfile.NamedTemporaryFile(delete=False)
+ self.http_key_file.write(OpenSSL.crypto.dump_privatekey(type=OpenSSL.crypto.FILETYPE_PEM, pkey=http_key))
+ self.http_key_file.close()
+ os.chmod(self.http_key_file.name, 0o644)
+
+ self.transport_cert_file = tempfile.NamedTemporaryFile(delete=False)
+ self.transport_cert_file.write(OpenSSL.crypto.dump_certificate(type=OpenSSL.crypto.FILETYPE_PEM, cert=transport_cert))
+ self.transport_cert_file.close()
+ os.chmod(self.transport_cert_file.name, 0o644)
+
+ self.transport_key_file = tempfile.NamedTemporaryFile(delete=False)
+ self.transport_key_file.write(OpenSSL.crypto.dump_privatekey(type=OpenSSL.crypto.FILETYPE_PEM, pkey=transport_key))
+ self.transport_key_file.close()
+ os.chmod(self.transport_key_file.name, 0o644)
def get_startup_finished_log_entry(self):
return '"current.health":"GREEN"'
@@ -29,13 +62,40 @@ class ElasticsearchContainer(Container):
if not self.set_deployed():
return
+ mounts = [
+ docker.types.Mount(
+ type='bind',
+ source=os.environ['TEST_DIRECTORY'] + "/resources/elasticsearch/elasticsearch.yml",
+ target='/usr/share/elasticsearch/config/elasticsearch.yml'),
+ docker.types.Mount(
+ type='bind',
+ source=self.http_key_file.name,
+ target='/usr/share/elasticsearch/config/certs/elastic_http.key'),
+ docker.types.Mount(
+ type='bind',
+ source=self.http_cert_file.name,
+ target='/usr/share/elasticsearch/config/certs/elastic_http.crt'),
+ docker.types.Mount(
+ type='bind',
+ source=self.transport_key_file.name,
+ target='/usr/share/elasticsearch/config/certs/elastic_transport.key'),
+ docker.types.Mount(
+ type='bind',
+ source=self.transport_cert_file.name,
+ target='/usr/share/elasticsearch/config/certs/elastic_transport.crt'),
+ docker.types.Mount(
+ type='bind',
+ source=self.root_ca_file.name,
+ target='/usr/share/elasticsearch/config/certs/root_ca.crt')]
+
logging.info('Creating and running Elasticsearch docker container...')
self.client.containers.run(
- self.image_store.get_image(self.get_engine()),
+ image="elasticsearch:8.2.2",
detach=True,
name=self.name,
environment=[
"ELASTIC_PASSWORD=password",
],
- network=self.network.name)
+ network=self.network.name,
+ mounts=mounts)
logging.info('Added container \'%s\'', self.name)
diff --git a/docker/test/integration/cluster/containers/FakeGcsServerContainer.py b/docker/test/integration/cluster/containers/FakeGcsServerContainer.py
index 977037bdc..32a29cb71 100644
--- a/docker/test/integration/cluster/containers/FakeGcsServerContainer.py
+++ b/docker/test/integration/cluster/containers/FakeGcsServerContainer.py
@@ -18,8 +18,8 @@ from .Container import Container
class FakeGcsServerContainer(Container):
- def __init__(self, name, vols, network, image_store, command=None):
- super().__init__(name, 'fake-gcs-server', vols, network, image_store, command)
+ def __init__(self, feature_context, name, vols, network, image_store, command=None):
+ super().__init__(feature_context, name, 'fake-gcs-server', vols, network, image_store, command)
def get_startup_finished_log_entry(self):
return "server started at http"
@@ -35,7 +35,6 @@ class FakeGcsServerContainer(Container):
name=self.name,
network=self.network.name,
entrypoint=self.command,
- ports={'4443/tcp': 4443},
volumes=[os.environ['TEST_DIRECTORY'] + "/resources/fake-gcs-server-data:/data"],
- command='-scheme http -host fake-gcs-server')
+ command=f'-scheme http -host fake-gcs-server-{self.feature_context.id}')
logging.info('Added container \'%s\'', self.name)
diff --git a/docker/test/integration/cluster/containers/FlowContainer.py b/docker/test/integration/cluster/containers/FlowContainer.py
index 0ce1655c5..ff040f4bd 100644
--- a/docker/test/integration/cluster/containers/FlowContainer.py
+++ b/docker/test/integration/cluster/containers/FlowContainer.py
@@ -18,8 +18,14 @@ from .Container import Container
class FlowContainer(Container):
- def __init__(self, config_dir, name, engine, vols, network, image_store, command):
- super().__init__(name, engine, vols, network, image_store, command)
+ def __init__(self, feature_context, config_dir, name, engine, vols, network, image_store, command):
+ super().__init__(feature_context=feature_context,
+ name=name,
+ engine=engine,
+ vols=vols,
+ network=network,
+ image_store=image_store,
+ command=command)
self.start_nodes = []
self.config_dir = config_dir
self.controllers = []
diff --git a/docker/test/integration/cluster/containers/HttpProxyContainer.py b/docker/test/integration/cluster/containers/HttpProxyContainer.py
index 17ebed547..6f9859a3e 100644
--- a/docker/test/integration/cluster/containers/HttpProxyContainer.py
+++ b/docker/test/integration/cluster/containers/HttpProxyContainer.py
@@ -19,8 +19,8 @@ from .Container import Container
class HttpProxyContainer(Container):
- def __init__(self, name, vols, network, image_store, command=None):
- super().__init__(name, 'http-proxy', vols, network, image_store, command)
+ def __init__(self, feature_context, name, vols, network, image_store, command=None):
+ super().__init__(feature_context, name, 'http-proxy', vols, network, image_store, command)
def get_startup_finished_log_entry(self):
return "Accepting HTTP Socket connections at"
@@ -35,6 +35,5 @@ class HttpProxyContainer(Container):
detach=True,
name=self.name,
network=self.network.name,
- ports={'3128/tcp': 3128},
entrypoint=self.command)
logging.info('Added container \'%s\'', self.name)
diff --git a/docker/test/integration/cluster/containers/KafkaBrokerContainer.py b/docker/test/integration/cluster/containers/KafkaBrokerContainer.py
index 510c1a735..13062cd36 100644
--- a/docker/test/integration/cluster/containers/KafkaBrokerContainer.py
+++ b/docker/test/integration/cluster/containers/KafkaBrokerContainer.py
@@ -15,26 +15,76 @@
import logging
+import tempfile
+import os
+import docker.types
+import jks
+from OpenSSL import crypto
+
from .Container import Container
+from ssl_utils.SSL_cert_utils import make_server_cert
class KafkaBrokerContainer(Container):
- def __init__(self, name, vols, network, image_store, command=None):
- super().__init__(name, 'kafka-broker', vols, network, image_store, command)
+ def __init__(self, feature_context, name, vols, network, image_store, command=None):
+ super().__init__(feature_context, name, 'kafka-broker', vols, network, image_store, command)
+
+ kafka_cert, kafka_key = make_server_cert(f"kafka-broker-{feature_context.id}", feature_context.root_ca_cert, feature_context.root_ca_key)
+
+ pke = jks.PrivateKeyEntry.new('kafka-broker-cert', [crypto.dump_certificate(crypto.FILETYPE_ASN1, kafka_cert)], crypto.dump_privatekey(crypto.FILETYPE_ASN1, kafka_key), 'rsa_raw')
+ server_keystore = jks.KeyStore.new('jks', [pke])
+
+ self.server_keystore_file = tempfile.NamedTemporaryFile(delete=False)
+ server_keystore.save(self.server_keystore_file.name, 'abcdefgh')
+ self.server_keystore_file.close()
+
+ self.server_truststore_file = tempfile.NamedTemporaryFile(delete=False)
+ self.server_truststore_file.write(crypto.dump_certificate(crypto.FILETYPE_PEM, feature_context.root_ca_cert))
+ self.server_truststore_file.close()
+
+ self.server_properties_file = tempfile.NamedTemporaryFile(delete=False)
+ with open(os.environ['TEST_DIRECTORY'] + "/resources/kafka_broker/conf/server.properties") as server_properties_file:
+ server_properties_content = server_properties_file.read()
+ patched_server_properties_content = server_properties_content.replace("kafka-broker", f"kafka-broker-{feature_context.id}")
+ self.server_properties_file.write(patched_server_properties_content.encode())
+ self.server_properties_file.close()
+ os.chmod(self.server_properties_file.name, 0o644)
def get_startup_finished_log_entry(self):
- return "Kafka startTimeMs"
+ return "Recorded new controller, from now on will use broker kafka-broker"
def deploy(self):
if not self.set_deployed():
return
logging.info('Creating and running kafka broker docker container...')
+
self.client.containers.run(
- self.image_store.get_image(self.get_engine()),
+ image="ubuntu/kafka:3.1-22.04_beta",
detach=True,
- name='kafka-broker',
+ name=self.name,
network=self.network.name,
ports={'9092/tcp': 9092, '29092/tcp': 29092, '9093/tcp': 9093, '29093/tcp': 29093, '9094/tcp': 9094, '29094/tcp': 29094, '9094/tcp': 9094, '29095/tcp': 29095},
+ mounts=[
+ docker.types.Mount(
+ type='bind',
+ source=self.server_properties_file.name,
+ target='/opt/kafka/config/kraft/server.properties'
+ ),
+ docker.types.Mount(
+ type='bind',
+ source=self.server_properties_file.name,
+ target='/opt/kafka/config/server.properties'
+ ),
+ docker.types.Mount(
+ type='bind',
+ source=self.server_keystore_file.name,
+ target='/usr/local/etc/kafka/certs/server_keystore.jks'
+ ),
+ docker.types.Mount(
+ type='bind',
+ source=self.server_truststore_file.name,
+ target='/usr/local/etc/kafka/certs/server_truststore.pem'
+ )],
entrypoint=self.command)
logging.info('Added container \'%s\'', self.name)
diff --git a/docker/test/integration/cluster/containers/MinifiAsPodInKubernetesCluster.py b/docker/test/integration/cluster/containers/MinifiAsPodInKubernetesCluster.py
index b404a3234..beecb9a83 100644
--- a/docker/test/integration/cluster/containers/MinifiAsPodInKubernetesCluster.py
+++ b/docker/test/integration/cluster/containers/MinifiAsPodInKubernetesCluster.py
@@ -27,11 +27,18 @@ class MinifiAsPodInKubernetesCluster(MinifiContainer):
MINIFI_IMAGE_NAME = 'apacheminificpp'
MINIFI_IMAGE_TAG = 'docker_test'
- def __init__(self, kubernetes_proxy, config_dir, minifi_options, name, vols, network, image_store, command=None):
+ def __init__(self, feature_context, kubernetes_proxy, config_dir, minifi_options, name, vols, network, image_store, command=None):
test_dir = os.environ['TEST_DIRECTORY']
shutil.copy(os.path.join(test_dir, os.pardir, os.pardir, os.pardir, 'conf', 'minifi.properties'), config_dir)
shutil.copy(os.path.join(test_dir, 'resources', 'kubernetes', 'minifi-conf', 'minifi-log.properties'), config_dir)
- super().__init__(config_dir, minifi_options, name, vols, network, image_store, command)
+ super().__init__(feature_context=feature_context,
+ config_dir=config_dir,
+ options=minifi_options,
+ name=name,
+ vols=vols,
+ network=network,
+ image_store=image_store,
+ command=command)
self.kubernetes_proxy = kubernetes_proxy
diff --git a/docker/test/integration/cluster/containers/MinifiC2ServerContainer.py b/docker/test/integration/cluster/containers/MinifiC2ServerContainer.py
index 88d0991e8..15186dd5e 100644
--- a/docker/test/integration/cluster/containers/MinifiC2ServerContainer.py
+++ b/docker/test/integration/cluster/containers/MinifiC2ServerContainer.py
@@ -12,15 +12,47 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+import os
+import tempfile
+import docker.types
+import jks
from .Container import Container
+from OpenSSL import crypto
+from ssl_utils.SSL_cert_utils import make_server_cert
class MinifiC2ServerContainer(Container):
- def __init__(self, name, vols, network, image_store, command=None, ssl=False):
+ def __init__(self, feature_context, name, vols, network, image_store, command=None, ssl=False):
engine = "minifi-c2-server-ssl" if ssl else "minifi-c2-server"
- super().__init__(name, engine, vols, network, image_store, command)
+ super().__init__(feature_context, name, engine, vols, network, image_store, command)
+ self.ssl = ssl
+ if ssl:
+ c2_cert, c2_key = make_server_cert(f"minifi-c2-server-{feature_context.id}", feature_context.root_ca_cert, feature_context.root_ca_key)
+ pke = jks.PrivateKeyEntry.new('c2-server-cert', [crypto.dump_certificate(crypto.FILETYPE_ASN1, c2_cert)], crypto.dump_privatekey(crypto.FILETYPE_ASN1, c2_key), 'rsa_raw')
+ server_keystore = jks.KeyStore.new('jks', [pke])
+ self.server_keystore_file = tempfile.NamedTemporaryFile(delete=False)
+ server_keystore.save(self.server_keystore_file.name, 'abcdefgh')
+ os.chmod(self.server_keystore_file.name, 0o644)
+
+ self.server_truststore_file = tempfile.NamedTemporaryFile(delete=False)
+ pkcs12 = crypto.PKCS12()
+ pkcs12.set_privatekey(feature_context.root_ca_key)
+ pkcs12.set_certificate(feature_context.root_ca_cert)
+ pkcs12_data = pkcs12.export(passphrase="abcdefgh")
+ self.server_truststore_file.write(pkcs12_data)
+ self.server_truststore_file.close()
+ os.chmod(self.server_truststore_file.name, 0o644)
+
+ self.authorities_file = tempfile.NamedTemporaryFile(delete=False)
+ authorities_file_content = """
+CN=minifi-cpp-flow-{feature_id}:
+ - CLASS_MINIFI_CPP
+""".format(feature_id=self.feature_context.id)
+ self.authorities_file.write(authorities_file_content.encode())
+ self.authorities_file.close()
+ os.chmod(self.authorities_file.name, 0o644)
def get_startup_finished_log_entry(self):
return "Server Started"
@@ -29,9 +61,43 @@ class MinifiC2ServerContainer(Container):
if not self.set_deployed():
return
+ test_dir = os.environ['TEST_DIRECTORY'] # Based on DockerVerify.sh
+ mounts = [docker.types.Mount(
+ type='bind',
+ source=os.path.join(test_dir, "resources/minifi-c2-server/config-ssl.yml" if self.ssl else "resources/minifi-c2-server/config.yml"),
+ target='/opt/minifi-c2/minifi-c2-current/files/minifi-test-class/config.text.yml.v1'
+ )]
+
+ if self.ssl:
+ mounts.append(docker.types.Mount(
+ type='bind',
+ source=self.authorities_file.name,
+ target='/opt/minifi-c2/minifi-c2-current/conf/authorities.yaml'
+ ))
+ mounts.append(docker.types.Mount(
+ type='bind',
+ source=os.path.join(test_dir, "resources/minifi-c2-server/authorizations.yaml"),
+ target='/opt/minifi-c2/minifi-c2-current/conf/authorizations.yaml'
+ ))
+ mounts.append(docker.types.Mount(
+ type='bind',
+ source=os.path.join(test_dir, "resources/minifi-c2-server/c2.properties"),
+ target='/opt/minifi-c2/minifi-c2-current/conf/c2.properties'
+ ))
+ mounts.append(docker.types.Mount(
+ type='bind',
+ source=self.server_keystore_file.name,
+ target='/opt/minifi-c2/minifi-c2-current/certs/minifi-c2-server-keystore.jks'
+ ))
+ mounts.append(docker.types.Mount(
+ type='bind',
+ source=self.server_truststore_file.name,
+ target='/opt/minifi-c2/minifi-c2-current/certs/minifi-c2-server-truststore.p12'
+ ))
self.docker_container = self.client.containers.run(
- self.image_store.get_image(self.get_engine()),
+ image='apache/nifi-minifi-c2:1.21.0',
detach=True,
name=self.name,
network=self.network.name,
+ mounts=mounts,
entrypoint=self.command)
diff --git a/docker/test/integration/cluster/containers/MinifiContainer.py b/docker/test/integration/cluster/containers/MinifiContainer.py
index d9f47a254..de1fa6eed 100644
--- a/docker/test/integration/cluster/containers/MinifiContainer.py
+++ b/docker/test/integration/cluster/containers/MinifiContainer.py
@@ -16,9 +16,10 @@
import os
import logging
-import uuid
+import shortuuid
import shutil
import copy
+
from .FlowContainer import FlowContainer
from minifi.flow_serialization.Minifi_flow_yaml_serializer import Minifi_flow_yaml_serializer
from minifi.flow_serialization.Minifi_flow_json_serializer import Minifi_flow_json_serializer
@@ -42,15 +43,22 @@ class MinifiContainer(FlowContainer):
MINIFI_VERSION = os.environ['MINIFI_VERSION']
MINIFI_ROOT = '/opt/minifi/nifi-minifi-cpp-' + MINIFI_VERSION
- def __init__(self, config_dir, options, name, vols, network, image_store, command=None):
+ def __init__(self, feature_context, config_dir, options, name, vols, network, image_store, command=None):
self.options = options
- super().__init__(config_dir, name, 'minifi-cpp', copy.copy(vols), network, image_store, command)
+ super().__init__(feature_context=feature_context,
+ config_dir=config_dir,
+ name=name,
+ engine='minifi-cpp',
+ vols=copy.copy(vols),
+ network=network,
+ image_store=image_store,
+ command=command)
self.container_specific_config_dir = self._create_container_config_dir(self.config_dir)
os.chmod(self.container_specific_config_dir, 0o777)
def _create_container_config_dir(self, config_dir):
- container_config_dir = os.path.join(config_dir, str(uuid.uuid4()))
+ container_config_dir = os.path.join(config_dir, str(shortuuid.uuid()))
os.makedirs(container_config_dir)
for file_name in os.listdir(config_dir):
source = os.path.join(config_dir, file_name)
@@ -81,19 +89,19 @@ class MinifiContainer(FlowContainer):
with open(properties_file_path, 'a') as f:
if self.options.enable_c2:
f.write("nifi.c2.enable=true\n")
- f.write("nifi.c2.rest.url=http://minifi-c2-server:10090/c2/config/heartbeat\n")
- f.write("nifi.c2.rest.url.ack=http://minifi-c2-server:10090/c2/config/acknowledge\n")
- f.write("nifi.c2.flow.base.url=http://minifi-c2-server:10090/c2/config/\n")
+ f.write(f"nifi.c2.rest.url=http://minifi-c2-server-{self.feature_context.id}:10090/c2/config/heartbeat\n")
+ f.write(f"nifi.c2.rest.url.ack=http://minifi-c2-server-{self.feature_context.id}:10090/c2/config/acknowledge\n")
+ f.write(f"nifi.c2.flow.base.url=http://minifi-c2-server-{self.feature_context.id}:10090/c2/config/\n")
f.write("nifi.c2.root.classes=DeviceInfoNode,AgentInformation,FlowInformation\n")
f.write("nifi.c2.full.heartbeat=false\n")
f.write("nifi.c2.agent.class=minifi-test-class\n")
f.write("nifi.c2.agent.identifier=minifi-test-id\n")
elif self.options.enable_c2_with_ssl:
f.write("nifi.c2.enable=true\n")
- f.write("nifi.c2.rest.url=https://minifi-c2-server:10090/c2/config/heartbeat\n")
- f.write("nifi.c2.rest.url.ack=https://minifi-c2-server:10090/c2/config/acknowledge\n")
+ f.write(f"nifi.c2.rest.url=https://minifi-c2-server-{self.feature_context.id}:10090/c2/config/heartbeat\n")
+ f.write(f"nifi.c2.rest.url.ack=https://minifi-c2-server-{self.feature_context.id}:10090/c2/config/acknowledge\n")
f.write("nifi.c2.rest.ssl.context.service=SSLContextService\n")
- f.write("nifi.c2.flow.base.url=https://minifi-c2-server:10090/c2/config/\n")
+ f.write(f"nifi.c2.flow.base.url=https://minifi-c2-server-{self.feature_context.id}:10090/c2/config/\n")
f.write("nifi.c2.root.classes=DeviceInfoNode,AgentInformation,FlowInformation\n")
f.write("nifi.c2.full.heartbeat=false\n")
f.write("nifi.c2.agent.class=minifi-test-class\n")
@@ -101,10 +109,10 @@ class MinifiContainer(FlowContainer):
if self.options.set_ssl_context_properties:
f.write("nifi.remote.input.secure=true\n")
- f.write("nifi.security.client.certificate=/tmp/resources/minifi-c2-server-ssl/minifi-cpp-flow.crt\n")
- f.write("nifi.security.client.private.key=/tmp/resources/minifi-c2-server-ssl/minifi-cpp-flow.key\n")
+ f.write("nifi.security.client.certificate=/tmp/resources/minifi-cpp-flow.crt\n")
+ f.write("nifi.security.client.private.key=/tmp/resources/minifi-cpp-flow.key\n")
f.write("nifi.security.client.pass.phrase=abcdefgh\n")
- f.write("nifi.security.client.ca.certificate=/tmp/resources/minifi-c2-server-ssl/root-ca.pem\n")
+ f.write("nifi.security.client.ca.certificate=/tmp/resources/root_ca.crt\n")
if not self.options.enable_provenance:
f.write("nifi.provenance.repository.class.name=NoOpRepository\n")
@@ -116,7 +124,7 @@ class MinifiContainer(FlowContainer):
f.write("nifi.metrics.publisher.metrics=RepositoryMetrics,QueueMetrics,PutFileMetrics,processorMetrics/Get.*,FlowInformation,DeviceInfoNode,AgentStatus\n")
if self.options.use_flow_config_from_url:
- f.write("nifi.c2.flow.url=http://minifi-c2-server:10090/c2/config?class=minifi-test-class\n")
+ f.write(f"nifi.c2.flow.url=http://minifi-c2-server-{self.feature_context.id}:10090/c2/config?class=minifi-test-class\n")
if self.options.enable_controller_socket:
f.write("controller.socket.enable=true\n")
diff --git a/docker/test/integration/cluster/containers/MqttBrokerContainer.py b/docker/test/integration/cluster/containers/MqttBrokerContainer.py
index 3cc618adc..b33c41b7e 100644
--- a/docker/test/integration/cluster/containers/MqttBrokerContainer.py
+++ b/docker/test/integration/cluster/containers/MqttBrokerContainer.py
@@ -19,8 +19,8 @@ from .Container import Container
class MqttBrokerContainer(Container):
- def __init__(self, name, vols, network, image_store, command=None):
- super().__init__(name, 'mqtt-broker', vols, network, image_store, command)
+ def __init__(self, feature_context, name, vols, network, image_store, command=None):
+ super().__init__(feature_context, name, 'mqtt-broker', vols, network, image_store, command)
def get_startup_finished_log_entry(self):
return "mosquitto version [0-9\\.]+ running"
@@ -35,6 +35,5 @@ class MqttBrokerContainer(Container):
detach=True,
name=self.name,
network=self.network.name,
- ports={'1883/tcp': 1883},
entrypoint=self.command)
logging.info('Added container \'%s\'', self.name)
diff --git a/docker/test/integration/cluster/containers/NifiContainer.py b/docker/test/integration/cluster/containers/NifiContainer.py
index f4870bb31..164a99317 100644
--- a/docker/test/integration/cluster/containers/NifiContainer.py
+++ b/docker/test/integration/cluster/containers/NifiContainer.py
@@ -26,13 +26,13 @@ class NifiContainer(FlowContainer):
NIFI_VERSION = '1.20.0'
NIFI_ROOT = '/opt/nifi/nifi-' + NIFI_VERSION
- def __init__(self, config_dir, name, vols, network, image_store, command=None):
+ def __init__(self, feature_context, config_dir, name, vols, network, image_store, command=None):
if not command:
entry_command = (r"sed -i -e 's/^\(nifi.remote.input.host\)=.*/\1={name}/' {nifi_root}/conf/nifi.properties && "
r"sed -i -e 's/^\(nifi.sensitive.props.key\)=.*/\1=secret_key_12345/' {nifi_root}/conf/nifi.properties && "
r"cp /tmp/nifi_config/flow.xml.gz {nifi_root}/conf && /opt/nifi/scripts/start.sh").format(name=name, nifi_root=NifiContainer.NIFI_ROOT)
command = ["/bin/sh", "-c", entry_command]
- super().__init__(config_dir, name, 'nifi', vols, network, image_store, command)
+ super().__init__(feature_context, config_dir, name, 'nifi', vols, network, image_store, command)
def get_startup_finished_log_entry(self):
return "Starting Flow Controller"
diff --git a/docker/test/integration/cluster/containers/OPCUAServerContainer.py b/docker/test/integration/cluster/containers/OPCUAServerContainer.py
index 778349b79..b112e9c87 100644
--- a/docker/test/integration/cluster/containers/OPCUAServerContainer.py
+++ b/docker/test/integration/cluster/containers/OPCUAServerContainer.py
@@ -19,8 +19,8 @@ from .Container import Container
class OPCUAServerContainer(Container):
- def __init__(self, name, vols, network, image_store, command=None):
- super().__init__(name, 'opcua-server', vols, network, image_store, command)
+ def __init__(self, feature_context, name, vols, network, image_store, command=None):
+ super().__init__(feature_context, name, 'opcua-server', vols, network, image_store, command)
def get_startup_finished_log_entry(self):
return "TCP network layer listening on"
@@ -35,6 +35,5 @@ class OPCUAServerContainer(Container):
detach=True,
name=self.name,
network=self.network.name,
- ports={'4840/tcp': 4840},
entrypoint=self.command)
logging.info('Added container \'%s\'', self.name)
diff --git a/docker/test/integration/cluster/containers/OpensearchContainer.py b/docker/test/integration/cluster/containers/OpensearchContainer.py
index 07f462b71..06997a20a 100644
--- a/docker/test/integration/cluster/containers/OpensearchContainer.py
+++ b/docker/test/integration/cluster/containers/OpensearchContainer.py
@@ -15,12 +15,35 @@
import logging
+import os
+import tempfile
+
+import OpenSSL.crypto
+import docker.types
+
from .Container import Container
+from ssl_utils.SSL_cert_utils import make_server_cert
class OpensearchContainer(Container):
- def __init__(self, name, vols, network, image_store, command=None):
- super().__init__(name, 'opensearch', vols, network, image_store, command)
+ def __init__(self, feature_context, name, vols, network, image_store, command=None):
+ super().__init__(feature_context, name, 'opensearch', vols, network, image_store, command)
+ cert, key = make_server_cert(f"opensearch-{feature_context.id}", feature_context.root_ca_cert, feature_context.root_ca_key)
+
+ self.root_ca_file = tempfile.NamedTemporaryFile(delete=False)
+ self.root_ca_file.write(OpenSSL.crypto.dump_certificate(type=OpenSSL.crypto.FILETYPE_PEM, cert=feature_context.root_ca_cert))
+ self.root_ca_file.close()
+ os.chmod(self.root_ca_file.name, 0o644)
+
+ self.admin_pem = tempfile.NamedTemporaryFile(delete=False)
+ self.admin_pem.write(OpenSSL.crypto.dump_certificate(type=OpenSSL.crypto.FILETYPE_PEM, cert=cert))
+ self.admin_pem.close()
+ os.chmod(self.admin_pem.name, 0o644)
+
+ self.admin_key = tempfile.NamedTemporaryFile(delete=False)
+ self.admin_key.write(OpenSSL.crypto.dump_privatekey(type=OpenSSL.crypto.FILETYPE_PEM, pkey=key))
+ self.admin_key.close()
+ os.chmod(self.admin_key.name, 0o644)
def get_startup_finished_log_entry(self):
return 'Hot-reloading of audit configuration is enabled'
@@ -28,11 +51,30 @@ class OpensearchContainer(Container):
def deploy(self):
if not self.set_deployed():
return
+ mounts = [
+ docker.types.Mount(
+ type='bind',
+ source=os.environ['TEST_DIRECTORY'] + "/resources/opensearch/opensearch.yml",
+ target='/usr/share/opensearch/config/opensearch.yml'),
+ docker.types.Mount(
+ type='bind',
+ source=self.admin_pem.name,
+ target='/usr/share/opensearch/config/admin.pem'),
+ docker.types.Mount(
+ type='bind',
+ source=self.admin_key.name,
+ target='/usr/share/opensearch/config/admin-key.pem'),
+ docker.types.Mount(
+ type='bind',
+ source=self.root_ca_file.name,
+ target='/usr/share/opensearch/config/root-ca.pem')]
logging.info('Creating and running Opensearch docker container...')
self.client.containers.run(
- self.image_store.get_image(self.get_engine()),
+ image='opensearchproject/opensearch:2.6.0',
detach=True,
name=self.name,
- network=self.network.name)
+ network=self.network.name,
+ mounts=mounts)
+
logging.info('Added container \'%s\'', self.name)
diff --git a/docker/test/integration/cluster/containers/PostgreSQLServerContainer.py b/docker/test/integration/cluster/containers/PostgreSQLServerContainer.py
index 664095f05..d0ebed9c7 100644
--- a/docker/test/integration/cluster/containers/PostgreSQLServerContainer.py
+++ b/docker/test/integration/cluster/containers/PostgreSQLServerContainer.py
@@ -18,8 +18,8 @@ from .Container import Container
class PostgreSQLServerContainer(Container):
- def __init__(self, name, vols, network, image_store, command=None):
- super().__init__(name, 'postgresql-server', vols, network, image_store, command)
+ def __init__(self, feature_context, name, vols, network, image_store, command=None):
+ super().__init__(feature_context, name, 'postgresql-server', vols, network, image_store, command)
def get_startup_finished_log_entry(self):
return "database system is ready to accept connections"
@@ -31,7 +31,7 @@ class PostgreSQLServerContainer(Container):
self.docker_container = self.client.containers.run(
self.image_store.get_image(self.get_engine()),
detach=True,
- name='postgresql-server',
+ name=self.name,
network=self.network.name,
environment=["POSTGRES_PASSWORD=password"],
entrypoint=self.command)
diff --git a/docker/test/integration/cluster/containers/PrometheusContainer.py b/docker/test/integration/cluster/containers/PrometheusContainer.py
index 7ba2068e9..a5f1fc430 100644
--- a/docker/test/integration/cluster/containers/PrometheusContainer.py
+++ b/docker/test/integration/cluster/containers/PrometheusContainer.py
@@ -13,12 +13,28 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
+import os
+import tempfile
+import docker.types
from .Container import Container
class PrometheusContainer(Container):
- def __init__(self, name, vols, network, image_store, command=None):
- super().__init__(name, 'prometheus', vols, network, image_store, command)
+ def __init__(self, feature_context, name, vols, network, image_store, command=None):
+ super().__init__(feature_context, name, 'prometheus', vols, network, image_store, command)
+ prometheus_yml_content = """
+global:
+ scrape_interval: 2s
+ evaluation_interval: 15s
+scrape_configs:
+ - job_name: "minifi"
+ static_configs:
+ - targets: ["minifi-cpp-flow-{feature_id}:9936"]
+""".format(feature_id=self.feature_context.id)
+ self.yaml_file = tempfile.NamedTemporaryFile(delete=False)
+ self.yaml_file.write(prometheus_yml_content.encode())
+ self.yaml_file.close()
+ os.chmod(self.yaml_file.name, 0o644)
def get_startup_finished_log_entry(self):
return "Server is ready to receive web requests."
@@ -28,11 +44,16 @@ class PrometheusContainer(Container):
return
logging.info('Creating and running Prometheus docker container...')
+
self.client.containers.run(
- self.image_store.get_image(self.get_engine()),
+ image="prom/prometheus:v2.35.0",
detach=True,
name=self.name,
network=self.network.name,
ports={'9090/tcp': 9090},
+ mounts=[docker.types.Mount(
+ type='bind',
+ source=self.yaml_file.name,
+ target='/etc/prometheus/prometheus.yml'
+ )],
entrypoint=self.command)
- logging.info('Added container \'%s\'', self.name)
diff --git a/docker/test/integration/cluster/containers/S3ServerContainer.py b/docker/test/integration/cluster/containers/S3ServerContainer.py
index 6fe39c188..c00082a31 100644
--- a/docker/test/integration/cluster/containers/S3ServerContainer.py
+++ b/docker/test/integration/cluster/containers/S3ServerContainer.py
@@ -19,8 +19,8 @@ from .Container import Container
class S3ServerContainer(Container):
- def __init__(self, name, vols, network, image_store, command):
- super().__init__(name, 's3-server', vols, network, image_store, command)
+ def __init__(self, feature_context, name, vols, network, image_store, command):
+ super().__init__(feature_context, name, 's3-server', vols, network, image_store, command)
def get_startup_finished_log_entry(self):
return "Started S3MockApplication"
@@ -35,7 +35,6 @@ class S3ServerContainer(Container):
detach=True,
name=self.name,
network=self.network.name,
- ports={'9090/tcp': 9090, '9191/tcp': 9191},
environment=["initialBuckets=test_bucket"],
entrypoint=self.command)
logging.info('Added container \'%s\'', self.name)
diff --git a/docker/test/integration/cluster/containers/SplunkContainer.py b/docker/test/integration/cluster/containers/SplunkContainer.py
index ea18da435..9a5734276 100644
--- a/docker/test/integration/cluster/containers/SplunkContainer.py
+++ b/docker/test/integration/cluster/containers/SplunkContainer.py
@@ -19,8 +19,8 @@ from .Container import Container
class SplunkContainer(Container):
- def __init__(self, name, vols, network, image_store, command=None):
- super().__init__(name, 'splunk', vols, network, image_store, command)
+ def __init__(self, feature_context, name, vols, network, image_store, command=None):
+ super().__init__(feature_context, name, 'splunk', vols, network, image_store, command)
def get_startup_finished_log_entry(self):
return "Ansible playbook complete, will begin streaming splunkd_stderr.log"
diff --git a/docker/test/integration/cluster/containers/SyslogTcpClientContainer.py b/docker/test/integration/cluster/containers/SyslogTcpClientContainer.py
index 699676f77..d34757124 100644
--- a/docker/test/integration/cluster/containers/SyslogTcpClientContainer.py
+++ b/docker/test/integration/cluster/containers/SyslogTcpClientContainer.py
@@ -17,8 +17,8 @@ from .Container import Container
class SyslogTcpClientContainer(Container):
- def __init__(self, name, vols, network, image_store, command=None):
- super().__init__(name, 'syslog-tcp-client', vols, network, image_store, command)
+ def __init__(self, feature_context, name, vols, network, image_store, command=None):
+ super().__init__(feature_context, name, 'syslog-tcp-client', vols, network, image_store, command)
def get_startup_finished_log_entry(self):
return "Syslog TCP client started"
@@ -33,5 +33,6 @@ class SyslogTcpClientContainer(Container):
detach=True,
name=self.name,
network=self.network.name,
- entrypoint='/bin/bash -c "echo Syslog TCP client started; while true; do logger --tcp -n minifi-cpp-flow -P 514 sample_log; sleep 1; done"')
+ entrypoint='/bin/bash -c "echo Syslog TCP client started; while true; do logger --tcp -n '
+ f'minifi-cpp-flow-{self.feature_context.id} -P 514 sample_log; sleep 1; done"')
logging.info('Added container \'%s\'', self.name)
diff --git a/docker/test/integration/cluster/containers/SyslogUdpClientContainer.py b/docker/test/integration/cluster/containers/SyslogUdpClientContainer.py
index f1f4a9bfd..409c33fda 100644
--- a/docker/test/integration/cluster/containers/SyslogUdpClientContainer.py
+++ b/docker/test/integration/cluster/containers/SyslogUdpClientContainer.py
@@ -17,8 +17,8 @@ from .Container import Container
class SyslogUdpClientContainer(Container):
- def __init__(self, name, vols, network, image_store, command=None):
- super().__init__(name, 'syslog-udp-client', vols, network, image_store, command)
+ def __init__(self, feature_context, name, vols, network, image_store, command=None):
+ super().__init__(feature_context, name, 'syslog-udp-client', vols, network, image_store, command)
def get_startup_finished_log_entry(self):
return "Syslog UDP client started"
@@ -33,5 +33,6 @@ class SyslogUdpClientContainer(Container):
detach=True,
name=self.name,
network=self.network.name,
- entrypoint='/bin/bash -c "echo Syslog UDP client started; while true; do logger --udp -n minifi-cpp-flow -P 514 sample_log; sleep 1; done"')
+ entrypoint='/bin/bash -c "echo Syslog UDP client started; while true; do logger --udp -n '
+ f'minifi-cpp-flow-{self.feature_context.id} -P 514 sample_log; sleep 1; done"')
logging.info('Added container \'%s\'', self.name)
diff --git a/docker/test/integration/cluster/containers/TcpClientContainer.py b/docker/test/integration/cluster/containers/TcpClientContainer.py
index c29c00fa2..dc27e53e4 100644
--- a/docker/test/integration/cluster/containers/TcpClientContainer.py
+++ b/docker/test/integration/cluster/containers/TcpClientContainer.py
@@ -17,8 +17,8 @@ from .Container import Container
class TcpClientContainer(Container):
- def __init__(self, name, vols, network, image_store, command=None):
- super().__init__(name, 'tcp-client', vols, network, image_store, command)
+ def __init__(self, feature_context, name, vols, network, image_store, command=None):
+ super().__init__(feature_context, name, 'tcp-client', vols, network, image_store, command)
def get_startup_finished_log_entry(self):
return "TCP client container started"
@@ -29,9 +29,11 @@ class TcpClientContainer(Container):
logging.info('Creating and running a tcp client docker container...')
self.client.containers.run(
- self.image_store.get_image(self.get_engine()),
+ "alpine:3.17.3",
detach=True,
name=self.name,
network=self.network.name,
- entrypoint=self.command)
+ entrypoint='/bin/sh',
+ command="-c 'apk add netcat-openbsd && echo TCP client container started; while true; do echo "
+ f"test_tcp_message | nc minifi-cpp-flow-{self.feature_context.id} 10254; sleep 1; done'")
logging.info('Added container \'%s\'', self.name)
diff --git a/docker/test/integration/cluster/containers/ZookeeperContainer.py b/docker/test/integration/cluster/containers/ZookeeperContainer.py
index e8d0061dd..55829c78f 100644
--- a/docker/test/integration/cluster/containers/ZookeeperContainer.py
+++ b/docker/test/integration/cluster/containers/ZookeeperContainer.py
@@ -19,8 +19,8 @@ from .Container import Container
class ZookeeperContainer(Container):
- def __init__(self, name, vols, network, image_store, command):
- super().__init__(name, 'zookeeper', vols, network, image_store, command)
+ def __init__(self, feature_context, name, vols, network, image_store, command):
+ super().__init__(feature_context, name, 'zookeeper', vols, network, image_store, command)
def get_startup_finished_log_entry(self):
return "binding to port"
@@ -31,9 +31,9 @@ class ZookeeperContainer(Container):
logging.info('Creating and running zookeeper docker container...')
self.client.containers.run(
- "zookeeper:3.8.1",
+ image="zookeeper:3.8.1",
detach=True,
- name='zookeeper',
+ name=self.name,
network=self.network.name,
ports={'2181/tcp': 2181},
entrypoint=self.command)
diff --git a/docker/test/integration/features/CMakeLists.txt b/docker/test/integration/features/CMakeLists.txt
deleted file mode 100644
index 93abb54cd..000000000
--- a/docker/test/integration/features/CMakeLists.txt
+++ /dev/null
@@ -1,117 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set(ENABLED_BEHAVE_TESTS "")
-
-if (ENABLE_LIBRDKAFKA)
- list(APPEND ENABLED_BEHAVE_TESTS "${CMAKE_CURRENT_SOURCE_DIR}/kafka.feature")
-endif()
-
-list(APPEND ENABLED_BEHAVE_TESTS "${CMAKE_CURRENT_SOURCE_DIR}/attributes_to_json.feature")
-list(APPEND ENABLED_BEHAVE_TESTS "${CMAKE_CURRENT_SOURCE_DIR}/core_functionality.feature")
-list(APPEND ENABLED_BEHAVE_TESTS "${CMAKE_CURRENT_SOURCE_DIR}/defragtextflowfiles.feature")
-list(APPEND ENABLED_BEHAVE_TESTS "${CMAKE_CURRENT_SOURCE_DIR}/file_system_operations.feature")
-list(APPEND ENABLED_BEHAVE_TESTS "${CMAKE_CURRENT_SOURCE_DIR}/hashcontent.feature")
-list(APPEND ENABLED_BEHAVE_TESTS "${CMAKE_CURRENT_SOURCE_DIR}/replace_text.feature")
-list(APPEND ENABLED_BEHAVE_TESTS "${CMAKE_CURRENT_SOURCE_DIR}/routetext.feature")
-list(APPEND ENABLED_BEHAVE_TESTS "${CMAKE_CURRENT_SOURCE_DIR}/s2s.feature")
-list(APPEND ENABLED_BEHAVE_TESTS "${CMAKE_CURRENT_SOURCE_DIR}/syslog_listener.feature")
-list(APPEND ENABLED_BEHAVE_TESTS "${CMAKE_CURRENT_SOURCE_DIR}/network_listener.feature")
-list(APPEND ENABLED_BEHAVE_TESTS "${CMAKE_CURRENT_SOURCE_DIR}/minifi_c2_server.feature")
-
-if (ENABLE_AZURE)
- list(APPEND ENABLED_BEHAVE_TESTS "${CMAKE_CURRENT_SOURCE_DIR}/azure_storage.feature")
-endif()
-
-if (ENABLE_GCP)
- list(APPEND ENABLED_BEHAVE_TESTS "${CMAKE_CURRENT_SOURCE_DIR}/google_cloud_storage.feature")
-endif()
-
-if (NOT DISABLE_CIVET)
- list(APPEND ENABLED_BEHAVE_TESTS "${CMAKE_CURRENT_SOURCE_DIR}/http.feature")
- list(APPEND ENABLED_BEHAVE_TESTS "${CMAKE_CURRENT_SOURCE_DIR}/https.feature")
-endif()
-
-if (ENABLE_KUBERNETES)
- list(APPEND ENABLED_BEHAVE_TESTS "${CMAKE_CURRENT_SOURCE_DIR}/kubernetes_logging.feature")
- list(APPEND ENABLED_BEHAVE_TESTS "${CMAKE_CURRENT_SOURCE_DIR}/kubernetes_metrics.feature")
-endif()
-
-if (ENABLE_MQTT)
- list(APPEND ENABLED_BEHAVE_TESTS "${CMAKE_CURRENT_SOURCE_DIR}/mqtt.feature")
-endif()
-
-if (ENABLE_OPC)
- list(APPEND ENABLED_BEHAVE_TESTS "${CMAKE_CURRENT_SOURCE_DIR}/opcua.feature")
-endif()
-
-if (ENABLE_PYTHON_SCRIPTING)
- list(APPEND ENABLED_BEHAVE_TESTS "${CMAKE_CURRENT_SOURCE_DIR}/python.feature")
- list(APPEND ENABLED_BEHAVE_TESTS "${CMAKE_CURRENT_SOURCE_DIR}/python_script.feature")
-endif()
-
-if (ENABLE_LUA_SCRIPTING)
- list(APPEND ENABLED_BEHAVE_TESTS "${CMAKE_CURRENT_SOURCE_DIR}/lua_script.feature")
-endif()
-
-if (ENABLE_AWS)
- list(APPEND ENABLED_BEHAVE_TESTS "${CMAKE_CURRENT_SOURCE_DIR}/s3.feature")
-endif()
-
-if (ENABLE_SPLUNK)
- list(APPEND ENABLED_BEHAVE_TESTS "${CMAKE_CURRENT_SOURCE_DIR}/splunk.feature")
-endif()
-
-if (ENABLE_SQL)
- list(APPEND ENABLED_BEHAVE_TESTS "${CMAKE_CURRENT_SOURCE_DIR}/sql.feature")
-endif()
-
-if (ENABLE_ELASTICSEARCH AND NOT CI_BUILD) # Elasticsearch/Opensearch containers require more RAM than what the CI environment has
- list(APPEND ENABLED_BEHAVE_TESTS "${CMAKE_CURRENT_SOURCE_DIR}/elasticsearch.feature")
- list(APPEND ENABLED_BEHAVE_TESTS "${CMAKE_CURRENT_SOURCE_DIR}/opensearch.feature")
-endif()
-
-if (ENABLE_PROMETHEUS)
- list(APPEND ENABLED_BEHAVE_TESTS "${CMAKE_CURRENT_SOURCE_DIR}/prometheus.feature")
-endif()
-
-if (NOT DISABLE_CURL AND NOT DISABLE_CONTROLLER)
- list(APPEND ENABLED_BEHAVE_TESTS "${CMAKE_CURRENT_SOURCE_DIR}/minifi_controller.feature")
-endif()
-
-file(GLOB FEATURE_FILES "*.feature")
-
-list(LENGTH ENABLED_BEHAVE_TESTS ENABLED_BEHAVE_TESTS_LENGTH)
-
-math(EXPR ENABLED_BEHAVE_TESTS_QUADRANT_LENGTH ${ENABLED_BEHAVE_TESTS_LENGTH}/4)
-
-math(EXPR ENABLED_BEHAVE_TESTS_SECOND_QUADRANT_START ${ENABLED_BEHAVE_TESTS_QUADRANT_LENGTH})
-math(EXPR ENABLED_BEHAVE_TESTS_THIRD_QUADRANT_START ${ENABLED_BEHAVE_TESTS_QUADRANT_LENGTH}*2)
-math(EXPR ENABLED_BEHAVE_TESTS_LAST_QUADRANT_START ${ENABLED_BEHAVE_TESTS_QUADRANT_LENGTH}*3)
-
-
-list(SUBLIST ENABLED_BEHAVE_TESTS 0 ${ENABLED_BEHAVE_TESTS_QUADRANT_LENGTH} ENABLED_BEHAVE_TESTS_FIRST_QUADRANT)
-list(SUBLIST ENABLED_BEHAVE_TESTS ${ENABLED_BEHAVE_TESTS_SECOND_QUADRANT_START} ${ENABLED_BEHAVE_TESTS_QUADRANT_LENGTH} ENABLED_BEHAVE_TESTS_SECOND_QUADRANT)
-list(SUBLIST ENABLED_BEHAVE_TESTS ${ENABLED_BEHAVE_TESTS_THIRD_QUADRANT_START} ${ENABLED_BEHAVE_TESTS_QUADRANT_LENGTH} ENABLED_BEHAVE_TESTS_THIRD_QUADRANT)
-list(SUBLIST ENABLED_BEHAVE_TESTS ${ENABLED_BEHAVE_TESTS_LAST_QUADRANT_START} -1 ENABLED_BEHAVE_TESTS_LAST_QUADRANT)
-
-set(ENABLED_BEHAVE_TESTS ${ENABLED_BEHAVE_TESTS} PARENT_SCOPE)
-set(ENABLED_BEHAVE_TESTS_FIRST_QUADRANT ${ENABLED_BEHAVE_TESTS_FIRST_QUADRANT} PARENT_SCOPE)
-set(ENABLED_BEHAVE_TESTS_SECOND_QUADRANT ${ENABLED_BEHAVE_TESTS_SECOND_QUADRANT} PARENT_SCOPE)
-set(ENABLED_BEHAVE_TESTS_THIRD_QUADRANT ${ENABLED_BEHAVE_TESTS_THIRD_QUADRANT} PARENT_SCOPE)
-set(ENABLED_BEHAVE_TESTS_LAST_QUADRANT ${ENABLED_BEHAVE_TESTS_LAST_QUADRANT} PARENT_SCOPE)
-set(ALL_BEHAVE_TESTS ${FEATURE_FILES} PARENT_SCOPE)
diff --git a/docker/test/integration/MiNiFi_integration_test_driver.py b/docker/test/integration/features/MiNiFi_integration_test_driver.py
similarity index 86%
rename from docker/test/integration/MiNiFi_integration_test_driver.py
rename to docker/test/integration/features/MiNiFi_integration_test_driver.py
index bcaddb33b..dd8c3cc18 100644
--- a/docker/test/integration/MiNiFi_integration_test_driver.py
+++ b/docker/test/integration/features/MiNiFi_integration_test_driver.py
@@ -18,8 +18,11 @@ import logging
import time
import uuid
+import OpenSSL.crypto
+
from pydoc import locate
+from ssl_utils.SSL_cert_utils import make_ca, make_client_cert
from minifi.core.InputPort import InputPort
from cluster.DockerTestCluster import DockerTestCluster
@@ -37,9 +40,9 @@ from utils import decode_escaped_str, get_minifi_pid, get_peak_memory_usage
class MiNiFi_integration_test:
- def __init__(self, context):
- self.test_id = context.test_id
- self.cluster = DockerTestCluster(context)
+ def __init__(self, context, feature_id: str):
+ self.feature_id = feature_id
+ self.cluster = DockerTestCluster(context, feature_id=feature_id)
self.connectable_nodes = []
# Remote process groups are not connectables
@@ -47,38 +50,58 @@ class MiNiFi_integration_test:
self.file_system_observer = None
self.docker_directory_bindings = context.directory_bindings
- self.cluster.set_directory_bindings(self.docker_directory_bindings.get_directory_bindings(self.test_id), self.docker_directory_bindings.get_data_directories(self.test_id))
+ self.cluster.set_directory_bindings(self.docker_directory_bindings.get_directory_bindings(self.feature_id), self.docker_directory_bindings.get_data_directories(self.feature_id))
+ self.root_ca_cert, self.root_ca_key = make_ca("root CA")
+
+ minifi_client_cert, minifi_client_key = make_client_cert(common_name=f"minifi-cpp-flow-{self.feature_id}",
+ ca_cert=self.root_ca_cert,
+ ca_key=self.root_ca_key)
+ self.put_test_resource('root_ca.crt',
+ OpenSSL.crypto.dump_certificate(type=OpenSSL.crypto.FILETYPE_PEM,
+ cert=self.root_ca_cert))
+
+ self.put_test_resource('minifi_client.crt',
+ OpenSSL.crypto.dump_certificate(type=OpenSSL.crypto.FILETYPE_PEM,
+ cert=minifi_client_cert))
+ self.put_test_resource('minifi_client.key',
+ OpenSSL.crypto.dump_privatekey(type=OpenSSL.crypto.FILETYPE_PEM,
+ pkey=minifi_client_key))
+
+ def get_container_name_with_postfix(self, container_name: str):
+ return self.cluster.container_store.get_container_name_with_postfix(container_name)
def cleanup(self):
self.cluster.cleanup()
+ if self.file_system_observer:
+ self.file_system_observer.observer.unschedule_all()
- def acquire_container(self, name, engine='minifi-cpp', command=None):
- return self.cluster.acquire_container(name, engine, command)
+ def acquire_container(self, context, name, engine='minifi-cpp', command=None):
+ return self.cluster.acquire_container(context=context, name=name, engine=engine, command=command)
- def start_kafka_broker(self):
- self.cluster.acquire_container('kafka-broker', 'kafka-broker')
- self.cluster.deploy_container('zookeeper')
- self.cluster.deploy_container('kafka-broker')
+ def start_kafka_broker(self, context):
+ self.cluster.acquire_container(context=context, name='kafka-broker', engine='kafka-broker')
+ self.cluster.deploy_container(name='zookeeper')
+ self.cluster.deploy_container(name='kafka-broker')
assert self.cluster.wait_for_container_startup_to_finish('kafka-broker')
- def start_splunk(self):
- self.cluster.acquire_container('splunk', 'splunk')
- self.cluster.deploy_container('splunk')
+ def start_splunk(self, context):
+ self.cluster.acquire_container(context=context, name='splunk', engine='splunk')
+ self.cluster.deploy_container(name='splunk')
assert self.cluster.wait_for_container_startup_to_finish('splunk')
assert self.cluster.enable_splunk_hec_indexer('splunk', 'splunk_hec_token')
- def start_elasticsearch(self):
- self.cluster.acquire_container('elasticsearch', 'elasticsearch')
+ def start_elasticsearch(self, context):
+ self.cluster.acquire_container(context=context, name='elasticsearch', engine='elasticsearch')
self.cluster.deploy_container('elasticsearch')
assert self.cluster.wait_for_container_startup_to_finish('elasticsearch')
- def start_opensearch(self):
- self.cluster.acquire_container('opensearch', 'opensearch')
+ def start_opensearch(self, context):
+ self.cluster.acquire_container(context=context, name='opensearch', engine='opensearch')
self.cluster.deploy_container('opensearch')
assert self.cluster.wait_for_container_startup_to_finish('opensearch')
- def start_minifi_c2_server(self):
- self.cluster.acquire_container("minifi-c2-server", "minifi-c2-server")
+ def start_minifi_c2_server(self, context):
+ self.cluster.acquire_container(context=context, name="minifi-c2-server", engine="minifi-c2-server")
self.cluster.deploy_container('minifi-c2-server')
assert self.cluster.wait_for_container_startup_to_finish('minifi-c2-server')
@@ -147,13 +170,13 @@ class MiNiFi_integration_test:
if file_name is None:
file_name = str(uuid.uuid4())
test_data = decode_escaped_str(test_data)
- self.docker_directory_bindings.put_file_to_docker_path(self.test_id, path, file_name, test_data.encode('utf-8'))
+ self.docker_directory_bindings.put_file_to_docker_path(self.feature_id, path, file_name, test_data.encode('utf-8'))
def put_test_resource(self, file_name, contents):
- self.docker_directory_bindings.put_test_resource(self.test_id, file_name, contents)
+ self.docker_directory_bindings.put_test_resource(self.feature_id, file_name, contents)
def rm_out_child(self):
- self.docker_directory_bindings.rm_out_child(self.test_id)
+ self.docker_directory_bindings.rm_out_child(self.feature_id)
def add_file_system_observer(self, file_system_observer):
self.file_system_observer = file_system_observer
diff --git a/docker/test/integration/README.md b/docker/test/integration/features/README.md
similarity index 100%
rename from docker/test/integration/README.md
rename to docker/test/integration/features/README.md
diff --git a/docker/test/integration/features/attributes_to_json.feature b/docker/test/integration/features/attributes_to_json.feature
index 5d63f8fd4..87138993d 100644
--- a/docker/test/integration/features/attributes_to_json.feature
+++ b/docker/test/integration/features/attributes_to_json.feature
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+@CORE
Feature: Writing attribute data using AttributesToJSON processor
Background:
Given the content of "/tmp/output" is monitored
diff --git a/docker/test/integration/features/azure_storage.feature b/docker/test/integration/features/azure_storage.feature
index c18e29be9..9338da83f 100644
--- a/docker/test/integration/features/azure_storage.feature
+++ b/docker/test/integration/features/azure_storage.feature
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+@ENABLE_AZURE
Feature: Sending data from MiNiFi-C++ to an Azure storage server
In order to transfer data to interact with Azure servers
As a user of MiNiFi
diff --git a/docker/test/integration/features/core_functionality.feature b/docker/test/integration/features/core_functionality.feature
index b6682d5ed..e43a6cb6f 100644
--- a/docker/test/integration/features/core_functionality.feature
+++ b/docker/test/integration/features/core_functionality.feature
@@ -19,6 +19,7 @@ Feature: Core flow functionalities
Background:
Given the content of "/tmp/output" is monitored
+ @CORE
Scenario: A funnel can merge multiple connections from different processors
Given a GenerateFlowFile processor with the name "Generate1" and the "Custom Text" property set to "first_custom_text"
And the "Data Format" property of the Generate1 processor is set to "Text"
@@ -38,25 +39,28 @@ Feature: Core flow functionalities
Then at least one flowfile with the content "first_custom_text" is placed in the monitored directory in less than 20 seconds
And at least one flowfile with the content "second_custom_text" is placed in the monitored directory in less than 20 seconds
+ @CORE
Scenario: The default configuration uses RocksDB for both the flow file and content repositories
Given a GenerateFlowFile processor with the "Data Format" property set to "Text"
When the MiNiFi instance starts up
Then the Minifi logs contain the following message: "Using plaintext FlowFileRepository" in less than 5 seconds
And the Minifi logs contain the following message: "Using plaintext DatabaseContentRepository" in less than 1 second
- @requires_test_processors
+ @ENABLE_TEST_PROCESSORS
Scenario: Processors are destructed when agent is stopped
Given a transient MiNiFi flow with the name "transient-minifi" is set up
And a LogOnDestructionProcessor processor with the name "logOnDestruction" in the "transient-minifi" flow
When the MiNiFi instance starts up
Then the Minifi logs contain the following message: "LogOnDestructionProcessor is being destructed" in less than 100 seconds
+ @CORE
Scenario: Agent does not crash when using provenance repositories
Given a GenerateFlowFile processor with the name "generateFlowFile" in the "minifi-cpp-with-provenance-repo" flow
And the provenance repository is enabled in MiNiFi
When the MiNiFi instance starts up
- Then the "minifi-cpp-with-provenance-repo" flow has a log line matching "MiNiFi started" in less than 30 seconds
+ Then the "minifi-cpp-with-provenance-repo-${feature_id}" flow has a log line matching "MiNiFi started" in less than 30 seconds
+ @SKIP_CI
Scenario: Memory usage returns after peak usage
Given a GenerateFlowFile processor with the "Batch Size" property set to "50000"
And the "Data Format" property of the GenerateFlowFile processor is set to "Text"
diff --git a/docker/test/integration/features/defragtextflowfiles.feature b/docker/test/integration/features/defragtextflowfiles.feature
index 65fd31664..44eb0cf60 100644
--- a/docker/test/integration/features/defragtextflowfiles.feature
+++ b/docker/test/integration/features/defragtextflowfiles.feature
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+@CORE
Feature: DefragmentText can defragment fragmented data from TailFile
Background:
Given the content of "/tmp/output" is monitored
diff --git a/docker/test/integration/features/elasticsearch.feature b/docker/test/integration/features/elasticsearch.feature
index 995d47544..182e016e2 100644
--- a/docker/test/integration/features/elasticsearch.feature
+++ b/docker/test/integration/features/elasticsearch.feature
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+@ENABLE_ELASTICSEARCH
Feature: Managing documents on Elasticsearch with PostElasticsearch
Background:
diff --git a/docker/test/integration/environment.py b/docker/test/integration/features/environment.py
similarity index 63%
rename from docker/test/integration/environment.py
rename to docker/test/integration/features/environment.py
index 9e3927ea6..489c25ada 100644
--- a/docker/test/integration/environment.py
+++ b/docker/test/integration/features/environment.py
@@ -17,16 +17,27 @@
import logging
import datetime
import sys
-import uuid
+import shortuuid
import os
import platform
+
sys.path.append('../minifi')
from MiNiFi_integration_test_driver import MiNiFi_integration_test # noqa: E402
from minifi import * # noqa
-from cluster.ImageStore import ImageStore # noqa
-from cluster.DockerTestDirectoryBindings import DockerTestDirectoryBindings # noqa
-from cluster.KubernetesProxy import KubernetesProxy # noqa
+from cluster.ImageStore import ImageStore # noqa
+from cluster.DockerTestDirectoryBindings import DockerTestDirectoryBindings # noqa
+from cluster.KubernetesProxy import KubernetesProxy # noqa
+
+
+def inject_feature_id(context, step):
+ if "${feature_id}" in step.name:
+ step.name = step.name.replace("${feature_id}", context.feature_id)
+ if step.table:
+ for row in step.table:
+ for i in range(len(row.cells)):
+ if "${feature_id}" in row.cells[i]:
+ row.cells[i] = row.cells[i].replace("${feature_id}", context.feature_id)
def before_scenario(context, scenario):
@@ -34,16 +45,14 @@ def before_scenario(context, scenario):
scenario.skip("Marked with @skip")
return
- if "requires_test_processors" in scenario.effective_tags and context.config.userdata.get('test_processors') != "ON":
- scenario.skip("Test processors are not available, skipping test scenario")
- return
-
logging.info("Integration test setup at {time:%H:%M:%S.%f}".format(time=datetime.datetime.now()))
- context.test = MiNiFi_integration_test(context)
+ context.test = MiNiFi_integration_test(context=context, feature_id=context.feature_id)
+ for step in scenario.steps:
+ inject_feature_id(context, step)
def after_scenario(context, scenario):
- if scenario.status == "skipped":
+ if "skip" in scenario.effective_tags:
logging.info("Scenario was skipped, no need for clean up.")
return
@@ -55,29 +64,29 @@ def after_scenario(context, scenario):
def before_all(context):
- context.test_id = str(uuid.uuid4())
context.config.setup_logging()
context.image_store = ImageStore()
- context.directory_bindings = DockerTestDirectoryBindings(context.test_id)
- context.directory_bindings.create_new_data_directories()
context.kubernetes_proxy = None
-def before_tag(context, tag):
- if tag == "requires.kubernetes.cluster":
- context.kubernetes_proxy = KubernetesProxy(context.directory_bindings.get_data_directories(context.test_id)["kubernetes_temp_dir"], os.path.join(os.environ['TEST_DIRECTORY'], 'resources', 'kubernetes', 'pods-etc'))
- context.kubernetes_proxy.create_config(context.directory_bindings.get_directory_bindings(context.test_id))
- context.kubernetes_proxy.start_cluster()
-
-
-def after_tag(context, tag):
- if tag == "requires.kubernetes.cluster" and context.kubernetes_proxy:
- context.kubernetes_proxy.cleanup()
- context.kubernetes_proxy = None
-
-
def before_feature(context, feature):
if "x86_x64_only" in feature.tags:
is_x86 = platform.machine() in ("i386", "AMD64", "x86_64")
if not is_x86:
feature.skip("This feature is only x86/x64 compatible")
+ feature_id = shortuuid.uuid()
+ context.feature_id = feature_id
+ context.directory_bindings = DockerTestDirectoryBindings(feature_id)
+ context.directory_bindings.create_new_data_directories()
+ if "requires.kubernetes.cluster" in feature.tags:
+ context.kubernetes_proxy = KubernetesProxy(
+ context.directory_bindings.get_data_directories(context.feature_id)["kubernetes_temp_dir"],
+ os.path.join(os.environ['TEST_DIRECTORY'], 'resources', 'kubernetes', 'pods-etc'))
+ context.kubernetes_proxy.create_config(context.directory_bindings.get_directory_bindings(context.feature_id))
+ context.kubernetes_proxy.start_cluster()
+
+
+def after_feature(context, feature):
+ if "requires.kubernetes.cluster" in feature.tags and context.kubernetes_proxy:
+ context.kubernetes_proxy.cleanup()
+ context.kubernetes_proxy = None
diff --git a/docker/test/integration/features/file_system_operations.feature b/docker/test/integration/features/file_system_operations.feature
index 5b157284a..5979bca7d 100644
--- a/docker/test/integration/features/file_system_operations.feature
+++ b/docker/test/integration/features/file_system_operations.feature
@@ -1,3 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+@CORE
Feature: File system operations are handled by the GetFile, PutFile, ListFile and FetchFile processors
In order to store and access data on the local file system
As a user of MiNiFi
diff --git a/docker/test/integration/features/google_cloud_storage.feature b/docker/test/integration/features/google_cloud_storage.feature
index e69d5cd5c..de64caff8 100644
--- a/docker/test/integration/features/google_cloud_storage.feature
+++ b/docker/test/integration/features/google_cloud_storage.feature
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+@ENABLE_GCP
Feature: Sending data to Google Cloud Storage using PutGCSObject
Background:
diff --git a/docker/test/integration/features/hashcontent.feature b/docker/test/integration/features/hashcontent.feature
index e25545657..5ca52419a 100644
--- a/docker/test/integration/features/hashcontent.feature
+++ b/docker/test/integration/features/hashcontent.feature
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+@CORE
Feature: Hash value is added to Flowfiles by HashContent processor
In order to avoid duplication of content of Flowfiles
As a user of MiNiFi
diff --git a/docker/test/integration/features/http.feature b/docker/test/integration/features/http.feature
index 1acb19efc..ac20fca22 100644
--- a/docker/test/integration/features/http.feature
+++ b/docker/test/integration/features/http.feature
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+@CORE
Feature: Sending data using InvokeHTTP to a receiver using ListenHTTP
In order to send and receive data via HTTP
As a user of MiNiFi
@@ -25,7 +26,7 @@ Feature: Sending data using InvokeHTTP to a receiver using ListenHTTP
Given a GetFile processor with the "Input Directory" property set to "/tmp/input"
And the "Keep Source File" property of the GetFile processor is set to "true"
And a file with the content "test" is present in "/tmp/input"
- And a InvokeHTTP processor with the "Remote URL" property set to "http://secondary:8080/contentListener"
+ And a InvokeHTTP processor with the "Remote URL" property set to "http://secondary-${feature_id}:8080/contentListener"
And the "HTTP Method" property of the InvokeHTTP processor is set to "POST"
And the "success" relationship of the GetFile processor is connected to the InvokeHTTP
@@ -40,14 +41,14 @@ Feature: Sending data using InvokeHTTP to a receiver using ListenHTTP
Given a GetFile processor with the "Input Directory" property set to "/tmp/input"
And the "Keep Source File" property of the GetFile processor is set to "true"
And a file with the content "test" is present in "/tmp/input"
- And a InvokeHTTP processor with the "Remote URL" property set to "http://minifi-listen:8080/contentListener"
+ And a InvokeHTTP processor with the "Remote URL" property set to "http://minifi-listen-${feature_id}:8080/contentListener"
And these processor properties are set to match the http proxy:
- | processor name | property name | property value |
- | InvokeHTTP | HTTP Method | POST |
- | InvokeHTTP | Proxy Host | http-proxy |
- | InvokeHTTP | Proxy Port | 3128 |
- | InvokeHTTP | invokehttp-proxy-username | admin |
- | InvokeHTTP | invokehttp-proxy-password | test101 |
+ | processor name | property name | property value |
+ | InvokeHTTP | HTTP Method | POST |
+ | InvokeHTTP | Proxy Host | http-proxy-${feature_id} |
+ | InvokeHTTP | Proxy Port | 3128 |
+ | InvokeHTTP | invokehttp-proxy-username | admin |
+ | InvokeHTTP | invokehttp-proxy-password | test101 |
And the "success" relationship of the GetFile processor is connected to the InvokeHTTP
And a http proxy server is set up accordingly
@@ -58,14 +59,14 @@ Feature: Sending data using InvokeHTTP to a receiver using ListenHTTP
When all instances start up
Then at least one flowfile with the content "test" is placed in the monitored directory in less than 120 seconds
- And no errors were generated on the http-proxy regarding "http://minifi-listen:8080/contentListener"
+ And no errors were generated on the http-proxy regarding "http://minifi-listen-${feature_id}:8080/contentListener"
Scenario: A MiNiFi instance and transfers hashed data to another MiNiFi instance
Given a GetFile processor with the "Input Directory" property set to "/tmp/input"
And the "Keep Source File" property of the GetFile processor is set to "true"
And a file with the content "test" is present in "/tmp/input"
And a HashContent processor with the "Hash Attribute" property set to "hash"
- And a InvokeHTTP processor with the "Remote URL" property set to "http://secondary:8080/contentListener"
+ And a InvokeHTTP processor with the "Remote URL" property set to "http://secondary-${feature_id}:8080/contentListener"
And the "HTTP Method" property of the InvokeHTTP processor is set to "POST"
And the "success" relationship of the GetFile processor is connected to the HashContent
And the "success" relationship of the HashContent processor is connected to the InvokeHTTP
@@ -81,7 +82,7 @@ Feature: Sending data using InvokeHTTP to a receiver using ListenHTTP
Given a GetFile processor with the "Input Directory" property set to "/tmp/input"
And the "Keep Source File" property of the GetFile processor is set to "true"
And a file with the content "test" is present in "/tmp/input"
- And a InvokeHTTP processor with the "Remote URL" property set to "http://secondary:8080/contentListener"
+ And a InvokeHTTP processor with the "Remote URL" property set to "http://secondary-${feature_id}:8080/contentListener"
And the "HTTP Method" property of the InvokeHTTP processor is set to "POST"
And the "Send Message Body" property of the InvokeHTTP processor is set to "false"
And the "success" relationship of the GetFile processor is connected to the InvokeHTTP
@@ -97,7 +98,7 @@ Feature: Sending data using InvokeHTTP to a receiver using ListenHTTP
Given a GetFile processor with the "Input Directory" property set to "/tmp/input"
And the "Keep Source File" property of the GetFile processor is set to "true"
And a file with the content "test" is present in "/tmp/input"
- And a InvokeHTTP processor with the "Remote URL" property set to "http://nifi:8081/contentListener"
+ And a InvokeHTTP processor with the "Remote URL" property set to "http://nifi-${feature_id}:8081/contentListener"
And the "HTTP Method" property of the InvokeHTTP processor is set to "POST"
And the "success" relationship of the GetFile processor is connected to the InvokeHTTP
diff --git a/docker/test/integration/features/https.feature b/docker/test/integration/features/https.feature
index 8997ed929..f7f3cae4f 100644
--- a/docker/test/integration/features/https.feature
+++ b/docker/test/integration/features/https.feature
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+@CORE
Feature: Using SSL context service to send data with TLS
In order to send data via HTTPS
As a user of MiNiFi
@@ -24,7 +25,7 @@ Feature: Using SSL context service to send data with TLS
Scenario: A MiNiFi instance sends data using InvokeHTTP to a receiver using ListenHTTP with TLS
Given a GetFile processor with the "Input Directory" property set to "/tmp/input"
And 200 files with the content "test" are present in "/tmp/input"
- And a InvokeHTTP processor with the "Remote URL" property set to "https://secondary:4430/contentListener"
+ And a InvokeHTTP processor with the "Remote URL" property set to "https://secondary-${feature_id}:4430/contentListener"
And the "HTTP Method" property of the InvokeHTTP processor is set to "POST"
And the "success" relationship of the GetFile processor is connected to the InvokeHTTP
diff --git a/docker/test/integration/features/kafka.feature b/docker/test/integration/features/kafka.feature
index 40a83cfe9..0a48b550d 100644
--- a/docker/test/integration/features/kafka.feature
+++ b/docker/test/integration/features/kafka.feature
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+@ENABLE_LIBRDKAFKA
Feature: Sending data to using Kafka streaming platform using PublishKafka
In order to send data to a Kafka stream
As a user of MiNiFi
@@ -31,16 +32,16 @@ Feature: Sending data to using Kafka streaming platform using PublishKafka
| UpdateAttribute | kafka_message_key | unique_message_key_123 |
And a PublishKafka processor set up to communicate with a kafka broker instance
And these processor properties are set:
- | processor name | property name | property value |
- | PublishKafka | Topic Name | test |
- | PublishKafka | Delivery Guarantee | ${kafka_require_num_acks} |
- | PublishKafka | Request Timeout | 12 s |
- | PublishKafka | Message Timeout | 13 s |
- | PublishKafka | Known Brokers | kafka-broker:${literal(9000):plus(92)} |
- | PublishKafka | Client Name | client_no_${literal(6):multiply(7)} |
- | PublishKafka | Kafka Key | ${kafka_message_key} |
- | PublishKafka | retry.backoff.ms | ${literal(2):multiply(25):multiply(3)} |
- | PublishKafka | Message Key Field | kafka.key |
+ | processor name | property name | property value |
+ | PublishKafka | Topic Name | test |
+ | PublishKafka | Delivery Guarantee | ${kafka_require_num_acks} |
+ | PublishKafka | Request Timeout | 12 s |
+ | PublishKafka | Message Timeout | 13 s |
+ | PublishKafka | Known Brokers | kafka-broker-${feature_id}:${literal(9000):plus(92)} |
+ | PublishKafka | Client Name | client_no_${literal(6):multiply(7)} |
+ | PublishKafka | Kafka Key | ${kafka_message_key} |
+ | PublishKafka | retry.backoff.ms | ${literal(2):multiply(25):multiply(3)} |
+ | PublishKafka | Message Key Field | kafka.key |
And a PutFile processor with the "Directory" property set to "/tmp/output"
And the "success" relationship of the GetFile processor is connected to the UpdateAttribute
And the "success" relationship of the UpdateAttribute processor is connected to the PublishKafka
@@ -54,7 +55,7 @@ Feature: Sending data to using Kafka streaming platform using PublishKafka
And the Minifi logs contain the following message: "PublishKafka: request.required.acks [1]" in less than 10 seconds
And the Minifi logs contain the following message: "PublishKafka: request.timeout.ms [12000]" in less than 10 seconds
And the Minifi logs contain the following message: "PublishKafka: message.timeout.ms [13000]" in less than 10 seconds
- And the Minifi logs contain the following message: "PublishKafka: bootstrap.servers [kafka-broker:9092]" in less than 10 seconds
+ And the Minifi logs contain the following message: "PublishKafka: bootstrap.servers [kafka-broker-${feature_id}:9092]" in less than 10 seconds
And the Minifi logs contain the following message: "PublishKafka: client.id [client_no_42]" in less than 10 seconds
And the Minifi logs contain the following message: "PublishKafka: Message Key [unique_message_key_123]" in less than 10 seconds
And the Minifi logs contain the following message: "PublishKafka: DynamicProperty: [retry.backoff.ms] -> [150]" in less than 10 seconds
@@ -76,20 +77,19 @@ Feature: Sending data to using Kafka streaming platform using PublishKafka
And a file with the content "test" is present in "/tmp/input"
And a PublishKafka processor set up to communicate with a kafka broker instance
And these processor properties are set:
- | processor name | property name | property value |
- | PublishKafka | Client Name | LMN |
- | PublishKafka | Known Brokers | kafka-broker:9093 |
- | PublishKafka | Topic Name | test |
- | PublishKafka | Batch Size | 10 |
- | PublishKafka | Compress Codec | none |
- | PublishKafka | Delivery Guarantee | 1 |
- | PublishKafka | Request Timeout | 10 sec |
- | PublishKafka | Message Timeout | 12 sec |
- | PublishKafka | Security CA | /tmp/resources/certs/ca-cert |
- | PublishKafka | Security Cert | /tmp/resources/certs/client_cert.pem |
- | PublishKafka | Security Pass Phrase | abcdefgh |
- | PublishKafka | Security Private Key | /tmp/resources/certs/client_cert.key |
- | PublishKafka | Security Protocol | ssl |
+ | processor name | property name | property value |
+ | PublishKafka | Client Name | LMN |
+ | PublishKafka | Known Brokers | kafka-broker-${feature_id}:9093 |
+ | PublishKafka | Topic Name | test |
+ | PublishKafka | Batch Size | 10 |
+ | PublishKafka | Compress Codec | none |
+ | PublishKafka | Delivery Guarantee | 1 |
+ | PublishKafka | Request Timeout | 10 sec |
+ | PublishKafka | Message Timeout | 12 sec |
+ | PublishKafka | Security CA | /tmp/resources/root_ca.crt |
+ | PublishKafka | Security Cert | /tmp/resources/minifi_client.crt |
+ | PublishKafka | Security Private Key | /tmp/resources/minifi_client.key |
+ | PublishKafka | Security Protocol | ssl |
And a PutFile processor with the "Directory" property set to "/tmp/output"
And the "success" relationship of the GetFile processor is connected to the PublishKafka
And the "success" relationship of the PublishKafka processor is connected to the PutFile
@@ -111,7 +111,7 @@ Feature: Sending data to using Kafka streaming platform using PublishKafka
| PublishKafka | Topic Name | test |
| PublishKafka | Request Timeout | 10 sec |
| PublishKafka | Message Timeout | 12 sec |
- | PublishKafka | Known Brokers | kafka-broker:9094 |
+ | PublishKafka | Known Brokers | kafka-broker-${feature_id}:9094 |
| PublishKafka | Client Name | LMN |
| PublishKafka | Security Protocol | sasl_plaintext |
| PublishKafka | SASL Mechanism | PLAIN |
@@ -131,23 +131,22 @@ Feature: Sending data to using Kafka streaming platform using PublishKafka
And a file with the content "test" is present in "/tmp/input"
And a PublishKafka processor set up to communicate with a kafka broker instance
And these processor properties are set:
- | processor name | property name | property value |
- | PublishKafka | Client Name | LMN |
- | PublishKafka | Known Brokers | kafka-broker:9095 |
- | PublishKafka | Topic Name | test |
- | PublishKafka | Batch Size | 10 |
- | PublishKafka | Compress Codec | none |
- | PublishKafka | Delivery Guarantee | 1 |
- | PublishKafka | Request Timeout | 10 sec |
- | PublishKafka | Message Timeout | 12 sec |
- | PublishKafka | Security CA | /tmp/resources/certs/ca-cert |
- | PublishKafka | Security Cert | /tmp/resources/certs/client_cert.pem |
- | PublishKafka | Security Pass Phrase | abcdefgh |
- | PublishKafka | Security Private Key | /tmp/resources/certs/client_cert.key |
- | PublishKafka | Security Protocol | sasl_ssl |
- | PublishKafka | SASL Mechanism | PLAIN |
- | PublishKafka | Username | alice |
- | PublishKafka | Password | alice-secret |
+ | processor name | property name | property value |
+ | PublishKafka | Client Name | LMN |
+ | PublishKafka | Known Brokers | kafka-broker-${feature_id}:9095 |
+ | PublishKafka | Topic Name | test |
+ | PublishKafka | Batch Size | 10 |
+ | PublishKafka | Compress Codec | none |
+ | PublishKafka | Delivery Guarantee | 1 |
+ | PublishKafka | Request Timeout | 10 sec |
+ | PublishKafka | Message Timeout | 12 sec |
+ | PublishKafka | Security CA | /tmp/resources/root_ca.crt |
+ | PublishKafka | Security Cert | /tmp/resources/minifi_client.crt |
+ | PublishKafka | Security Private Key | /tmp/resources/minifi_client.key |
+ | PublishKafka | Security Protocol | sasl_ssl |
+ | PublishKafka | SASL Mechanism | PLAIN |
+ | PublishKafka | Username | alice |
+ | PublishKafka | Password | alice-secret |
And a PutFile processor with the "Directory" property set to "/tmp/output"
And the "success" relationship of the GetFile processor is connected to the PublishKafka
And the "success" relationship of the PublishKafka processor is connected to the PutFile
@@ -164,7 +163,7 @@ Feature: Sending data to using Kafka streaming platform using PublishKafka
And these processor properties are set:
| processor name | property name | property value |
| PublishKafka | Client Name | LMN |
- | PublishKafka | Known Brokers | kafka-broker:9095 |
+ | PublishKafka | Known Brokers | kafka-broker-${feature_id}:9095 |
| PublishKafka | Topic Name | test |
| PublishKafka | Batch Size | 10 |
| PublishKafka | Compress Codec | none |
@@ -192,7 +191,7 @@ Feature: Sending data to using Kafka streaming platform using PublishKafka
And these processor properties are set:
| processor name | property name | property value |
| PublishKafka | Client Name | LMN |
- | PublishKafka | Known Brokers | kafka-broker:9093 |
+ | PublishKafka | Known Brokers | kafka-broker-${feature_id}:9093 |
| PublishKafka | Topic Name | test |
| PublishKafka | Batch Size | 10 |
| PublishKafka | Compress Codec | none |
@@ -291,7 +290,7 @@ Feature: Sending data to using Kafka streaming platform using PublishKafka
Examples: Key attribute encoding values
| message 1 | message 2 | key attribute encoding |
- | The Odyssey | Ὅμηρος | (not set) |
+ | The Odyssey | Ὅμηρος | (not set) |
| Lolita | Владимир Владимирович Набоков | utf-8 |
| Crime and Punishment | Фёдор Михайлович Достоевский | hex |
| Paradise Lost | John Milton | hEX |
@@ -412,7 +411,7 @@ Feature: Sending data to using Kafka streaming platform using PublishKafka
Given a ConsumeKafka processor set up in a "kafka-consumer-flow" flow
And these processor properties are set:
| processor name | property name | property value |
- | ConsumeKafka | Kafka Brokers | kafka-broker:9093 |
+ | ConsumeKafka | Kafka Brokers | kafka-broker-${feature_id}:9093 |
| ConsumeKafka | Security Protocol | ssl |
And a PutFile processor with the "Directory" property set to "/tmp/output" in the "kafka-consumer-flow" flow
And an ssl context service set up for ConsumeKafka
@@ -430,7 +429,7 @@ Feature: Sending data to using Kafka streaming platform using PublishKafka
Given a ConsumeKafka processor set up in a "kafka-consumer-flow" flow
And these processor properties are set:
| processor name | property name | property value |
- | ConsumeKafka | Kafka Brokers | kafka-broker:9095 |
+ | ConsumeKafka | Kafka Brokers | kafka-broker-${feature_id}:9095 |
| ConsumeKafka | Security Protocol | sasl_ssl |
| ConsumeKafka | SASL Mechanism | PLAIN |
| ConsumeKafka | Username | alice |
@@ -453,7 +452,7 @@ Feature: Sending data to using Kafka streaming platform using PublishKafka
And the "success" relationship of the ConsumeKafka processor is connected to the PutFile
And these processor properties are set:
| processor name | property name | property value |
- | ConsumeKafka | Kafka Brokers | kafka-broker:9094 |
+ | ConsumeKafka | Kafka Brokers | kafka-broker-${feature_id}:9094 |
| ConsumeKafka | Security Protocol | sasl_plaintext |
| ConsumeKafka | SASL Mechanism | PLAIN |
| ConsumeKafka | Username | alice |
diff --git a/docker/test/integration/features/kubernetes_logging.feature b/docker/test/integration/features/kubernetes_logging.feature
index 5a24755e3..5e411c879 100644
--- a/docker/test/integration/features/kubernetes_logging.feature
+++ b/docker/test/integration/features/kubernetes_logging.feature
@@ -14,6 +14,7 @@
# limitations under the License.
@requires.kubernetes.cluster
+@ENABLE_KUBERNETES
Feature: TailFile can collect logs from Kubernetes pods
Background:
diff --git a/docker/test/integration/features/kubernetes_metrics.feature b/docker/test/integration/features/kubernetes_metrics.feature
index e9b849dd1..96d26a4b9 100644
--- a/docker/test/integration/features/kubernetes_metrics.feature
+++ b/docker/test/integration/features/kubernetes_metrics.feature
@@ -14,6 +14,7 @@
# limitations under the License.
@requires.kubernetes.cluster
+@ENABLE_KUBERNETES
Feature: Minifi can collect metrics from Kubernetes pods
Background:
diff --git a/docker/test/integration/features/lua_script.feature b/docker/test/integration/features/lua_script.feature
index 11129d4b1..4160c5087 100644
--- a/docker/test/integration/features/lua_script.feature
+++ b/docker/test/integration/features/lua_script.feature
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+@ENABLE_LUA_SCRIPT
Feature: MiNiFi can execute Lua scripts
Background:
Given the content of "/tmp/output" is monitored
diff --git a/docker/test/integration/features/minifi_c2_server.feature b/docker/test/integration/features/minifi_c2_server.feature
index 24dc491c5..1b847407a 100644
--- a/docker/test/integration/features/minifi_c2_server.feature
+++ b/docker/test/integration/features/minifi_c2_server.feature
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+@CORE
Feature: MiNiFi can communicate with Apache NiFi MiNiFi C2 server
Background:
diff --git a/docker/test/integration/features/minifi_controller.feature b/docker/test/integration/features/minifi_controller.feature
index 700a782bd..486cb1370 100644
--- a/docker/test/integration/features/minifi_controller.feature
+++ b/docker/test/integration/features/minifi_controller.feature
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+@ENABLE_CONTROLLER
Feature: MiNiFi Controller functionalities
Test MiNiFi Controller functionalities
diff --git a/docker/test/integration/features/mqtt.feature b/docker/test/integration/features/mqtt.feature
index 2d64c04af..2268dd49d 100644
--- a/docker/test/integration/features/mqtt.feature
+++ b/docker/test/integration/features/mqtt.feature
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+@ENABLE_MQTT
Feature: Sending data to MQTT streaming platform using PublishMQTT
In order to send and receive data via MQTT
As a user of MiNiFi
diff --git a/docker/test/integration/features/network_listener.feature b/docker/test/integration/features/network_listener.feature
index ffefaa466..fe659a669 100644
--- a/docker/test/integration/features/network_listener.feature
+++ b/docker/test/integration/features/network_listener.feature
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+@CORE
Feature: Minifi C++ can act as a network listener
Background:
diff --git a/docker/test/integration/features/opcua.feature b/docker/test/integration/features/opcua.feature
index 37f65bc21..b75d39b95 100644
--- a/docker/test/integration/features/opcua.feature
+++ b/docker/test/integration/features/opcua.feature
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+@ENABLE_OPCUA
Feature: Putting and fetching data to OPC UA server
In order to send and fetch data from an OPC UA server
As a user of MiNiFi
@@ -28,20 +29,20 @@ Feature: Putting and fetching data to OPC UA server
And a FetchOPCProcessor processor in the "fetch-opc-ua-node" flow
And a PutFile processor with the "Directory" property set to "/tmp/output" in the "fetch-opc-ua-node" flow
And these processor properties are set:
- | processor name | property name | property value |
- | PutOPCProcessor | Parent node ID | 85 |
- | PutOPCProcessor | Parent node ID type | Int |
- | PutOPCProcessor | Target node ID | 9999 |
- | PutOPCProcessor | Target node ID type | Int |
- | PutOPCProcessor | Target node namespace index | 1 |
- | PutOPCProcessor | Value type | <Value Type> |
- | PutOPCProcessor | OPC server endpoint | opc.tcp://opcua-server:4840/ |
- | PutOPCProcessor | Target node browse name | testnodename |
- | FetchOPCProcessor | Node ID | 9999 |
- | FetchOPCProcessor | Node ID type | Int |
- | FetchOPCProcessor | Namespace index | 1 |
- | FetchOPCProcessor | OPC server endpoint | opc.tcp://opcua-server:4840/ |
- | FetchOPCProcessor | Max depth | 1 |
+ | processor name | property name | property value |
+ | PutOPCProcessor | Parent node ID | 85 |
+ | PutOPCProcessor | Parent node ID type | Int |
+ | PutOPCProcessor | Target node ID | 9999 |
+ | PutOPCProcessor | Target node ID type | Int |
+ | PutOPCProcessor | Target node namespace index | 1 |
+ | PutOPCProcessor | Value type | <Value Type> |
+ | PutOPCProcessor | OPC server endpoint | opc.tcp://opcua-server-${feature_id}:4840/ |
+ | PutOPCProcessor | Target node browse name | testnodename |
+ | FetchOPCProcessor | Node ID | 9999 |
+ | FetchOPCProcessor | Node ID type | Int |
+ | FetchOPCProcessor | Namespace index | 1 |
+ | FetchOPCProcessor | OPC server endpoint | opc.tcp://opcua-server-${feature_id}:4840/ |
+ | FetchOPCProcessor | Max depth | 1 |
And the "success" relationship of the GetFile processor is connected to the PutOPCProcessor
And the "success" relationship of the FetchOPCProcessor processor is connected to the PutFile
@@ -65,20 +66,20 @@ Feature: Putting and fetching data to OPC UA server
And a FetchOPCProcessor processor in the "fetch-opc-ua-node" flow
And a PutFile processor with the "Directory" property set to "/tmp/output" in the "fetch-opc-ua-node" flow
And these processor properties are set:
- | processor name | property name | property value |
- | PutOPCProcessor | Parent node ID | 85 |
- | PutOPCProcessor | Parent node ID type | Int |
- | PutOPCProcessor | Target node ID | <Node ID> |
- | PutOPCProcessor | Target node ID type | <Node ID Type> |
- | PutOPCProcessor | Target node namespace index | 1 |
- | PutOPCProcessor | Value type | <Value Type> |
- | PutOPCProcessor | OPC server endpoint | opc.tcp://opcua-server:4840/ |
- | PutOPCProcessor | Target node browse name | testnodename |
- | FetchOPCProcessor | Node ID | <Node ID> |
- | FetchOPCProcessor | Node ID type | <Node ID Type> |
- | FetchOPCProcessor | Namespace index | 1 |
- | FetchOPCProcessor | OPC server endpoint | opc.tcp://opcua-server:4840/ |
- | FetchOPCProcessor | Max depth | 1 |
+ | processor name | property name | property value |
+ | PutOPCProcessor | Parent node ID | 85 |
+ | PutOPCProcessor | Parent node ID type | Int |
+ | PutOPCProcessor | Target node ID | <Node ID> |
+ | PutOPCProcessor | Target node ID type | <Node ID Type> |
+ | PutOPCProcessor | Target node namespace index | 1 |
+ | PutOPCProcessor | Value type | <Value Type> |
+ | PutOPCProcessor | OPC server endpoint | opc.tcp://opcua-server-${feature_id}:4840/ |
+ | PutOPCProcessor | Target node browse name | testnodename |
+ | FetchOPCProcessor | Node ID | <Node ID> |
+ | FetchOPCProcessor | Node ID type | <Node ID Type> |
+ | FetchOPCProcessor | Namespace index | 1 |
+ | FetchOPCProcessor | OPC server endpoint | opc.tcp://opcua-server-${feature_id}:4840/ |
+ | FetchOPCProcessor | Max depth | 1 |
And the "success" relationship of the GetFile processor is connected to the PutOPCProcessor
And the "success" relationship of the FetchOPCProcessor processor is connected to the PutFile
@@ -105,26 +106,26 @@ Feature: Putting and fetching data to OPC UA server
And a FetchOPCProcessor processor in the "fetch-opc-ua-node" flow
And a PutFile processor with the "Directory" property set to "/tmp/output" in the "fetch-opc-ua-node" flow
And these processor properties are set:
- | processor name | property name | property value |
- | PutOPCProcessor | Parent node ID | 85 |
- | PutOPCProcessor | Parent node ID type | Int |
- | PutOPCProcessor | Target node ID | 9999 |
- | PutOPCProcessor | Target node ID type | Int |
- | PutOPCProcessor | Target node namespace index | 1 |
- | PutOPCProcessor | Value type | String |
- | PutOPCProcessor | OPC server endpoint | opc.tcp://opcua-server:4840/ |
- | PutOPCProcessor | Target node browse name | testnodename |
- | PutOPCProcessor | Certificate path | /tmp/resources/opcua/opcua_client_cert.der |
- | PutOPCProcessor | Key path | /tmp/resources/opcua/opcua_client_key.der |
- | PutOPCProcessor | Application URI | urn:open62541.server.application |
- | FetchOPCProcessor | Node ID | 9999 |
- | FetchOPCProcessor | Node ID type | Int |
- | FetchOPCProcessor | Namespace index | 1 |
- | FetchOPCProcessor | OPC server endpoint | opc.tcp://opcua-server:4840/ |
- | FetchOPCProcessor | Max depth | 1 |
- | FetchOPCProcessor | Certificate path | /tmp/resources/opcua/opcua_client_cert.der |
- | FetchOPCProcessor | Key path | /tmp/resources/opcua/opcua_client_key.der |
- | FetchOPCProcessor | Application URI | urn:open62541.server.application |
+ | processor name | property name | property value |
+ | PutOPCProcessor | Parent node ID | 85 |
+ | PutOPCProcessor | Parent node ID type | Int |
+ | PutOPCProcessor | Target node ID | 9999 |
+ | PutOPCProcessor | Target node ID type | Int |
+ | PutOPCProcessor | Target node namespace index | 1 |
+ | PutOPCProcessor | Value type | String |
+ | PutOPCProcessor | OPC server endpoint | opc.tcp://opcua-server-${feature_id}:4840/ |
+ | PutOPCProcessor | Target node browse name | testnodename |
+ | PutOPCProcessor | Certificate path | /tmp/resources/opcua/opcua_client_cert.der |
+ | PutOPCProcessor | Key path | /tmp/resources/opcua/opcua_client_key.der |
+ | PutOPCProcessor | Application URI | urn:open62541.server.application |
+ | FetchOPCProcessor | Node ID | 9999 |
+ | FetchOPCProcessor | Node ID type | Int |
+ | FetchOPCProcessor | Namespace index | 1 |
+ | FetchOPCProcessor | OPC server endpoint | opc.tcp://opcua-server-${feature_id}:4840/ |
+ | FetchOPCProcessor | Max depth | 1 |
+ | FetchOPCProcessor | Certificate path | /tmp/resources/opcua/opcua_client_cert.der |
+ | FetchOPCProcessor | Key path | /tmp/resources/opcua/opcua_client_key.der |
+ | FetchOPCProcessor | Application URI | urn:open62541.server.application |
And the "success" relationship of the GetFile processor is connected to the PutOPCProcessor
And the "success" relationship of the FetchOPCProcessor processor is connected to the PutFile
@@ -142,24 +143,24 @@ Feature: Putting and fetching data to OPC UA server
And a FetchOPCProcessor processor in the "fetch-opc-ua-node" flow
And a PutFile processor with the "Directory" property set to "/tmp/output" in the "fetch-opc-ua-node" flow
And these processor properties are set:
- | processor name | property name | property value |
- | PutOPCProcessor | Parent node ID | 85 |
- | PutOPCProcessor | Parent node ID type | Int |
- | PutOPCProcessor | Target node ID | 9999 |
- | PutOPCProcessor | Target node ID type | Int |
- | PutOPCProcessor | Target node namespace index | 1 |
- | PutOPCProcessor | Value type | String |
- | PutOPCProcessor | OPC server endpoint | opc.tcp://opcua-server:4840/ |
- | PutOPCProcessor | Target node browse name | testnodename |
- | PutOPCProcessor | Username | peter |
- | PutOPCProcessor | Password | peter123 |
- | FetchOPCProcessor | Node ID | 9999 |
- | FetchOPCProcessor | Node ID type | Int |
- | FetchOPCProcessor | Namespace index | 1 |
- | FetchOPCProcessor | OPC server endpoint | opc.tcp://opcua-server:4840/ |
- | FetchOPCProcessor | Max depth | 1 |
- | FetchOPCProcessor | Username | peter |
- | FetchOPCProcessor | Password | peter123 |
+ | processor name | property name | property value |
+ | PutOPCProcessor | Parent node ID | 85 |
+ | PutOPCProcessor | Parent node ID type | Int |
+ | PutOPCProcessor | Target node ID | 9999 |
+ | PutOPCProcessor | Target node ID type | Int |
+ | PutOPCProcessor | Target node namespace index | 1 |
+ | PutOPCProcessor | Value type | String |
+ | PutOPCProcessor | OPC server endpoint | opc.tcp://opcua-server-${feature_id}:4840/ |
+ | PutOPCProcessor | Target node browse name | testnodename |
+ | PutOPCProcessor | Username | peter |
+ | PutOPCProcessor | Password | peter123 |
+ | FetchOPCProcessor | Node ID | 9999 |
+ | FetchOPCProcessor | Node ID type | Int |
+ | FetchOPCProcessor | Namespace index | 1 |
+ | FetchOPCProcessor | OPC server endpoint | opc.tcp://opcua-server-${feature_id}:4840/ |
+ | FetchOPCProcessor | Max depth | 1 |
+ | FetchOPCProcessor | Username | peter |
+ | FetchOPCProcessor | Password | peter123 |
And the "success" relationship of the GetFile processor is connected to the PutOPCProcessor
And the "success" relationship of the FetchOPCProcessor processor is connected to the PutFile
diff --git a/docker/test/integration/features/opensearch.feature b/docker/test/integration/features/opensearch.feature
index 20f0ea9b9..f4e2fabb8 100644
--- a/docker/test/integration/features/opensearch.feature
+++ b/docker/test/integration/features/opensearch.feature
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+@ENABLE_ELASTICSEARCH
Feature: PostElasticsearch works on Opensearch (Opensearch doesnt support API Keys)
Background:
@@ -23,7 +24,7 @@ Feature: PostElasticsearch works on Opensearch (Opensearch doesnt support API Ke
And a GetFile processor with the "Input Directory" property set to "/tmp/input"
And a file with the content "{ "field1" : "value1" }" is present in "/tmp/input"
And a PostElasticsearch processor
- And the "Hosts" property of the PostElasticsearch processor is set to "https://opensearch:9200"
+ And the "Hosts" property of the PostElasticsearch processor is set to "https://opensearch-${feature_id}:9200"
And the "Index" property of the PostElasticsearch processor is set to "my_index"
And the "Identifier" property of the PostElasticsearch processor is set to "my_id"
And the "Action" property of the PostElasticsearch processor is set to <action>
@@ -47,7 +48,7 @@ Feature: PostElasticsearch works on Opensearch (Opensearch doesnt support API Ke
And a GetFile processor with the "Input Directory" property set to "/tmp/input"
And a file with the content "hello world" is present in "/tmp/input"
And a PostElasticsearch processor
- And the "Hosts" property of the PostElasticsearch processor is set to "https://opensearch:9200"
+ And the "Hosts" property of the PostElasticsearch processor is set to "https://opensearch-${feature_id}:9200"
And the "Index" property of the PostElasticsearch processor is set to "my_index"
And the "Identifier" property of the PostElasticsearch processor is set to "preloaded_id"
And the "Action" property of the PostElasticsearch processor is set to "delete"
@@ -66,7 +67,7 @@ Feature: PostElasticsearch works on Opensearch (Opensearch doesnt support API Ke
And a GetFile processor with the "Input Directory" property set to "/tmp/input"
And a file with the content "{ "field2" : "value2" }" is present in "/tmp/input"
And a PostElasticsearch processor
- And the "Hosts" property of the PostElasticsearch processor is set to "https://opensearch:9200"
+ And the "Hosts" property of the PostElasticsearch processor is set to "https://opensearch-${feature_id}:9200"
And the "Index" property of the PostElasticsearch processor is set to "my_index"
And the "Identifier" property of the PostElasticsearch processor is set to "preloaded_id"
And the "Action" property of the PostElasticsearch processor is set to "update"
diff --git a/docker/test/integration/features/prometheus.feature b/docker/test/integration/features/prometheus.feature
index c98946ee4..3140b72da 100644
--- a/docker/test/integration/features/prometheus.feature
+++ b/docker/test/integration/features/prometheus.feature
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+@ENABLE_PROMETHEUS
Feature: MiNiFi can publish metrics to Prometheus server
Background:
diff --git a/docker/test/integration/features/python.feature b/docker/test/integration/features/python.feature
index 51b6f7028..29f054c95 100644
--- a/docker/test/integration/features/python.feature
+++ b/docker/test/integration/features/python.feature
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+@ENABLE_PYTHON_SCRIPTING
Feature: MiNiFi can use python processors in its flows
Background:
Given the content of "/tmp/output" is monitored
diff --git a/docker/test/integration/features/python_script.feature b/docker/test/integration/features/python_script.feature
index d634f279c..dfdf2b310 100644
--- a/docker/test/integration/features/python_script.feature
+++ b/docker/test/integration/features/python_script.feature
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+@ENABLE_PYTHON_SCRIPTING
Feature: MiNiFi can execute Python scripts
Background:
Given the content of "/tmp/output" is monitored
diff --git a/docker/test/integration/features/python_with_modules.feature b/docker/test/integration/features/python_with_modules.feature
index f9c9a191b..0e766df67 100644
--- a/docker/test/integration/features/python_with_modules.feature
+++ b/docker/test/integration/features/python_with_modules.feature
@@ -13,6 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+@ENABLE_PYTHON_SCRIPTING
+@NEEDS_NUMPY
Feature: MiNiFi can use python modules
Background:
Given the content of "/tmp/output" is monitored
diff --git a/docker/test/integration/features/replace_text.feature b/docker/test/integration/features/replace_text.feature
index d5e25e7db..71166e3a4 100644
--- a/docker/test/integration/features/replace_text.feature
+++ b/docker/test/integration/features/replace_text.feature
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+@CORE
Feature: Changing flowfile contents using the ReplaceText processor
Background:
Given the content of "/tmp/output" is monitored
diff --git a/docker/test/integration/features/routetext.feature b/docker/test/integration/features/routetext.feature
index 9543e48bf..f627807b3 100644
--- a/docker/test/integration/features/routetext.feature
+++ b/docker/test/integration/features/routetext.feature
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+@CORE
Feature: Processing log files line-by-line using RouteText
Background:
Given the content of "/tmp/output" is monitored
diff --git a/docker/test/integration/features/s2s.feature b/docker/test/integration/features/s2s.feature
index a51589a02..6b5f2e6f0 100644
--- a/docker/test/integration/features/s2s.feature
+++ b/docker/test/integration/features/s2s.feature
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+@CORE
Feature: Sending data from MiNiFi-C++ to NiFi using S2S protocol
In order to transfer data inbetween NiFi and MiNiFi flows
As a user of MiNiFi
@@ -24,7 +25,7 @@ Feature: Sending data from MiNiFi-C++ to NiFi using S2S protocol
Scenario: A MiNiFi instance produces and transfers data to a NiFi instance via s2s
Given a GetFile processor with the "Input Directory" property set to "/tmp/input"
And a file with the content "test" is present in "/tmp/input"
- And a RemoteProcessGroup node opened on "http://nifi:8080/nifi"
+ And a RemoteProcessGroup node opened on "http://nifi-${feature_id}:8080/nifi"
And the "success" relationship of the GetFile processor is connected to the input port on the RemoteProcessGroup
And a NiFi flow receiving data from a RemoteProcessGroup "from-minifi" on port 8080
@@ -37,7 +38,7 @@ Feature: Sending data from MiNiFi-C++ to NiFi using S2S protocol
Scenario: Zero length files are transfered between via s2s if the "drop empty" connection property is false
Given a MiNiFi CPP server with yaml config
And a GenerateFlowFile processor with the "File Size" property set to "0B"
- And a RemoteProcessGroup node opened on "http://nifi:8080/nifi"
+ And a RemoteProcessGroup node opened on "http://nifi-${feature_id}:8080/nifi"
And the "success" relationship of the GenerateFlowFile processor is connected to the input port on the RemoteProcessGroup
And a NiFi flow receiving data from a RemoteProcessGroup "from-minifi" on port 8080
@@ -51,7 +52,7 @@ Feature: Sending data from MiNiFi-C++ to NiFi using S2S protocol
# "drop empty" is only supported with yaml config
Given a MiNiFi CPP server with yaml config
And a GenerateFlowFile processor with the "File Size" property set to "0B"
- And a RemoteProcessGroup node opened on "http://nifi:8080/nifi"
+ And a RemoteProcessGroup node opened on "http://nifi-${feature_id}:8080/nifi"
And the "success" relationship of the GenerateFlowFile processor is connected to the input port on the RemoteProcessGroup
And the connection going to the RemoteProcessGroup has "drop empty" set
diff --git a/docker/test/integration/features/s3.feature b/docker/test/integration/features/s3.feature
index 7f7ee9ba3..5f2290239 100644
--- a/docker/test/integration/features/s3.feature
+++ b/docker/test/integration/features/s3.feature
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+@ENABLE_AWS
Feature: Sending data from MiNiFi-C++ to an AWS server
In order to transfer data to interact with AWS servers
As a user of MiNiFi
@@ -42,11 +43,11 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
And a file with the content "LH_O#L|FD<FASD{FO#@$#$%^ \"#\"$L%:\"@#$L\":test_data#$#%#$%?{\"F{" is present in "/tmp/input"
And a PutS3Object processor set up to communicate with an s3 server
And these processor properties are set to match the http proxy:
- | processor name | property name | property value |
- | PutS3Object | Proxy Host | http-proxy |
- | PutS3Object | Proxy Port | 3128 |
- | PutS3Object | Proxy Username | admin |
- | PutS3Object | Proxy Password | test101 |
+ | processor name | property name | property value |
+ | PutS3Object | Proxy Host | http-proxy-${feature_id} |
+ | PutS3Object | Proxy Port | 3128 |
+ | PutS3Object | Proxy Username | admin |
+ | PutS3Object | Proxy Password | test101 |
And a PutFile processor with the "Directory" property set to "/tmp/output"
And the "success" relationship of the GetFile processor is connected to the PutS3Object
And the "success" relationship of the PutS3Object processor is connected to the PutFile
@@ -58,7 +59,7 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
Then a flowfile with the content "test" is placed in the monitored directory in less than 150 seconds
And the object on the s3 server is "LH_O#L|FD<FASD{FO#@$#$%^ \"#\"$L%:\"@#$L\":test_data#$#%#$%?{\"F{"
And the object content type on the s3 server is "application/octet-stream" and the object metadata matches use metadata
- And no errors were generated on the http-proxy regarding "http://s3-server:9090/test_bucket/test_object_key"
+ And no errors were generated on the http-proxy regarding "http://s3-server-${feature_id}:9090/test_bucket/test_object_key"
Scenario: A MiNiFi instance can remove s3 bucket objects
Given a GetFile processor with the "Input Directory" property set to "/tmp/input"
@@ -101,11 +102,11 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
And a DeleteS3Object processor set up to communicate with the same s3 server
And a PutFile processor with the "Directory" property set to "/tmp/output"
And these processor properties are set to match the http proxy:
- | processor name | property name | property value |
- | DeleteS3Object | Proxy Host | http-proxy |
- | DeleteS3Object | Proxy Port | 3128 |
- | DeleteS3Object | Proxy Username | admin |
- | DeleteS3Object | Proxy Password | test101 |
+ | processor name | property name | property value |
+ | DeleteS3Object | Proxy Host | http-proxy-${feature_id} |
+ | DeleteS3Object | Proxy Port | 3128 |
+ | DeleteS3Object | Proxy Username | admin |
+ | DeleteS3Object | Proxy Password | test101 |
And the processors are connected up as described here:
| source name | relationship name | destination name |
| GetFile | success | PutS3Object |
@@ -119,7 +120,7 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
Then a flowfile with the content "test" is placed in the monitored directory in less than 150 seconds
And the object bucket on the s3 server is empty
- And no errors were generated on the http-proxy regarding "http://s3-server:9090/test_bucket/test_object_key"
+ And no errors were generated on the http-proxy regarding "http://s3-server-${feature_id}:9090/test_bucket/test_object_key"
Scenario: A MiNiFi instance can download s3 bucket objects directly
Given a GetFile processor with the "Input Directory" property set to "/tmp/input"
@@ -150,11 +151,11 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
Given a GenerateFlowFile processor with the "File Size" property set to "1 kB" in a "secondary" flow
And a FetchS3Object processor set up to communicate with the same s3 server
And these processor properties are set to match the http proxy:
- | processor name | property name | property value |
- | FetchS3Object | Proxy Host | http-proxy |
- | FetchS3Object | Proxy Port | 3128 |
- | FetchS3Object | Proxy Username | admin |
- | FetchS3Object | Proxy Password | test101 |
+ | processor name | property name | property value |
+ | FetchS3Object | Proxy Host | http-proxy-${feature_id} |
+ | FetchS3Object | Proxy Port | 3128 |
+ | FetchS3Object | Proxy Username | admin |
+ | FetchS3Object | Proxy Password | test101 |
And a PutFile processor with the "Directory" property set to "/tmp/output"
And the processors are connected up as described here:
| source name | relationship name | destination name |
@@ -167,7 +168,7 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
When all instances start up
Then a flowfile with the content "test" is placed in the monitored directory in less than 150 seconds
- And no errors were generated on the http-proxy regarding "http://s3-server:9090/test_bucket/test_object_key"
+ And no errors were generated on the http-proxy regarding "http://s3-server-${feature_id}:9090/test_bucket/test_object_key"
Scenario: A MiNiFi instance can list an S3 bucket directly
Given a GetFile processor with the "Input Directory" property set to "/tmp/input"
@@ -197,11 +198,11 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
Given a ListS3 processor in the "secondary" flow
And these processor properties are set to match the http proxy:
- | processor name | property name | property value |
- | ListS3 | Proxy Host | http-proxy |
- | ListS3 | Proxy Port | 3128 |
- | ListS3 | Proxy Username | admin |
- | ListS3 | Proxy Password | test101 |
+ | processor name | property name | property value |
+ | ListS3 | Proxy Host | http-proxy-${feature_id} |
+ | ListS3 | Proxy Port | 3128 |
+ | ListS3 | Proxy Username | admin |
+ | ListS3 | Proxy Password | test101 |
And a PutFile processor with the "Directory" property set to "/tmp/output"
And the "success" relationship of the ListS3 processor is connected to the PutFile
@@ -211,4 +212,4 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
When all instances start up
Then 1 flowfile is placed in the monitored directory in 120 seconds
- And no errors were generated on the http-proxy regarding "http://s3-server:9090/test_bucket"
+ And no errors were generated on the http-proxy regarding "http://s3-server-${feature_id}:9090/test_bucket"
diff --git a/docker/test/integration/features/splunk.feature b/docker/test/integration/features/splunk.feature
index 548fc8cf4..a6f99285e 100644
--- a/docker/test/integration/features/splunk.feature
+++ b/docker/test/integration/features/splunk.feature
@@ -14,6 +14,7 @@
# limitations under the License.
@x86_x64_only
+@ENABLE_SPLUNK
Feature: Sending data to Splunk HEC using PutSplunkHTTP
Background:
@@ -34,8 +35,8 @@ Feature: Sending data to Splunk HEC using PutSplunkHTTP
And the "success" relationship of the PutSplunkHTTP processor is connected to the QuerySplunkIndexingStatus
And the "undetermined" relationship of the QuerySplunkIndexingStatus processor is connected to the QuerySplunkIndexingStatus
And the "acknowledged" relationship of the QuerySplunkIndexingStatus processor is connected to the PutFile
- And the "Hostname" property of the PutSplunkHTTP processor is set to "http://splunk"
- And the "Hostname" property of the QuerySplunkIndexingStatus processor is set to "http://splunk"
+ And the "Hostname" property of the PutSplunkHTTP processor is set to "http://splunk-${feature_id}"
+ And the "Hostname" property of the QuerySplunkIndexingStatus processor is set to "http://splunk-${feature_id}"
When both instances start up
Then a flowfile with the content "foobar" is placed in the monitored directory in less than 20 seconds
@@ -58,8 +59,8 @@ Feature: Sending data to Splunk HEC using PutSplunkHTTP
And the "undetermined" relationship of the QuerySplunkIndexingStatus processor is connected to the QuerySplunkIndexingStatus
And the "acknowledged" relationship of the QuerySplunkIndexingStatus processor is connected to the PutFile
And SSL is enabled for the Splunk HEC and the SSL context service is set up for PutSplunkHTTP and QuerySplunkIndexingStatus
- And the "Hostname" property of the PutSplunkHTTP processor is set to "https://splunk"
- And the "Hostname" property of the QuerySplunkIndexingStatus processor is set to "https://splunk"
+ And the "Hostname" property of the PutSplunkHTTP processor is set to "https://splunk-${feature_id}"
+ And the "Hostname" property of the QuerySplunkIndexingStatus processor is set to "https://splunk-${feature_id}"
When both instances start up
Then a flowfile with the content "foobar" is placed in the monitored directory in less than 20 seconds
diff --git a/docker/test/integration/features/sql.feature b/docker/test/integration/features/sql.feature
index bafab8e95..e3edfdf06 100644
--- a/docker/test/integration/features/sql.feature
+++ b/docker/test/integration/features/sql.feature
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+@ENABLE_SQL
Feature: Executing SQL operations from MiNiFi-C++
As a user of MiNiFi
I need to have ExecuteSQL, QueryDatabaseTable and PutSQL processors
diff --git a/docker/test/integration/steps/steps.py b/docker/test/integration/features/steps/steps.py
similarity index 84%
rename from docker/test/integration/steps/steps.py
rename to docker/test/integration/features/steps/steps.py
index 06978dcdb..00f6093ed 100644
--- a/docker/test/integration/steps/steps.py
+++ b/docker/test/integration/features/steps/steps.py
@@ -12,11 +12,11 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
+import tempfile
from filesystem_validation.FileSystemObserver import FileSystemObserver
from minifi.core.RemoteProcessGroup import RemoteProcessGroup
-from ssl_utils.SSL_cert_utils import make_ca, make_cert, dump_certificate, dump_privatekey
+from ssl_utils.SSL_cert_utils import make_client_cert, make_server_cert
from minifi.core.Funnel import Funnel
from minifi.controllers.SSLContextService import SSLContextService
@@ -34,6 +34,7 @@ import time
import uuid
import binascii
import humanfriendly
+import OpenSSL.crypto
from kafka import KafkaProducer
from confluent_kafka.admin import AdminClient, NewTopic
@@ -45,12 +46,12 @@ import os
# Background
@given("the content of \"{directory}\" is monitored")
def step_impl(context, directory):
- context.test.add_file_system_observer(FileSystemObserver(context.directory_bindings.docker_path_to_local_path(context.test_id, directory)))
+ context.test.add_file_system_observer(FileSystemObserver(context.directory_bindings.docker_path_to_local_path(context.feature_id, directory)))
def __create_processor(context, processor_type, processor_name, property_name, property_value, container_name, engine='minifi-cpp'):
- container = context.test.acquire_container(container_name, engine)
- processor = locate("minifi.processors." + processor_type + "." + processor_type)()
+ container = context.test.acquire_container(context=context, name=container_name, engine=engine)
+ processor = locate("minifi.processors." + processor_type + "." + processor_type)(context=context)
processor.set_name(processor_name)
if property_name is not None:
processor.set_property(property_name, property_value)
@@ -137,10 +138,10 @@ def step_impl(context, processor_type):
@given("a set of processors in the \"{minifi_container_name}\" flow")
def step_impl(context, minifi_container_name):
- container = context.test.acquire_container(minifi_container_name)
+ container = context.test.acquire_container(context=context, name=minifi_container_name)
logging.info(context.table)
for row in context.table:
- processor = locate("minifi.processors." + row["type"] + "." + row["type"])()
+ processor = locate("minifi.processors." + row["type"] + "." + row["type"])(context=context)
processor.set_name(row["name"])
processor.set_uuid(row["uuid"])
context.test.add_node(processor)
@@ -305,7 +306,7 @@ def step_impl(context, source_name, destination_name):
@given("\"{processor_name}\" processor is a start node")
def step_impl(context, processor_name):
- container = context.test.acquire_container("minifi-cpp-flow")
+ container = context.test.acquire_container(context=context, name="minifi-cpp-flow")
processor = context.test.get_or_create_node_by_name(processor_name)
container.add_start_node(processor)
@@ -316,7 +317,7 @@ def step_impl(context, source_name):
remote_process_group = context.test.get_remote_process_group_by_name("RemoteProcessGroup")
source = context.test.generate_input_port_for_remote_process_group(remote_process_group, source_name)
context.test.add_node(source)
- container = context.test.acquire_container('nifi', 'nifi')
+ container = context.test.acquire_container(context=context, name='nifi', engine='nifi')
# Assume that the first node declared is primary unless specified otherwise
if not container.get_start_nodes():
container.add_start_node(source)
@@ -324,12 +325,12 @@ def step_impl(context, source_name):
@given("a NiFi flow with the name \"{flow_name}\" is set up")
def step_impl(context, flow_name):
- context.test.acquire_container(flow_name, 'nifi')
+ context.test.acquire_container(context=context, name=flow_name, engine='nifi')
@given("a transient MiNiFi flow with the name \"{flow_name}\" is set up")
def step_impl(context, flow_name):
- context.test.acquire_container(flow_name, 'minifi-cpp', ["/bin/sh", "-c", "./bin/minifi.sh start && sleep 10 && ./bin/minifi.sh stop && sleep 100"])
+ context.test.acquire_container(context=context, name=flow_name, command=["/bin/sh", "-c", "./bin/minifi.sh start && sleep 10 && ./bin/minifi.sh stop && sleep 100"])
@given("the provenance repository is enabled in MiNiFi")
@@ -351,25 +352,20 @@ def step_impl(context):
@given("the http proxy server is set up")
@given("a http proxy server is set up accordingly")
def step_impl(context):
- context.test.acquire_container("http-proxy", "http-proxy")
+ context.test.acquire_container(context=context, name="http-proxy", engine="http-proxy")
# TLS
@given("an ssl context service set up for {producer_name} and {consumer_name}")
def step_impl(context, producer_name, consumer_name):
- root_ca_cert, root_ca_key = make_ca("root CA")
- minifi_cert, minifi_key = make_cert("minifi-cpp-flow", root_ca_cert, root_ca_key)
- secondary_cert, secondary_key = make_cert("secondary", root_ca_cert, root_ca_key)
- secondary_pem_file = '/tmp/resources/secondary.pem'
- minifi_crt_file = '/tmp/resources/minifi-cpp-flow.crt'
- minifi_key_file = '/tmp/resources/minifi-cpp-flow.key'
- root_ca_crt_file = '/tmp/resources/root_ca.pem'
+ minifi_crt_file = '/tmp/resources/minifi_client.crt'
+ minifi_key_file = '/tmp/resources/minifi_client.key'
+ root_ca_crt_file = '/tmp/resources/root_ca.crt'
ssl_context_service = SSLContextService(cert=minifi_crt_file, ca_cert=root_ca_crt_file, key=minifi_key_file)
- context.test.put_test_resource('secondary.pem', dump_certificate(secondary_cert) + dump_privatekey(secondary_key))
- context.test.put_test_resource('minifi-cpp-flow.crt', dump_certificate(minifi_cert))
- context.test.put_test_resource('minifi-cpp-flow.key', dump_privatekey(minifi_key))
- context.test.put_test_resource('root_ca.pem', dump_certificate(root_ca_cert))
+ secondary_cert, secondary_key = make_server_cert(context.test.get_container_name_with_postfix("secondary"), context.test.root_ca_cert, context.test.root_ca_key)
+ secondary_pem_file = '/tmp/resources/secondary.crt'
+ context.test.put_test_resource('secondary.crt', OpenSSL.crypto.dump_certificate(type=OpenSSL.crypto.FILETYPE_PEM, cert=secondary_cert) + OpenSSL.crypto.dump_privatekey(type=OpenSSL.crypto.FILETYPE_PEM, pkey=secondary_key))
producer = context.test.get_node_by_name(producer_name)
producer.controller_services.append(ssl_context_service)
producer.set_property("SSL Context Service", ssl_context_service.name)
@@ -381,7 +377,10 @@ def step_impl(context, producer_name, consumer_name):
@given("an ssl context service set up for {producer_name}")
def step_impl(context, producer_name):
- ssl_context_service = SSLContextService(cert="/tmp/resources/certs/client_cert.pem", ca_cert="/tmp/resources/certs/ca-cert", key="/tmp/resources/certs/client_cert.key", passphrase="abcdefgh")
+ minifi_crt_file = '/tmp/resources/minifi_client.crt'
+ minifi_key_file = '/tmp/resources/minifi_client.key'
+ root_ca_crt_file = '/tmp/resources/root_ca.crt'
+ ssl_context_service = SSLContextService(cert=minifi_crt_file, ca_cert=root_ca_crt_file, key=minifi_key_file)
producer = context.test.get_node_by_name(producer_name)
producer.controller_services.append(ssl_context_service)
producer.set_property("SSL Context Service", ssl_context_service.name)
@@ -412,7 +411,7 @@ def step_impl(context, processor_name, service_property_name, property_name, pro
@given("a kafka broker is set up in correspondence with the third-party kafka publisher")
@given("a kafka broker is set up in correspondence with the publisher flow")
def step_impl(context):
- context.test.acquire_container("kafka-broker", "kafka-broker")
+ context.test.acquire_container(context=context, name="kafka-broker", engine="kafka-broker")
# MQTT setup
@@ -420,27 +419,27 @@ def step_impl(context):
@given("an MQTT broker is set up in correspondence with the ConsumeMQTT")
@given("an MQTT broker is set up in correspondence with the PublishMQTT and ConsumeMQTT")
def step_impl(context):
- context.test.acquire_container("mqtt-broker", "mqtt-broker")
+ context.test.acquire_container(context=context, name="mqtt-broker", engine="mqtt-broker")
# s3 setup
@given("a s3 server is set up in correspondence with the PutS3Object")
@given("a s3 server is set up in correspondence with the DeleteS3Object")
def step_impl(context):
- context.test.acquire_container("s3-server", "s3-server")
+ context.test.acquire_container(context=context, name="s3-server", engine="s3-server")
# azure storage setup
@given("an Azure storage server is set up")
def step_impl(context):
- context.test.acquire_container("azure-storage-server", "azure-storage-server")
+ context.test.acquire_container(context=context, name="azure-storage-server", engine="azure-storage-server")
# syslog client
@given(u'a Syslog client with {protocol} protocol is setup to send logs to minifi')
def step_impl(context, protocol):
client_name = "syslog-" + protocol.lower() + "-client"
- context.test.acquire_container(client_name, client_name)
+ context.test.acquire_container(context=context, name=client_name, engine=client_name)
# google cloud storage setup
@@ -448,7 +447,7 @@ def step_impl(context, protocol):
@given("a Google Cloud storage server is set up with some test data")
@given('a Google Cloud storage server is set up and a single object with contents "preloaded data" is present')
def step_impl(context):
- context.test.acquire_container("fake-gcs-server", "fake-gcs-server")
+ context.test.acquire_container(context=context, name="fake-gcs-server", engine="fake-gcs-server")
# elasticsearch
@@ -456,8 +455,8 @@ def step_impl(context):
@given('an Elasticsearch server is set up and a single document is present with "preloaded_id" in "my_index"')
@given('an Elasticsearch server is set up and a single document is present with "preloaded_id" in "my_index" with "value1" in "field1"')
def step_impl(context):
- context.test.start_elasticsearch()
- context.test.create_doc_elasticsearch("elasticsearch", "my_index", "preloaded_id")
+ context.test.start_elasticsearch(context)
+ context.test.create_doc_elasticsearch(context.test.get_container_name_with_postfix("elasticsearch"), "my_index", "preloaded_id")
# opensearch
@@ -465,16 +464,16 @@ def step_impl(context):
@given('an Opensearch server is set up and a single document is present with "preloaded_id" in "my_index"')
@given('an Opensearch server is set up and a single document is present with "preloaded_id" in "my_index" with "value1" in "field1"')
def step_impl(context):
- context.test.start_opensearch()
- context.test.add_elastic_user_to_opensearch("opensearch")
- context.test.create_doc_elasticsearch("opensearch", "my_index", "preloaded_id")
+ context.test.start_opensearch(context)
+ context.test.add_elastic_user_to_opensearch(context.test.get_container_name_with_postfix("opensearch"))
+ context.test.create_doc_elasticsearch(context.test.get_container_name_with_postfix("opensearch"), "my_index", "preloaded_id")
@given(u'a SSL context service is set up for PostElasticsearch and Elasticsearch')
def step_impl(context):
- minifi_crt_file = '/tmp/resources/elasticsearch/minifi_client.crt'
- minifi_key_file = '/tmp/resources/elasticsearch/minifi_client.key'
- root_ca_crt_file = '/tmp/resources/elasticsearch/root_ca.crt'
+ minifi_crt_file = '/tmp/resources/minifi_client.crt'
+ minifi_key_file = '/tmp/resources/minifi_client.key'
+ root_ca_crt_file = '/tmp/resources/root_ca.crt'
ssl_context_service = SSLContextService(cert=minifi_crt_file, ca_cert=root_ca_crt_file, key=minifi_key_file)
post_elasticsearch_json = context.test.get_node_by_name("PostElasticsearch")
post_elasticsearch_json.controller_services.append(ssl_context_service)
@@ -483,7 +482,7 @@ def step_impl(context):
@given(u'a SSL context service is set up for PostElasticsearch and Opensearch')
def step_impl(context):
- root_ca_crt_file = '/tmp/resources/opensearch/root-ca.pem'
+ root_ca_crt_file = '/tmp/resources/root_ca.crt'
ssl_context_service = SSLContextService(ca_cert=root_ca_crt_file)
post_elasticsearch_json = context.test.get_node_by_name("PostElasticsearch")
post_elasticsearch_json.controller_services.append(ssl_context_service)
@@ -510,35 +509,30 @@ def step_impl(context):
# splunk hec
@given("a Splunk HEC is set up and running")
def step_impl(context):
- context.test.start_splunk()
+ context.test.start_splunk(context)
# TCP client
@given('a TCP client is set up to send a test TCP message to minifi')
def step_impl(context):
- context.test.acquire_container("tcp-client", "tcp-client")
+ context.test.acquire_container(context=context, name="tcp-client", engine="tcp-client")
@given("SSL is enabled for the Splunk HEC and the SSL context service is set up for PutSplunkHTTP and QuerySplunkIndexingStatus")
def step_impl(context):
- root_ca_cert, root_ca_key = make_ca("root CA")
- minifi_cert, minifi_key = make_cert("minifi-cpp-flow", root_ca_cert, root_ca_key)
- splunk_cert, splunk_key = make_cert("splunk", root_ca_cert, root_ca_key)
- minifi_crt_file = '/tmp/resources/minifi-cpp-flow.pem'
- minifi_key_file = '/tmp/resources/minifi-cpp-flow.key'
- root_ca_crt_file = '/tmp/resources/root_ca.pem'
+ minifi_crt_file = '/tmp/resources/minifi_client.crt'
+ minifi_key_file = '/tmp/resources/minifi_client.key'
+ root_ca_crt_file = '/tmp/resources/root_ca.crt'
ssl_context_service = SSLContextService(cert=minifi_crt_file, ca_cert=root_ca_crt_file, key=minifi_key_file)
- context.test.put_test_resource('minifi-cpp-flow.pem', dump_certificate(minifi_cert))
- context.test.put_test_resource('minifi-cpp-flow.key', dump_privatekey(minifi_key))
- context.test.put_test_resource('root_ca.pem', dump_certificate(root_ca_cert))
+ splunk_cert, splunk_key = make_server_cert(context.test.get_container_name_with_postfix("splunk"), context.test.root_ca_cert, context.test.root_ca_key)
put_splunk_http = context.test.get_node_by_name("PutSplunkHTTP")
put_splunk_http.controller_services.append(ssl_context_service)
put_splunk_http.set_property("SSL Context Service", ssl_context_service.name)
query_splunk_indexing_status = context.test.get_node_by_name("QuerySplunkIndexingStatus")
query_splunk_indexing_status.controller_services.append(ssl_context_service)
query_splunk_indexing_status.set_property("SSL Context Service", ssl_context_service.name)
- context.test.enable_splunk_hec_ssl('splunk', dump_certificate(splunk_cert), dump_privatekey(splunk_key), dump_certificate(root_ca_cert))
+ context.test.enable_splunk_hec_ssl('splunk', OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, splunk_cert), OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, splunk_key), OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, context.test.root_ca_cert))
@given(u'the {processor_one} processor is set up with a GCPCredentialsControllerService to communicate with the Google Cloud storage server')
@@ -547,6 +541,8 @@ def step_impl(context, processor_one):
p1 = context.test.get_node_by_name(processor_one)
p1.controller_services.append(gcp_controller_service)
p1.set_property("GCP Credentials Provider Service", gcp_controller_service.name)
+ processor = context.test.get_node_by_name(processor_one)
+ processor.set_property("Endpoint Override URL", f"fake-gcs-server-{context.feature_id}:4443")
@given(u'the {processor_one} and the {processor_two} processors are set up with a GCPCredentialsControllerService to communicate with the Google Cloud storage server')
@@ -558,11 +554,15 @@ def step_impl(context, processor_one, processor_two):
p1.set_property("GCP Credentials Provider Service", gcp_controller_service.name)
p2.controller_services.append(gcp_controller_service)
p2.set_property("GCP Credentials Provider Service", gcp_controller_service.name)
+ processor_one = context.test.get_node_by_name(processor_one)
+ processor_one.set_property("Endpoint Override URL", f"fake-gcs-server-{context.feature_id}:4443")
+ processor_two = context.test.get_node_by_name(processor_two)
+ processor_two.set_property("Endpoint Override URL", f"fake-gcs-server-{context.feature_id}:4443")
@given("the kafka broker is started")
def step_impl(context):
- context.test.start_kafka_broker()
+ context.test.start_kafka_broker(context)
@given("the topic \"{topic_name}\" is initialized on the kafka broker")
@@ -582,7 +582,8 @@ def step_impl(context, topic_name):
# SQL
@given("an ODBCService is setup up for {processor_name} with the name \"{service_name}\"")
def step_impl(context, processor_name, service_name):
- odbc_service = ODBCService(name=service_name, connection_string="Driver={PostgreSQL ANSI};Server=postgresql-server;Port=5432;Database=postgres;Uid=postgres;Pwd=password;")
+ odbc_service = ODBCService(name=service_name,
+ connection_string="Driver={{PostgreSQL ANSI}};Server={server_hostname};Port=5432;Database=postgres;Uid=postgres;Pwd=password;".format(server_hostname=context.test.get_container_name_with_postfix("postgresql-server")))
processor = context.test.get_node_by_name(processor_name)
processor.controller_services.append(odbc_service)
processor.set_property("DB Controller Service", odbc_service.name)
@@ -591,18 +592,18 @@ def step_impl(context, processor_name, service_name):
@given("a PostgreSQL server is set up")
def step_impl(context):
context.test.enable_sql_in_minifi()
- context.test.acquire_container("postgresql-server", "postgresql-server")
+ context.test.acquire_container(context=context, name="postgresql-server", engine="postgresql-server")
# OPC UA
@given("an OPC UA server is set up")
def step_impl(context):
- context.test.acquire_container("opcua-server", "opcua-server")
+ context.test.acquire_container(context=context, name="opcua-server", engine="opcua-server")
@given("an OPC UA server is set up with access control")
def step_impl(context):
- context.test.acquire_container("opcua-server", "opcua-server", ["/opt/open62541/examples/access_control_server"])
+ context.test.acquire_container(context=context, name="opcua-server", engine="opcua-server", command=["/opt/open62541/examples/access_control_server"])
@when("the MiNiFi instance starts up")
@@ -662,14 +663,29 @@ def step_impl(context, content, topic_name):
@when("a message with content \"{content}\" is published to the \"{topic_name}\" topic using an ssl connection")
def step_impl(context, content, topic_name):
- test_dir = os.environ['TEST_DIRECTORY'] # Based on DockerVerify.sh
+ ca_cert_file = tempfile.NamedTemporaryFile(delete=False)
+ ca_cert_file.write(OpenSSL.crypto.dump_certificate(type=OpenSSL.crypto.FILETYPE_PEM, cert=context.test.root_ca_cert))
+ ca_cert_file.close()
+ os.chmod(ca_cert_file.name, 0o644)
+
+ client_cert, client_key = make_client_cert(socket.gethostname(), context.test.root_ca_cert, context.test.root_ca_key)
+ client_cert_file = tempfile.NamedTemporaryFile(delete=False)
+ client_cert_file.write(OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, client_cert))
+ client_cert_file.close()
+ os.chmod(client_cert_file.name, 0o644)
+
+ client_key_file = tempfile.NamedTemporaryFile(delete=False)
+ client_key_file.write(OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, client_key))
+ client_key_file.close()
+ os.chmod(client_key_file.name, 0o644)
+
producer = Producer({
"bootstrap.servers": "localhost:29093",
"security.protocol": "ssl",
- "ssl.ca.location": os.path.join(test_dir, "resources/kafka_broker/conf/certs/ca-cert"),
- "ssl.certificate.location": os.path.join(test_dir, "resources/kafka_broker/conf/certs/client_cert.pem"),
- "ssl.key.location": os.path.join(test_dir, "resources/kafka_broker/conf/certs/client_cert.key"),
- "ssl.key.password": "abcdefgh",
+ "ssl.ca.location": ca_cert_file.name,
+ "ssl.certificate.location": client_cert_file.name,
+ "ssl.key.location": client_key_file.name,
+ "ssl.key.password": "",
"client.id": socket.gethostname()})
producer.produce(topic_name, content.encode("utf-8"), callback=delivery_report)
producer.flush(10)
@@ -776,7 +792,7 @@ def step_impl(context, content, duration):
@then("{num_flowfiles} flowfiles are placed in the monitored directory in less than {duration}")
def step_impl(context, num_flowfiles, duration):
if num_flowfiles == 0:
- context.execute_steps("""no files are placed in the monitored directory in {duration} of running time""".format(duration=duration))
+ context.execute_steps(f"no files are placed in the monitored directory in {duration} of running time")
return
context.test.check_for_num_files_generated(int(num_flowfiles), humanfriendly.parse_timespan(duration))
@@ -882,7 +898,7 @@ def step_impl(context, blob_and_snapshot_count, timeout_seconds):
# SQL
@then("the query \"{query}\" returns {number_of_rows:d} rows in less than {timeout_seconds:d} seconds on the PostgreSQL server")
def step_impl(context, query, number_of_rows, timeout_seconds):
- context.test.check_query_results("postgresql-server", query, number_of_rows, timeout_seconds)
+ context.test.check_query_results(context.test.get_container_name_with_postfix("postgresql-server"), query, number_of_rows, timeout_seconds)
@then("the Minifi logs contain the following message: \"{log_message}\" in less than {duration}")
@@ -914,12 +930,12 @@ def step_impl(context, log_message, duration):
# MQTT
@then("the MQTT broker has a log line matching \"{log_pattern}\"")
def step_impl(context, log_pattern):
- context.test.check_container_log_matches_regex('mqtt-broker', log_pattern, 10, count=1)
+ context.test.check_container_log_matches_regex('mqtt-broker', log_pattern, 60, count=1)
@then("the MQTT broker has {log_count} log lines matching \"{log_pattern}\"")
def step_impl(context, log_count, log_pattern):
- context.test.check_container_log_matches_regex('mqtt-broker', log_pattern, 10, count=int(log_count))
+ context.test.check_container_log_matches_regex('mqtt-broker', log_pattern, 60, count=int(log_count))
@then("the \"{minifi_container_name}\" flow has a log line matching \"{log_pattern}\" in less than {duration}")
@@ -929,7 +945,7 @@ def step_impl(context, minifi_container_name, log_pattern, duration):
@then("an MQTT broker is deployed in correspondence with the PublishMQTT")
def step_impl(context):
- context.test.acquire_container("mqtt-broker", "mqtt-broker")
+ context.test.acquire_container(context=context, name="mqtt-broker", engine="mqtt-broker")
context.test.start()
@@ -964,7 +980,7 @@ def step_imp(context, content, source, source_type, host):
# Prometheus
@given("a Prometheus server is set up")
def step_impl(context):
- context.test.acquire_container("prometheus", "prometheus")
+ context.test.acquire_container(context=context, name="prometheus", engine="prometheus")
@then("\"{metric_class}\" are published to the Prometheus server in less than {timeout_seconds:d} seconds")
@@ -980,30 +996,33 @@ def step_impl(context, metric_class, timeout_seconds, processor_name):
@then("Elasticsearch is empty")
def step_impl(context):
- context.test.check_empty_elastic("elasticsearch")
+ context.test.check_empty_elastic(context.test.get_container_name_with_postfix("elasticsearch"))
@then(u'Elasticsearch has a document with "{doc_id}" in "{index}" that has "{value}" set in "{field}"')
def step_impl(context, doc_id, index, value, field):
- context.test.check_elastic_field_value("elasticsearch", index_name=index, doc_id=doc_id, field_name=field, field_value=value)
+ context.test.check_elastic_field_value(context.test.get_container_name_with_postfix("elasticsearch"), index_name=index, doc_id=doc_id, field_name=field, field_value=value)
@then("Opensearch is empty")
def step_impl(context):
- context.test.check_empty_elastic("opensearch")
+ context.test.check_empty_elastic(f"opensearch-{context.feature_id}")
@then(u'Opensearch has a document with "{doc_id}" in "{index}" that has "{value}" set in "{field}"')
def step_impl(context, doc_id, index, value, field):
- context.test.check_elastic_field_value("opensearch", index_name=index, doc_id=doc_id, field_name=field, field_value=value)
+ context.test.check_elastic_field_value(f"opensearch-{context.feature_id}", index_name=index, doc_id=doc_id, field_name=field, field_value=value)
# MiNiFi C2 Server
@given("a ssl context service is set up for MiNiFi C2 server")
def step_impl(context):
- ssl_context_service = SSLContextService(cert="/tmp/resources/minifi-c2-server-ssl/minifi-cpp-flow.crt", ca_cert="/tmp/resources/minifi-c2-server-ssl/root-ca.pem", key="/tmp/resources/minifi-c2-server-ssl/minifi-cpp-flow.key", passphrase="abcdefgh")
+ minifi_crt_file = '/tmp/resources/minifi_client.crt'
+ minifi_key_file = '/tmp/resources/minifi_client.key'
+ root_ca_crt_file = '/tmp/resources/root_ca.crt'
+ ssl_context_service = SSLContextService(cert=minifi_crt_file, ca_cert=root_ca_crt_file, key=minifi_key_file)
ssl_context_service.name = "SSLContextService"
- container = context.test.acquire_container("minifi-cpp-flow")
+ container = context.test.acquire_container(context=context, name="minifi-cpp-flow")
container.add_controller(ssl_context_service)
context.test.enable_c2_with_ssl_in_minifi()
@@ -1016,12 +1035,12 @@ def step_impl(context):
@given(u'a MiNiFi C2 server is set up')
def step_impl(context):
- context.test.acquire_container("minifi-c2-server", "minifi-c2-server")
+ context.test.acquire_container(context=context, name="minifi-c2-server", engine="minifi-c2-server")
@given(u'a MiNiFi C2 server is started')
def step_impl(context):
- context.test.start_minifi_c2_server()
+ context.test.start_minifi_c2_server(context)
@then("the MiNiFi C2 server logs contain the following message: \"{log_message}\" in less than {duration}")
@@ -1036,12 +1055,12 @@ def step_impl(context, log_message, duration):
@given(u'a MiNiFi C2 server is set up with SSL')
def step_impl(context):
- context.test.acquire_container("minifi-c2-server", "minifi-c2-server-ssl")
+ context.test.acquire_container(context=context, name="minifi-c2-server", engine="minifi-c2-server-ssl")
@given(u'flow configuration path is set up in flow url property')
def step_impl(context):
- context.test.acquire_container("minifi-cpp-flow", "minifi-cpp")
+ context.test.acquire_container(context=context, name="minifi-cpp-flow", engine="minifi-cpp")
context.test.fetch_flow_config_from_c2_url_in_minifi()
@@ -1079,7 +1098,7 @@ def step_impl(context, minifi_container_name: str):
@when(u'MiNiFi config is updated through MiNiFi controller')
def step_impl(context):
- context.execute_steps("""when MiNiFi config is updated through MiNiFi controller in the \"{minifi_container_name}\" flow""".format(minifi_container_name="minifi-cpp-flow"))
+ context.execute_steps(f"when MiNiFi config is updated through MiNiFi controller in the \"minifi-cpp-flow-{context.feature_id}\" flow")
@then(u'the updated config is persisted in the \"{minifi_container_name}\" flow')
@@ -1089,7 +1108,7 @@ def step_impl(context, minifi_container_name: str):
@then(u'the updated config is persisted')
def step_impl(context):
- context.execute_steps("""then the updated config is persisted in the \"{minifi_container_name}\" flow""".format(minifi_container_name="minifi-cpp-flow"))
+ context.execute_steps(f"then the updated config is persisted in the \"minifi-cpp-flow-{context.feature_id}\" flow")
@when(u'the {component} component is stopped through MiNiFi controller in the \"{minifi_container_name}\" flow')
@@ -1099,7 +1118,7 @@ def step_impl(context, minifi_container_name: str, component: str):
@when(u'the {component} component is stopped through MiNiFi controller')
def step_impl(context, component: str):
- context.execute_steps("""when the {component} component is stopped through MiNiFi controller in the \"{minifi_container_name}\" flow""".format(component=component, minifi_container_name="minifi-cpp-flow"))
+ context.execute_steps(f"when the {component} component is stopped through MiNiFi controller in the \"minifi-cpp-flow-{context.feature_id}\" flow")
@when(u'the {component} component is started through MiNiFi controller in the \"{minifi_container_name}\" flow')
@@ -1109,7 +1128,7 @@ def step_impl(context, minifi_container_name: str, component: str):
@when(u'the {component} component is started through MiNiFi controller')
def step_impl(context, component: str):
- context.execute_steps("""when the {component} component is started through MiNiFi controller in the \"{minifi_container_name}\" flow""".format(component=component, minifi_container_name="minifi-cpp-flow"))
+ context.execute_steps(f"when the {component} component is started through MiNiFi controller in the \"minifi-cpp-flow-{context.feature_id}\" flow")
@then(u'the {component} component is not running in the \"{minifi_container_name}\" flow')
@@ -1119,7 +1138,7 @@ def step_impl(context, component: str, minifi_container_name: str):
@then(u'the {component} component is not running')
def step_impl(context, component: str):
- context.execute_steps("""then the {component} component is not running in the \"{minifi_container_name}\" flow""".format(component=component, minifi_container_name="minifi-cpp-flow"))
+ context.execute_steps(f"then the {component} component is not running in the \"minifi-cpp-flow-{context.feature_id}\" flow")
@then(u'the {component} component is running in the \"{minifi_container_name}\" flow')
@@ -1129,7 +1148,7 @@ def step_impl(context, component: str, minifi_container_name: str):
@then(u'the {component} component is running')
def step_impl(context, component: str):
- context.execute_steps("""then the {component} component is running in the \"{minifi_container_name}\" flow""".format(component=component, minifi_container_name="minifi-cpp-flow"))
+ context.execute_steps(f"then the {component} component is running in the \"minifi-cpp-flow-{context.feature_id}\" flow")
@then(u'connection \"{connection}\" can be seen through MiNiFi controller in the \"{minifi_container_name}\" flow')
@@ -1139,7 +1158,7 @@ def step_impl(context, connection: str, minifi_container_name: str):
@then(u'connection \"{connection}\" can be seen through MiNiFi controller')
def step_impl(context, connection: str):
- context.execute_steps("""then connection \"{connection}\" can be seen through MiNiFi controller in the \"{minifi_container_name}\" flow""".format(connection=connection, minifi_container_name="minifi-cpp-flow"))
+ context.execute_steps(f"then connection \"{connection}\" can be seen through MiNiFi controller in the \"minifi-cpp-flow-{context.feature_id}\" flow")
@then(u'{connection_count:d} connections can be seen full through MiNiFi controller in the \"{minifi_container_name}\" flow')
@@ -1149,7 +1168,7 @@ def step_impl(context, connection_count: int, minifi_container_name: str):
@then(u'{connection_count:d} connections can be seen full through MiNiFi controller')
def step_impl(context, connection_count: int):
- context.execute_steps("""then {connection_count:d} connections can be seen full through MiNiFi controller in the \"{minifi_container_name}\" flow""".format(connection_count=connection_count, minifi_container_name="minifi-cpp-flow"))
+ context.execute_steps(f"then {connection_count:d} connections can be seen full through MiNiFi controller in the \"minifi-cpp-flow-{context.feature_id}\" flow")
@then(u'connection \"{connection}\" has {size:d} size and {max_size:d} max size through MiNiFi controller in the \"{minifi_container_name}\" flow')
@@ -1159,7 +1178,7 @@ def step_impl(context, connection: str, size: int, max_size: int, minifi_contain
@then(u'connection \"{connection}\" has {size:d} size and {max_size:d} max size through MiNiFi controller')
def step_impl(context, connection: str, size: int, max_size: int):
- context.execute_steps("""then connection \"{connection}\" has {size:d} size and {max_size:d} max size through MiNiFi controller in the \"{minifi_container_name}\" flow""".format(connection=connection, size=size, max_size=max_size, minifi_container_name="minifi-cpp-flow"))
+ context.execute_steps(f"then connection \"{connection}\" has {size:d} size and {max_size:d} max size through MiNiFi controller in the \"minifi-cpp-flow-{context.feature_id}\" flow")
@then(u'manifest can be retrieved through MiNiFi controller in the \"{minifi_container_name}\" flow')
@@ -1169,4 +1188,4 @@ def step_impl(context, minifi_container_name: str):
@then(u'manifest can be retrieved through MiNiFi controller')
def step_impl(context):
- context.execute_steps("""then manifest can be retrieved through MiNiFi controller in the \"{minifi_container_name}\" flow""".format(minifi_container_name="minifi-cpp-flow"))
+ context.execute_steps(f"then manifest can be retrieved through MiNiFi controller in the \"minifi-cpp-flow-{context.feature_id}\" flow")
diff --git a/docker/test/integration/features/syslog_listener.feature b/docker/test/integration/features/syslog_listener.feature
index 0e4a4d264..ed575a988 100644
--- a/docker/test/integration/features/syslog_listener.feature
+++ b/docker/test/integration/features/syslog_listener.feature
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+@CORE
Feature: Minifi C++ can act as a syslog listener
Background:
diff --git a/docker/test/integration/utils.py b/docker/test/integration/features/utils.py
similarity index 100%
rename from docker/test/integration/utils.py
rename to docker/test/integration/features/utils.py
diff --git a/docker/test/integration/minifi/core/Processor.py b/docker/test/integration/minifi/core/Processor.py
index 4f004f9d1..8dd0493fd 100644
--- a/docker/test/integration/minifi/core/Processor.py
+++ b/docker/test/integration/minifi/core/Processor.py
@@ -19,6 +19,7 @@ from .Connectable import Connectable
class Processor(Connectable):
def __init__(self,
+ context,
clazz,
properties=None,
schedule=None,
@@ -30,6 +31,8 @@ class Processor(Connectable):
super(Processor, self).__init__(name=name,
auto_terminate=auto_terminate)
+
+ self.context = context
self.class_prefix = class_prefix
if controller_services is None:
@@ -42,6 +45,7 @@ class Processor(Connectable):
properties = {}
self.clazz = clazz
+
self.properties = properties
self.controller_services = controller_services
self.max_concurrent_tasks = max_concurrent_tasks
diff --git a/docker/test/integration/minifi/processors/AddPythonAttribute.py b/docker/test/integration/minifi/processors/AddPythonAttribute.py
index 77f8834bc..1d651c8de 100644
--- a/docker/test/integration/minifi/processors/AddPythonAttribute.py
+++ b/docker/test/integration/minifi/processors/AddPythonAttribute.py
@@ -18,5 +18,5 @@ from ..core.Processor import Processor
class AddPythonAttribute(Processor):
- def __init__(self):
- super(AddPythonAttribute, self).__init__('AddPythonAttribute', class_prefix='org.apache.nifi.minifi.processors.examples.')
+ def __init__(self, context):
+ super(AddPythonAttribute, self).__init__(context=context, clazz='AddPythonAttribute', class_prefix='org.apache.nifi.minifi.processors.examples.')
diff --git a/docker/test/integration/minifi/processors/AttributesToJSON.py b/docker/test/integration/minifi/processors/AttributesToJSON.py
index a98fbf765..c07b154b4 100644
--- a/docker/test/integration/minifi/processors/AttributesToJSON.py
+++ b/docker/test/integration/minifi/processors/AttributesToJSON.py
@@ -18,5 +18,5 @@ from ..core.Processor import Processor
class AttributesToJSON(Processor):
- def __init__(self):
- super(AttributesToJSON, self).__init__('AttributesToJSON')
+ def __init__(self, context):
+ super(AttributesToJSON, self).__init__(context=context, clazz='AttributesToJSON')
diff --git a/docker/test/integration/minifi/processors/AzureBlobStorageProcessorBase.py b/docker/test/integration/minifi/processors/AzureBlobStorageProcessorBase.py
index 0701ac752..811c30685 100644
--- a/docker/test/integration/minifi/processors/AzureBlobStorageProcessorBase.py
+++ b/docker/test/integration/minifi/processors/AzureBlobStorageProcessorBase.py
@@ -18,12 +18,14 @@ from ..core.Processor import Processor
class AzureBlobStorageProcessorBase(Processor):
- def __init__(self, name, additional_properties={}, schedule={"scheduling strategy": "TIMER_DRIVEN"}):
+ def __init__(self, context, clazz, additional_properties={}, schedule={"scheduling strategy": "TIMER_DRIVEN"}):
+ hostname = f"http://azure-storage-server-{context.feature_id}"
super(AzureBlobStorageProcessorBase, self).__init__(
- name,
+ context=context,
+ clazz=clazz,
properties={
'Container Name': 'test-container',
- 'Connection String': 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-storage-server:10000/devstoreaccount1;QueueEndpoint=http://azure-storage-server:10001/devstoreaccount1;',
+ 'Connection String': 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint={hostname}:10000/devstoreaccount1;QueueEndpoint={hostname}:10001/devstoreaccount1;'.format(hostname=hostname),
'Blob': 'test-blob',
**additional_properties
},
diff --git a/docker/test/integration/minifi/processors/CollectKubernetesPodMetrics.py b/docker/test/integration/minifi/processors/CollectKubernetesPodMetrics.py
index 4da4e5066..edfac441e 100644
--- a/docker/test/integration/minifi/processors/CollectKubernetesPodMetrics.py
+++ b/docker/test/integration/minifi/processors/CollectKubernetesPodMetrics.py
@@ -18,7 +18,8 @@ from ..core.Processor import Processor
class CollectKubernetesPodMetrics(Processor):
- def __init__(self, schedule={'scheduling strategy': 'TIMER_DRIVEN', 'scheduling period': '1 second'}):
+ def __init__(self, context, schedule={'scheduling strategy': 'TIMER_DRIVEN', 'scheduling period': '1 second'}):
super(CollectKubernetesPodMetrics, self).__init__(
- 'CollectKubernetesPodMetrics',
+ context=context,
+ clazz='CollectKubernetesPodMetrics',
schedule=schedule)
diff --git a/docker/test/integration/minifi/processors/ConsumeKafka.py b/docker/test/integration/minifi/processors/ConsumeKafka.py
index 239364601..cbc1a482a 100644
--- a/docker/test/integration/minifi/processors/ConsumeKafka.py
+++ b/docker/test/integration/minifi/processors/ConsumeKafka.py
@@ -18,11 +18,12 @@ from ..core.Processor import Processor
class ConsumeKafka(Processor):
- def __init__(self, schedule=None):
+ def __init__(self, context, schedule=None):
super(ConsumeKafka, self).__init__(
- "ConsumeKafka",
+ context=context,
+ clazz="ConsumeKafka",
properties={
- "Kafka Brokers": "kafka-broker:9092",
+ "Kafka Brokers": f"kafka-broker-{context.feature_id}:9092",
"Topic Names": "ConsumeKafkaTest",
"Topic Name Format": "Names",
"Honor Transactions": "true",
diff --git a/docker/test/integration/minifi/processors/ConsumeMQTT.py b/docker/test/integration/minifi/processors/ConsumeMQTT.py
index d32b5bb40..8c64be6a3 100644
--- a/docker/test/integration/minifi/processors/ConsumeMQTT.py
+++ b/docker/test/integration/minifi/processors/ConsumeMQTT.py
@@ -18,11 +18,12 @@ from ..core.Processor import Processor
class ConsumeMQTT(Processor):
- def __init__(self, schedule={'scheduling strategy': 'TIMER_DRIVEN'}):
+ def __init__(self, context, schedule={'scheduling strategy': 'TIMER_DRIVEN'}):
super(ConsumeMQTT, self).__init__(
- 'ConsumeMQTT',
+ context=context,
+ clazz='ConsumeMQTT',
properties={
- 'Broker URI': 'mqtt-broker:1883',
+ 'Broker URI': f'mqtt-broker-{context.feature_id}:1883',
'Topic': 'testtopic',
'Client ID': 'consumer-client'},
auto_terminate=['success'],
diff --git a/docker/test/integration/minifi/processors/CountingProcessor.py b/docker/test/integration/minifi/processors/CountingProcessor.py
index 5431a4ea4..2b45636d6 100644
--- a/docker/test/integration/minifi/processors/CountingProcessor.py
+++ b/docker/test/integration/minifi/processors/CountingProcessor.py
@@ -18,5 +18,7 @@ from ..core.Processor import Processor
class CountingProcessor(Processor):
- def __init__(self):
- super(CountingProcessor, self).__init__('CountingProcessor', class_prefix='org.apache.nifi.minifi.processors.examples.')
+ def __init__(self, context):
+ super(CountingProcessor, self).__init__(context=context,
+ clazz='CountingProcessor',
+ class_prefix='org.apache.nifi.minifi.processors.examples.')
diff --git a/docker/test/integration/minifi/processors/DefragmentText.py b/docker/test/integration/minifi/processors/DefragmentText.py
index d1f272fd4..c00750fe9 100644
--- a/docker/test/integration/minifi/processors/DefragmentText.py
+++ b/docker/test/integration/minifi/processors/DefragmentText.py
@@ -18,8 +18,9 @@ from ..core.Processor import Processor
class DefragmentText(Processor):
- def __init__(self, delimiter="<[0-9]+>", schedule={'scheduling strategy': 'EVENT_DRIVEN'}):
- super(DefragmentText, self).__init__('DefragmentText',
+ def __init__(self, context, delimiter="<[0-9]+>", schedule={'scheduling strategy': 'EVENT_DRIVEN'}):
+ super(DefragmentText, self).__init__(context=context,
+ clazz='DefragmentText',
schedule=schedule,
properties={'Delimiter': delimiter},
auto_terminate=['success', 'failure'])
diff --git a/docker/test/integration/minifi/processors/DeleteAzureBlobStorage.py b/docker/test/integration/minifi/processors/DeleteAzureBlobStorage.py
index ca44ad84f..356057447 100644
--- a/docker/test/integration/minifi/processors/DeleteAzureBlobStorage.py
+++ b/docker/test/integration/minifi/processors/DeleteAzureBlobStorage.py
@@ -18,7 +18,8 @@ from .AzureBlobStorageProcessorBase import AzureBlobStorageProcessorBase
class DeleteAzureBlobStorage(AzureBlobStorageProcessorBase):
- def __init__(self):
+ def __init__(self, context):
super(DeleteAzureBlobStorage, self).__init__(
- 'DeleteAzureBlobStorage',
+ context=context,
+ clazz='DeleteAzureBlobStorage',
schedule={"scheduling strategy": "EVENT_DRIVEN"})
diff --git a/docker/test/integration/minifi/processors/DeleteGCSObject.py b/docker/test/integration/minifi/processors/DeleteGCSObject.py
index 90d304212..b39e77bff 100644
--- a/docker/test/integration/minifi/processors/DeleteGCSObject.py
+++ b/docker/test/integration/minifi/processors/DeleteGCSObject.py
@@ -16,13 +16,13 @@ from ..core.Processor import Processor
class DeleteGCSObject(Processor):
- def __init__(
- self):
+ def __init__(self, context):
super(DeleteGCSObject, self).__init__(
- 'DeleteGCSObject',
+ context=context,
+ clazz='DeleteGCSObject',
properties={
'Bucket': 'test-bucket',
- 'Endpoint Override URL': 'fake-gcs-server:4443',
+ 'Endpoint Override URL': f'fake-gcs-server-{context.feature_id}:4443',
'Number of retries': 2
},
auto_terminate=["success", "failure"])
diff --git a/docker/test/integration/minifi/processors/DeleteS3Object.py b/docker/test/integration/minifi/processors/DeleteS3Object.py
index 501df283f..0f56c02cf 100644
--- a/docker/test/integration/minifi/processors/DeleteS3Object.py
+++ b/docker/test/integration/minifi/processors/DeleteS3Object.py
@@ -20,18 +20,20 @@ from ..core.Processor import Processor
class DeleteS3Object(Processor):
def __init__(
self,
+ context,
proxy_host='',
proxy_port='',
proxy_username='',
proxy_password=''):
super(DeleteS3Object, self).__init__(
- 'DeleteS3Object',
+ context=context,
+ clazz='DeleteS3Object',
properties={
'Object Key': 'test_object_key',
'Bucket': 'test_bucket',
'Access Key': 'test_access_key',
'Secret Key': 'test_secret',
- 'Endpoint Override URL': "http://s3-server:9090",
+ 'Endpoint Override URL': f"http://s3-server-{context.feature_id}:9090",
'Proxy Host': proxy_host,
'Proxy Port': proxy_port,
'Proxy Username': proxy_username,
diff --git a/docker/test/integration/minifi/processors/ExecutePythonProcessor.py b/docker/test/integration/minifi/processors/ExecutePythonProcessor.py
index 559053500..22523ae7b 100644
--- a/docker/test/integration/minifi/processors/ExecutePythonProcessor.py
+++ b/docker/test/integration/minifi/processors/ExecutePythonProcessor.py
@@ -18,7 +18,8 @@ from ..core.Processor import Processor
class ExecutePythonProcessor(Processor):
- def __init__(self):
+ def __init__(self, context):
super(ExecutePythonProcessor, self).__init__(
- 'ExecutePythonProcessor',
+ context=context,
+ clazz='ExecutePythonProcessor',
auto_terminate=['success'])
diff --git a/docker/test/integration/minifi/processors/ExecuteSQL.py b/docker/test/integration/minifi/processors/ExecuteSQL.py
index 95457fd9d..e1570f67f 100644
--- a/docker/test/integration/minifi/processors/ExecuteSQL.py
+++ b/docker/test/integration/minifi/processors/ExecuteSQL.py
@@ -18,8 +18,9 @@ from ..core.Processor import Processor
class ExecuteSQL(Processor):
- def __init__(self, schedule={'scheduling strategy': 'EVENT_DRIVEN'}):
+ def __init__(self, context, schedule={'scheduling strategy': 'EVENT_DRIVEN'}):
super(ExecuteSQL, self).__init__(
- 'ExecuteSQL',
+ context=context,
+ clazz='ExecuteSQL',
auto_terminate=['success'],
schedule=schedule)
diff --git a/docker/test/integration/minifi/processors/ExecuteScript.py b/docker/test/integration/minifi/processors/ExecuteScript.py
index 02c09baef..30eb9b4c1 100644
--- a/docker/test/integration/minifi/processors/ExecuteScript.py
+++ b/docker/test/integration/minifi/processors/ExecuteScript.py
@@ -18,7 +18,8 @@ from ..core.Processor import Processor
class ExecuteScript(Processor):
- def __init__(self):
+ def __init__(self, context):
super(ExecuteScript, self).__init__(
- 'ExecuteScript',
+ context=context,
+ clazz='ExecuteScript',
auto_terminate=['success', 'failure'])
diff --git a/docker/test/integration/minifi/processors/FetchAzureBlobStorage.py b/docker/test/integration/minifi/processors/FetchAzureBlobStorage.py
index c6adc4b16..e1bda7b14 100644
--- a/docker/test/integration/minifi/processors/FetchAzureBlobStorage.py
+++ b/docker/test/integration/minifi/processors/FetchAzureBlobStorage.py
@@ -18,7 +18,8 @@ from .AzureBlobStorageProcessorBase import AzureBlobStorageProcessorBase
class FetchAzureBlobStorage(AzureBlobStorageProcessorBase):
- def __init__(self):
+ def __init__(self, context):
super(FetchAzureBlobStorage, self).__init__(
- 'FetchAzureBlobStorage',
+ context=context,
+ clazz='FetchAzureBlobStorage',
schedule={"scheduling strategy": "EVENT_DRIVEN"})
diff --git a/docker/test/integration/minifi/processors/FetchFile.py b/docker/test/integration/minifi/processors/FetchFile.py
index 24abf6db3..bfed60983 100644
--- a/docker/test/integration/minifi/processors/FetchFile.py
+++ b/docker/test/integration/minifi/processors/FetchFile.py
@@ -18,8 +18,9 @@ from ..core.Processor import Processor
class FetchFile(Processor):
- def __init__(self):
+ def __init__(self, context):
super(FetchFile, self).__init__(
- 'FetchFile',
+ context=context,
+ clazz='FetchFile',
schedule={"scheduling strategy": "EVENT_DRIVEN"},
auto_terminate=['success'])
diff --git a/docker/test/integration/minifi/processors/FetchGCSObject.py b/docker/test/integration/minifi/processors/FetchGCSObject.py
index 176cf687b..31583cf7c 100644
--- a/docker/test/integration/minifi/processors/FetchGCSObject.py
+++ b/docker/test/integration/minifi/processors/FetchGCSObject.py
@@ -16,13 +16,13 @@ from ..core.Processor import Processor
class FetchGCSObject(Processor):
- def __init__(
- self):
+ def __init__(self, context):
super(FetchGCSObject, self).__init__(
- 'FetchGCSObject',
+ context=context,
+ clazz='FetchGCSObject',
properties={
'Bucket': 'test-bucket',
- 'Endpoint Override URL': 'fake-gcs-server:4443',
+ 'Endpoint Override URL': f'fake-gcs-server-{context.feature_id}:4443',
'Number of retries': 2
},
auto_terminate=["success", "failure"])
diff --git a/docker/test/integration/minifi/processors/FetchOPCProcessor.py b/docker/test/integration/minifi/processors/FetchOPCProcessor.py
index c16422a91..32dd5335f 100644
--- a/docker/test/integration/minifi/processors/FetchOPCProcessor.py
+++ b/docker/test/integration/minifi/processors/FetchOPCProcessor.py
@@ -18,7 +18,8 @@ from ..core.Processor import Processor
class FetchOPCProcessor(Processor):
- def __init__(self):
+ def __init__(self, context):
super(FetchOPCProcessor, self).__init__(
- 'FetchOPCProcessor',
+ context=context,
+ clazz='FetchOPCProcessor',
auto_terminate=['success', 'failure'])
diff --git a/docker/test/integration/minifi/processors/FetchS3Object.py b/docker/test/integration/minifi/processors/FetchS3Object.py
index 28509e4cb..0c60096da 100644
--- a/docker/test/integration/minifi/processors/FetchS3Object.py
+++ b/docker/test/integration/minifi/processors/FetchS3Object.py
@@ -19,18 +19,20 @@ from ..core.Processor import Processor
class FetchS3Object(Processor):
def __init__(self,
+ context,
proxy_host='',
proxy_port='',
proxy_username='',
proxy_password=''):
super(FetchS3Object, self).__init__(
- 'FetchS3Object',
+ context=context,
+ clazz='FetchS3Object',
properties={
'Object Key': 'test_object_key',
'Bucket': 'test_bucket',
'Access Key': 'test_access_key',
'Secret Key': 'test_secret',
- 'Endpoint Override URL': "http://s3-server:9090",
+ 'Endpoint Override URL': f"http://s3-server-{context.feature_id}:9090",
'Proxy Host': proxy_host,
'Proxy Port': proxy_port,
'Proxy Username': proxy_username,
diff --git a/docker/test/integration/minifi/processors/GaussianDistributionWithNumpy.py b/docker/test/integration/minifi/processors/GaussianDistributionWithNumpy.py
index 1916704d3..52c225f52 100644
--- a/docker/test/integration/minifi/processors/GaussianDistributionWithNumpy.py
+++ b/docker/test/integration/minifi/processors/GaussianDistributionWithNumpy.py
@@ -18,5 +18,7 @@ from ..core.Processor import Processor
class GaussianDistributionWithNumpy(Processor):
- def __init__(self):
- super(GaussianDistributionWithNumpy, self).__init__('GaussianDistributionWithNumpy', class_prefix='org.apache.nifi.minifi.processors.examples.')
+ def __init__(self, context):
+ super(GaussianDistributionWithNumpy, self).__init__(context=context,
+ clazz='GaussianDistributionWithNumpy',
+ class_prefix='org.apache.nifi.minifi.processors.examples.')
diff --git a/docker/test/integration/minifi/processors/GenerateFlowFile.py b/docker/test/integration/minifi/processors/GenerateFlowFile.py
index 95e9f331d..697b507ba 100644
--- a/docker/test/integration/minifi/processors/GenerateFlowFile.py
+++ b/docker/test/integration/minifi/processors/GenerateFlowFile.py
@@ -18,8 +18,9 @@ from ..core.Processor import Processor
class GenerateFlowFile(Processor):
- def __init__(self, schedule={'scheduling period': '2 sec'}):
+ def __init__(self, context, schedule={'scheduling period': '2 sec'}):
super(GenerateFlowFile, self).__init__(
- 'GenerateFlowFile',
+ context=context,
+ clazz='GenerateFlowFile',
schedule=schedule,
auto_terminate=['success'])
diff --git a/docker/test/integration/minifi/processors/GetFile.py b/docker/test/integration/minifi/processors/GetFile.py
index 083103fe7..9fa08044d 100644
--- a/docker/test/integration/minifi/processors/GetFile.py
+++ b/docker/test/integration/minifi/processors/GetFile.py
@@ -18,9 +18,10 @@ from ..core.Processor import Processor
class GetFile(Processor):
- def __init__(self, input_dir="/tmp/input", schedule={'scheduling period': '2 sec'}):
+ def __init__(self, context, input_dir="/tmp/input", schedule={'scheduling period': '2 sec'}):
super(GetFile, self).__init__(
- 'GetFile',
+ context=context,
+ clazz='GetFile',
properties={
'Input Directory': input_dir,
},
diff --git a/docker/test/integration/minifi/processors/HashContent.py b/docker/test/integration/minifi/processors/HashContent.py
index 512a82599..144650e26 100644
--- a/docker/test/integration/minifi/processors/HashContent.py
+++ b/docker/test/integration/minifi/processors/HashContent.py
@@ -18,9 +18,10 @@ from ..core.Processor import Processor
class HashContent(Processor):
- def __init__(self, schedule={"scheduling strategy": "EVENT_DRIVEN"}):
+ def __init__(self, context, schedule={"scheduling strategy": "EVENT_DRIVEN"}):
super(HashContent, self).__init__(
- "HashContent",
+ context=context,
+ clazz="HashContent",
properties={"Hash Attribute": "hash"},
schedule=schedule,
auto_terminate=["success", "failure"])
diff --git a/docker/test/integration/minifi/processors/InvokeHTTP.py b/docker/test/integration/minifi/processors/InvokeHTTP.py
index 74b06bff2..6ef963d32 100644
--- a/docker/test/integration/minifi/processors/InvokeHTTP.py
+++ b/docker/test/integration/minifi/processors/InvokeHTTP.py
@@ -20,6 +20,7 @@ from ..core.Processor import Processor
class InvokeHTTP(Processor):
def __init__(
self,
+ context,
ssl_context_service=None,
schedule={'scheduling strategy': 'EVENT_DRIVEN'}):
properties = {
@@ -36,7 +37,8 @@ class InvokeHTTP(Processor):
controller_services.append(ssl_context_service)
super(InvokeHTTP, self).__init__(
- 'InvokeHTTP',
+ context=context,
+ clazz='InvokeHTTP',
properties=properties,
controller_services=controller_services,
auto_terminate=['success', 'response', 'retry', 'failure', 'no retry'],
diff --git a/docker/test/integration/minifi/processors/ListAzureBlobStorage.py b/docker/test/integration/minifi/processors/ListAzureBlobStorage.py
index f28801481..ad7186bcd 100644
--- a/docker/test/integration/minifi/processors/ListAzureBlobStorage.py
+++ b/docker/test/integration/minifi/processors/ListAzureBlobStorage.py
@@ -16,11 +16,12 @@ from ..core.Processor import Processor
class ListAzureBlobStorage(Processor):
- def __init__(self):
+ def __init__(self, context):
+ connection_string = 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-storage-server-{feature_id}:10000/devstoreaccount1;QueueEndpoint=http://azure-storage-server-{feature_id}:10001/devstoreaccount1;'.format(feature_id=context.feature_id)
super(ListAzureBlobStorage, self).__init__(
- 'ListAzureBlobStorage',
+ context=context,
+ clazz='ListAzureBlobStorage',
properties={
'Container Name': 'test-container',
- 'Connection String': 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-storage-server:10000/devstoreaccount1;QueueEndpoint=http://azure-storage-server:10001/devstoreaccount1;'
- },
+ 'Connection String': connection_string},
auto_terminate=['success'])
diff --git a/docker/test/integration/minifi/processors/ListFile.py b/docker/test/integration/minifi/processors/ListFile.py
index 5e0d9e6c6..649a8e8ea 100644
--- a/docker/test/integration/minifi/processors/ListFile.py
+++ b/docker/test/integration/minifi/processors/ListFile.py
@@ -18,8 +18,9 @@ from ..core.Processor import Processor
class ListFile(Processor):
- def __init__(self, schedule={'scheduling period': '2 sec'}):
+ def __init__(self, context, schedule={'scheduling period': '2 sec'}):
super(ListFile, self).__init__(
- 'ListFile',
+ context=context,
+ clazz='ListFile',
schedule=schedule,
auto_terminate=['success'])
diff --git a/docker/test/integration/minifi/processors/ListGCSBucket.py b/docker/test/integration/minifi/processors/ListGCSBucket.py
index 5e3ed024e..f6fa67447 100644
--- a/docker/test/integration/minifi/processors/ListGCSBucket.py
+++ b/docker/test/integration/minifi/processors/ListGCSBucket.py
@@ -16,13 +16,13 @@ from ..core.Processor import Processor
class ListGCSBucket(Processor):
- def __init__(
- self):
+ def __init__(self, context):
super(ListGCSBucket, self).__init__(
- 'ListGCSBucket',
+ context=context,
+ clazz='ListGCSBucket',
properties={
'Bucket': 'test-bucket',
- 'Endpoint Override URL': 'fake-gcs-server:4443',
+ 'Endpoint Override URL': f'fake-gcs-server-{context.feature_id}:4443',
'Number of retries': 2
},
auto_terminate=["success"])
diff --git a/docker/test/integration/minifi/processors/ListS3.py b/docker/test/integration/minifi/processors/ListS3.py
index 72b9c340a..9f54064a2 100644
--- a/docker/test/integration/minifi/processors/ListS3.py
+++ b/docker/test/integration/minifi/processors/ListS3.py
@@ -19,20 +19,23 @@ from ..core.Processor import Processor
class ListS3(Processor):
def __init__(self,
+ context,
proxy_host='',
proxy_port='',
proxy_username='',
proxy_password=''):
- super(ListS3, self).__init__('ListS3',
- properties={
- 'Bucket': 'test_bucket',
- 'Access Key': 'test_access_key',
- 'Secret Key': 'test_secret',
- 'Endpoint Override URL': "http://s3-server:9090",
- 'Proxy Host': proxy_host,
- 'Proxy Port': proxy_port,
- 'Proxy Username': proxy_username,
- 'Proxy Password': proxy_password,
- },
- schedule={'scheduling period': '2 sec'},
- auto_terminate=['success'])
+ super(ListS3, self).__init__(
+ context=context,
+ clazz='ListS3',
+ properties={
+ 'Bucket': 'test_bucket',
+ 'Access Key': 'test_access_key',
+ 'Secret Key': 'test_secret',
+ 'Endpoint Override URL': f"http://s3-server-{context.feature_id}:9090",
+ 'Proxy Host': proxy_host,
+ 'Proxy Port': proxy_port,
+ 'Proxy Username': proxy_username,
+ 'Proxy Password': proxy_password,
+ },
+ schedule={'scheduling period': '2 sec'},
+ auto_terminate=['success'])
diff --git a/docker/test/integration/minifi/processors/ListenHTTP.py b/docker/test/integration/minifi/processors/ListenHTTP.py
index d2da48b7f..ef68c9bc1 100644
--- a/docker/test/integration/minifi/processors/ListenHTTP.py
+++ b/docker/test/integration/minifi/processors/ListenHTTP.py
@@ -18,7 +18,7 @@ from ..core.Processor import Processor
class ListenHTTP(Processor):
- def __init__(self, cert=None, schedule=None):
+ def __init__(self, context, cert=None, schedule=None):
properties = {}
if cert is not None:
@@ -26,7 +26,8 @@ class ListenHTTP(Processor):
properties['SSL Verify Peer'] = 'no'
super(ListenHTTP, self).__init__(
- 'ListenHTTP',
+ context=context,
+ clazz='ListenHTTP',
properties=properties,
auto_terminate=['success'],
schedule=schedule)
diff --git a/docker/test/integration/minifi/processors/ListenSyslog.py b/docker/test/integration/minifi/processors/ListenSyslog.py
index 5d805f7c8..585cb69da 100644
--- a/docker/test/integration/minifi/processors/ListenSyslog.py
+++ b/docker/test/integration/minifi/processors/ListenSyslog.py
@@ -16,11 +16,12 @@ from ..core.Processor import Processor
class ListenSyslog(Processor):
- def __init__(self, schedule=None):
+ def __init__(self, context, schedule=None):
properties = {}
super(ListenSyslog, self).__init__(
- 'ListenSyslog',
+ context=context,
+ clazz='ListenSyslog',
properties=properties,
auto_terminate=['success'],
schedule=schedule)
diff --git a/docker/test/integration/minifi/processors/ListenTCP.py b/docker/test/integration/minifi/processors/ListenTCP.py
index 8e737be3e..f2f201f58 100644
--- a/docker/test/integration/minifi/processors/ListenTCP.py
+++ b/docker/test/integration/minifi/processors/ListenTCP.py
@@ -16,11 +16,12 @@ from ..core.Processor import Processor
class ListenTCP(Processor):
- def __init__(self, schedule=None):
+ def __init__(self, context, schedule=None):
properties = {}
super(ListenTCP, self).__init__(
- 'ListenTCP',
+ context=context,
+ clazz='ListenTCP',
properties=properties,
auto_terminate=['success'],
schedule=schedule)
diff --git a/docker/test/integration/minifi/processors/LogAttribute.py b/docker/test/integration/minifi/processors/LogAttribute.py
index eb335974d..f8f43f80d 100644
--- a/docker/test/integration/minifi/processors/LogAttribute.py
+++ b/docker/test/integration/minifi/processors/LogAttribute.py
@@ -18,8 +18,9 @@ from ..core.Processor import Processor
class LogAttribute(Processor):
- def __init__(self, schedule={'scheduling strategy': 'EVENT_DRIVEN'}):
+ def __init__(self, context, schedule={'scheduling strategy': 'EVENT_DRIVEN'}):
super(LogAttribute, self).__init__(
- 'LogAttribute',
+ context=context,
+ clazz='LogAttribute',
auto_terminate=['success'],
schedule=schedule)
diff --git a/docker/test/integration/minifi/processors/LogOnDestructionProcessor.py b/docker/test/integration/minifi/processors/LogOnDestructionProcessor.py
index 52e20bc67..b05225af0 100644
--- a/docker/test/integration/minifi/processors/LogOnDestructionProcessor.py
+++ b/docker/test/integration/minifi/processors/LogOnDestructionProcessor.py
@@ -16,7 +16,8 @@ from ..core.Processor import Processor
class LogOnDestructionProcessor(Processor):
- def __init__(self, schedule={'scheduling period': '2 sec'}):
+ def __init__(self, context, schedule={'scheduling period': '2 sec'}):
super(LogOnDestructionProcessor, self).__init__(
- 'LogOnDestructionProcessor',
+ context=context,
+ clazz='LogOnDestructionProcessor',
schedule=schedule)
diff --git a/docker/test/integration/minifi/processors/MoveContentToJson.py b/docker/test/integration/minifi/processors/MoveContentToJson.py
index 4d4013ce4..bff6bfb61 100644
--- a/docker/test/integration/minifi/processors/MoveContentToJson.py
+++ b/docker/test/integration/minifi/processors/MoveContentToJson.py
@@ -18,5 +18,7 @@ from ..core.Processor import Processor
class MoveContentToJson(Processor):
- def __init__(self):
- super(MoveContentToJson, self).__init__('MoveContentToJson', class_prefix='org.apache.nifi.minifi.processors.examples.')
+ def __init__(self, context):
+ super(MoveContentToJson, self).__init__(context=context,
+ clazz='MoveContentToJson',
+ class_prefix='org.apache.nifi.minifi.processors.examples.')
diff --git a/docker/test/integration/minifi/processors/PostElasticsearch.py b/docker/test/integration/minifi/processors/PostElasticsearch.py
index 4fd959eca..11e506ff1 100644
--- a/docker/test/integration/minifi/processors/PostElasticsearch.py
+++ b/docker/test/integration/minifi/processors/PostElasticsearch.py
@@ -18,11 +18,12 @@ from ..core.Processor import Processor
class PostElasticsearch(Processor):
- def __init__(self, schedule={'scheduling strategy': 'EVENT_DRIVEN'}):
+ def __init__(self, context, schedule={'scheduling strategy': 'EVENT_DRIVEN'}):
super(PostElasticsearch, self).__init__(
- 'PostElasticsearch',
+ context=context,
+ clazz='PostElasticsearch',
properties={
- 'Hosts': 'https://elasticsearch:9200',
+ 'Hosts': f'https://elasticsearch-{context.feature_id}:9200',
'Index': 'test',
'Identifier': '${filename}'
},
diff --git a/docker/test/integration/minifi/processors/PublishKafka.py b/docker/test/integration/minifi/processors/PublishKafka.py
index 08c5dbe58..a79e50728 100644
--- a/docker/test/integration/minifi/processors/PublishKafka.py
+++ b/docker/test/integration/minifi/processors/PublishKafka.py
@@ -18,12 +18,13 @@ from ..core.Processor import Processor
class PublishKafka(Processor):
- def __init__(self, schedule={'scheduling strategy': 'EVENT_DRIVEN'}):
+ def __init__(self, context, schedule={'scheduling strategy': 'EVENT_DRIVEN'}):
super(PublishKafka, self).__init__(
- 'PublishKafka',
+ context=context,
+ clazz='PublishKafka',
properties={
'Client Name': 'nghiaxlee',
- 'Known Brokers': 'kafka-broker:9092',
+ 'Known Brokers': f'kafka-broker-{context.feature_id}:9092',
'Topic Name': 'test',
'Batch Size': '10',
'Compress Codec': 'none',
diff --git a/docker/test/integration/minifi/processors/PublishMQTT.py b/docker/test/integration/minifi/processors/PublishMQTT.py
index 8506d3812..41387ce76 100644
--- a/docker/test/integration/minifi/processors/PublishMQTT.py
+++ b/docker/test/integration/minifi/processors/PublishMQTT.py
@@ -18,11 +18,12 @@ from ..core.Processor import Processor
class PublishMQTT(Processor):
- def __init__(self, schedule={'scheduling strategy': 'EVENT_DRIVEN'}):
+ def __init__(self, context, schedule={'scheduling strategy': 'EVENT_DRIVEN'}):
super(PublishMQTT, self).__init__(
- 'PublishMQTT',
+ context=context,
+ clazz='PublishMQTT',
properties={
- 'Broker URI': 'mqtt-broker:1883',
+ 'Broker URI': f'mqtt-broker-{context.feature_id}:1883',
'Topic': 'testtopic',
'Client ID': 'publisher-client'},
auto_terminate=['success', 'failure'],
diff --git a/docker/test/integration/minifi/processors/PutAzureBlobStorage.py b/docker/test/integration/minifi/processors/PutAzureBlobStorage.py
index b8197742f..f49a942bd 100644
--- a/docker/test/integration/minifi/processors/PutAzureBlobStorage.py
+++ b/docker/test/integration/minifi/processors/PutAzureBlobStorage.py
@@ -18,7 +18,8 @@ from .AzureBlobStorageProcessorBase import AzureBlobStorageProcessorBase
class PutAzureBlobStorage(AzureBlobStorageProcessorBase):
- def __init__(self):
+ def __init__(self, context):
super(PutAzureBlobStorage, self).__init__(
- 'PutAzureBlobStorage',
- {'Create Container': 'true'})
+ context=context,
+ clazz='PutAzureBlobStorage',
+ additional_properties={'Create Container': 'true'})
diff --git a/docker/test/integration/minifi/processors/PutFile.py b/docker/test/integration/minifi/processors/PutFile.py
index a99056207..5b9b665ff 100644
--- a/docker/test/integration/minifi/processors/PutFile.py
+++ b/docker/test/integration/minifi/processors/PutFile.py
@@ -18,9 +18,10 @@ from ..core.Processor import Processor
class PutFile(Processor):
- def __init__(self, output_dir="/tmp/output", schedule={'scheduling strategy': 'EVENT_DRIVEN'}):
+ def __init__(self, context, output_dir="/tmp/output", schedule={'scheduling strategy': 'EVENT_DRIVEN'}):
super(PutFile, self).__init__(
- 'PutFile',
+ context=context,
+ clazz='PutFile',
properties={'Directory': output_dir, 'Directory Permissions': '777', 'Permissions': '777'},
auto_terminate=['success', 'failure'],
schedule=schedule)
diff --git a/docker/test/integration/minifi/processors/PutGCSObject.py b/docker/test/integration/minifi/processors/PutGCSObject.py
index f4ffdddac..05a7fc6df 100644
--- a/docker/test/integration/minifi/processors/PutGCSObject.py
+++ b/docker/test/integration/minifi/processors/PutGCSObject.py
@@ -16,13 +16,13 @@ from ..core.Processor import Processor
class PutGCSObject(Processor):
- def __init__(
- self):
+ def __init__(self, context):
super(PutGCSObject, self).__init__(
- 'PutGCSObject',
+ context=context,
+ clazz='PutGCSObject',
properties={
'Bucket': 'test-bucket',
- 'Endpoint Override URL': 'fake-gcs-server:4443',
+ 'Endpoint Override URL': f'fake-gcs-server-{context.feature_id}:4443',
'Number of retries': 2
},
auto_terminate=["success", "failure"])
diff --git a/docker/test/integration/minifi/processors/PutOPCProcessor.py b/docker/test/integration/minifi/processors/PutOPCProcessor.py
index 24e12bb0d..6da447867 100644
--- a/docker/test/integration/minifi/processors/PutOPCProcessor.py
+++ b/docker/test/integration/minifi/processors/PutOPCProcessor.py
@@ -18,8 +18,9 @@ from ..core.Processor import Processor
class PutOPCProcessor(Processor):
- def __init__(self, schedule={'scheduling strategy': 'EVENT_DRIVEN'}):
+ def __init__(self, context, schedule={'scheduling strategy': 'EVENT_DRIVEN'}):
super(PutOPCProcessor, self).__init__(
- 'PutOPCProcessor',
+ context=context,
+ clazz='PutOPCProcessor',
auto_terminate=['success', 'failure'],
schedule=schedule)
diff --git a/docker/test/integration/minifi/processors/PutS3Object.py b/docker/test/integration/minifi/processors/PutS3Object.py
index 755825fc0..ac33cb43a 100644
--- a/docker/test/integration/minifi/processors/PutS3Object.py
+++ b/docker/test/integration/minifi/processors/PutS3Object.py
@@ -20,19 +20,21 @@ from ..core.Processor import Processor
class PutS3Object(Processor):
def __init__(
self,
+ context,
object_key='test_object_key',
proxy_host='',
proxy_port='',
proxy_username='',
proxy_password=''):
super(PutS3Object, self).__init__(
- 'PutS3Object',
+ context=context,
+ clazz='PutS3Object',
properties={
'Object Key': object_key,
'Bucket': 'test_bucket',
'Access Key': 'test_access_key',
'Secret Key': 'test_secret',
- 'Endpoint Override URL': "http://s3-server:9090",
+ 'Endpoint Override URL': f"http://s3-server-{context.feature_id}:9090",
'Proxy Host': proxy_host,
'Proxy Port': proxy_port,
'Proxy Username': proxy_username,
diff --git a/docker/test/integration/minifi/processors/PutSQL.py b/docker/test/integration/minifi/processors/PutSQL.py
index e5a2ce5cd..64564ad64 100644
--- a/docker/test/integration/minifi/processors/PutSQL.py
+++ b/docker/test/integration/minifi/processors/PutSQL.py
@@ -18,8 +18,9 @@ from ..core.Processor import Processor
class PutSQL(Processor):
- def __init__(self, schedule={'scheduling strategy': 'EVENT_DRIVEN'}):
+ def __init__(self, context, schedule={'scheduling strategy': 'EVENT_DRIVEN'}):
super(PutSQL, self).__init__(
- 'PutSQL',
+ context=context,
+ clazz='PutSQL',
auto_terminate=['success'],
schedule=schedule)
diff --git a/docker/test/integration/minifi/processors/PutSplunkHTTP.py b/docker/test/integration/minifi/processors/PutSplunkHTTP.py
index 51d5662fc..734244d1a 100644
--- a/docker/test/integration/minifi/processors/PutSplunkHTTP.py
+++ b/docker/test/integration/minifi/processors/PutSplunkHTTP.py
@@ -18,9 +18,10 @@ from ..core.Processor import Processor
class PutSplunkHTTP(Processor):
- def __init__(self, schedule={'scheduling strategy': 'EVENT_DRIVEN'}):
+ def __init__(self, context, schedule={'scheduling strategy': 'EVENT_DRIVEN'}):
super(PutSplunkHTTP, self).__init__(
- 'PutSplunkHTTP',
+ context=context,
+ clazz='PutSplunkHTTP',
properties={
'Hostname': 'splunk',
'Port': '8088',
diff --git a/docker/test/integration/minifi/processors/QueryDatabaseTable.py b/docker/test/integration/minifi/processors/QueryDatabaseTable.py
index 10b1d8d38..aa3b4a036 100644
--- a/docker/test/integration/minifi/processors/QueryDatabaseTable.py
+++ b/docker/test/integration/minifi/processors/QueryDatabaseTable.py
@@ -18,7 +18,8 @@ from ..core.Processor import Processor
class QueryDatabaseTable(Processor):
- def __init__(self):
+ def __init__(self, context):
super(QueryDatabaseTable, self).__init__(
- 'QueryDatabaseTable',
+ context=context,
+ clazz='QueryDatabaseTable',
auto_terminate=['success'])
diff --git a/docker/test/integration/minifi/processors/QuerySplunkIndexingStatus.py b/docker/test/integration/minifi/processors/QuerySplunkIndexingStatus.py
index 305a2acac..5dcdb0a67 100644
--- a/docker/test/integration/minifi/processors/QuerySplunkIndexingStatus.py
+++ b/docker/test/integration/minifi/processors/QuerySplunkIndexingStatus.py
@@ -18,10 +18,10 @@ from ..core.Processor import Processor
class QuerySplunkIndexingStatus(Processor):
- def __init__(self, schedule={'scheduling strategy': 'EVENT_DRIVEN',
- 'penalization period': '1 sec'}):
+ def __init__(self, context, schedule={'scheduling strategy': 'EVENT_DRIVEN', 'penalization period': '1 sec'}):
super(QuerySplunkIndexingStatus, self).__init__(
- 'QuerySplunkIndexingStatus',
+ context=context,
+ clazz='QuerySplunkIndexingStatus',
properties={
'Hostname': 'splunk',
'Port': '8088',
diff --git a/docker/test/integration/minifi/processors/RemoveFlowFile.py b/docker/test/integration/minifi/processors/RemoveFlowFile.py
index 3fc9b4da2..7625106ff 100644
--- a/docker/test/integration/minifi/processors/RemoveFlowFile.py
+++ b/docker/test/integration/minifi/processors/RemoveFlowFile.py
@@ -18,5 +18,7 @@ from ..core.Processor import Processor
class RemoveFlowFile(Processor):
- def __init__(self):
- super(RemoveFlowFile, self).__init__('RemoveFlowFile', class_prefix='org.apache.nifi.minifi.processors.examples.')
+ def __init__(self, context):
+ super(RemoveFlowFile, self).__init__(context=context,
+ clazz='RemoveFlowFile',
+ class_prefix='org.apache.nifi.minifi.processors.examples.')
diff --git a/docker/test/integration/minifi/processors/ReplaceText.py b/docker/test/integration/minifi/processors/ReplaceText.py
index d5b85ca17..af5bfcf76 100644
--- a/docker/test/integration/minifi/processors/ReplaceText.py
+++ b/docker/test/integration/minifi/processors/ReplaceText.py
@@ -18,9 +18,10 @@ from ..core.Processor import Processor
class ReplaceText(Processor):
- def __init__(self):
+ def __init__(self, context):
super(ReplaceText, self).__init__(
- 'ReplaceText',
+ context=context,
+ clazz='ReplaceText',
properties={},
schedule={'scheduling strategy': 'EVENT_DRIVEN'},
auto_terminate=['success', 'failure'])
diff --git a/docker/test/integration/minifi/processors/RouteOnAttribute.py b/docker/test/integration/minifi/processors/RouteOnAttribute.py
index 5a1b07605..cf103999e 100644
--- a/docker/test/integration/minifi/processors/RouteOnAttribute.py
+++ b/docker/test/integration/minifi/processors/RouteOnAttribute.py
@@ -18,9 +18,10 @@ from ..core.Processor import Processor
class RouteOnAttribute(Processor):
- def __init__(self, schedule={'scheduling strategy': 'EVENT_DRIVEN'}):
+ def __init__(self, context, schedule={'scheduling strategy': 'EVENT_DRIVEN'}):
super(RouteOnAttribute, self).__init__(
- 'RouteOnAttribute',
+ context=context,
+ clazz='RouteOnAttribute',
properties={},
schedule=schedule,
auto_terminate=['unmatched', "failure"])
diff --git a/docker/test/integration/minifi/processors/RouteText.py b/docker/test/integration/minifi/processors/RouteText.py
index 49090970b..67d61c1c7 100644
--- a/docker/test/integration/minifi/processors/RouteText.py
+++ b/docker/test/integration/minifi/processors/RouteText.py
@@ -18,8 +18,9 @@ from ..core.Processor import Processor
class RouteText(Processor):
- def __init__(self, schedule={'scheduling strategy': 'EVENT_DRIVEN'}):
+ def __init__(self, context, schedule={'scheduling strategy': 'EVENT_DRIVEN'}):
super(RouteText, self).__init__(
- 'RouteText',
+ context=context,
+ clazz='RouteText',
schedule=schedule,
auto_terminate=['unmatched', "matched", "original"])
diff --git a/docker/test/integration/minifi/processors/TailFile.py b/docker/test/integration/minifi/processors/TailFile.py
index 699a2a999..6649dcc51 100644
--- a/docker/test/integration/minifi/processors/TailFile.py
+++ b/docker/test/integration/minifi/processors/TailFile.py
@@ -18,7 +18,8 @@ from ..core.Processor import Processor
class TailFile(Processor):
- def __init__(self, filename="/tmp/input/test_file.log"):
- super(TailFile, self).__init__('TailFile',
+ def __init__(self, context, filename="/tmp/input/test_file.log"):
+ super(TailFile, self).__init__(context=context,
+ clazz='TailFile',
properties={'File to Tail': filename},
auto_terminate=['success'])
diff --git a/docker/test/integration/minifi/processors/UpdateAttribute.py b/docker/test/integration/minifi/processors/UpdateAttribute.py
index 37b64086b..c66b598ac 100644
--- a/docker/test/integration/minifi/processors/UpdateAttribute.py
+++ b/docker/test/integration/minifi/processors/UpdateAttribute.py
@@ -18,8 +18,9 @@ from ..core.Processor import Processor
class UpdateAttribute(Processor):
- def __init__(self, schedule={'scheduling strategy': 'EVENT_DRIVEN'}):
+ def __init__(self, context, schedule={'scheduling strategy': 'EVENT_DRIVEN'}):
super(UpdateAttribute, self).__init__(
- 'UpdateAttribute',
+ context=context,
+ clazz='UpdateAttribute',
auto_terminate=['success'],
schedule=schedule)
diff --git a/docker/test/integration/resources/elasticsearch/Dockerfile b/docker/test/integration/resources/elasticsearch/Dockerfile
deleted file mode 100644
index a7ca77146..000000000
--- a/docker/test/integration/resources/elasticsearch/Dockerfile
+++ /dev/null
@@ -1,7 +0,0 @@
-FROM elasticsearch:8.2.2
-COPY elasticsearch.yml /usr/share/elasticsearch/config/elasticsearch.yml
-COPY certs/elastic_http.key /usr/share/elasticsearch/config/certs/elastic_http.key
-COPY certs/elastic_http.crt /usr/share/elasticsearch/config/certs/elastic_http.crt
-COPY certs/elastic_transport.key /usr/share/elasticsearch/config/certs/elastic_transport.key
-COPY certs/elastic_transport.crt /usr/share/elasticsearch/config/certs/elastic_transport.crt
-COPY certs/root_ca.crt /usr/share/elasticsearch/config/certs/root_ca.crt
diff --git a/docker/test/integration/resources/elasticsearch/certs/elastic_http.crt b/docker/test/integration/resources/elasticsearch/certs/elastic_http.crt
deleted file mode 100644
index 90b292edd..000000000
--- a/docker/test/integration/resources/elasticsearch/certs/elastic_http.crt
+++ /dev/null
@@ -1,19 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDFzCCAf+gAwIBAgIEA0as9DANBgkqhkiG9w0BAQsFADASMRAwDgYDVQQDDAdy
-b290IENBMB4XDTIyMDYwODE1MjAxNVoXDTMyMDYwNTE1MjAxNVowGDEWMBQGA1UE
-AwwNZWxhc3RpY3NlYXJjaDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB
-AKRdem556wIlQGdtEHJM0FsKGcF2paVXl2uhsCLBduknaG+nVBqLQlw1iJh3Q2ru
-6C90hCI7F4Q4xTjeY1+MRzh1YjD/SwbhkUwFU3KB5QOQ+yJ/bM+sAWM+90PRlV8Z
-2WxV8YdGrXlO7PzI4QfWG/9n3krA6KhDofBK5oT0JgYA/QaLFHm6bRFV6E9Octf+
-5fcQMB1xPzC/XT/4aCVMIVsI+NDpn77x00UTLKLfQLzOwEwnyVKjOSghVpKfmF1u
-dt78FumAkFWl9W5MAll4lFTLmD4wgJLh8CkXhRfGRrKzSTTt4Gp8Nu3ZzIKwSqaP
-jwfthx0q+xQx9G1AW6BPvx0CAwEAAaNvMG0wCQYDVR0TBAIwADAdBgNVHQ4EFgQU
-2jmj7l5rSw0yVb/vlWAYkK/YBwkwHwYDVR0jBBgwFoAU2jmj7l5rSw0yVb/vlWAY
-kK/YBwkwEwYDVR0lBAwwCgYIKwYBBQUHAwEwCwYDVR0PBAQDAgeAMA0GCSqGSIb3
-DQEBCwUAA4IBAQBgMbCHkuLGDZKxhO337KvOlaeIBpEoKFpuuSA8GIF4qD4IkFlB
-dqk/2j6iTcRkfzfZSqoMpvJlUppdA3Qg3Xmly5IaqbXD8kj32fOSDsm+CQJb8Yyo
-Fb2YMWgQi/OgB99DOuPn43vlfjW7+dXNJhjg8QSLMxLMPEedEjZPrXe/7BvbJEH3
-IK7jZjuJzTuhzAZ6vAdPpMIKhaXJ9FdEU6zeubyo9GcfAjLiQt+MzuGsraRA5qxO
-SUYmw1pYQz+x9fgvNluaLq8mzaxI6vHVQCckoVFnahL1I2qjktrqr4lWd77fRdoV
-KtVEzApc8sOdWnK+hpgPja8+vqsiCG9PLAU/
------END CERTIFICATE-----
diff --git a/docker/test/integration/resources/elasticsearch/certs/elastic_http.key b/docker/test/integration/resources/elasticsearch/certs/elastic_http.key
deleted file mode 100644
index 847ad81a7..000000000
--- a/docker/test/integration/resources/elasticsearch/certs/elastic_http.key
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCkXXpueesCJUBn
-bRByTNBbChnBdqWlV5drobAiwXbpJ2hvp1Qai0JcNYiYd0Nq7ugvdIQiOxeEOMU4
-3mNfjEc4dWIw/0sG4ZFMBVNygeUDkPsif2zPrAFjPvdD0ZVfGdlsVfGHRq15Tuz8
-yOEH1hv/Z95KwOioQ6HwSuaE9CYGAP0GixR5um0RVehPTnLX/uX3EDAdcT8wv10/
-+GglTCFbCPjQ6Z++8dNFEyyi30C8zsBMJ8lSozkoIVaSn5hdbnbe/BbpgJBVpfVu
-TAJZeJRUy5g+MICS4fApF4UXxkays0k07eBqfDbt2cyCsEqmj48H7YcdKvsUMfRt
-QFugT78dAgMBAAECggEAA8WGBNlKVxBuGRF2clD96kuMihUZZEu4FJxaCNTMpY6l
-xa4exBnUSikLpd1SuuDu4Kvj86R3JoMQ+ibsO9SMYSjHDzMZ0Hdw7erPXo4e0wVK
-0KCrlAZb5SBG7JOiFv5mWqc24Bud6RiHtUiSFVZzP69CpA980eOfaH6nfynaYi0u
-g5Gp7hBFGvvTa5cAZ9msEVXEhVI4FeBAaZXdN/lrsoHfXzMhNvzZeLyDA24lsBQ3
-pYY//V7LT5vRRUmERFZT9FH2eK6YIbTy284B3H1SV50Lc+AP2OehKoTEC8rADJp6
-jB28o0lXTHe0NYFAQpqpwBoDkFwCu8v5kLxuOe74lwKBgQDfl38F4yHEcEBqMBbk
-X9/PNMFO05N8/mc2WpLS5mDaTTuQdn+ybovpx0my4BTZr7xcaH7ntSHkhRRwJDlA
-7r2D7P+icoEcsFDO9HaTCJ3f39qv7qNLOBj5edN5qZuKDXj3cmU7jnzYlfESKYsi
-T1sQV4XvbLu/EbBY8IrMr51pMwKBgQC8MFfd8syqATEAXQ2ClAaz93oO7IKWSgGC
-/aLXzjO11IkxlUHEUzIrKGPrFpo09uav+FUC7os63QVRJg8lNecePnZu2dWihZIc
-Ro9idsGnGstU7qFtl+40Mrl+U6yCQKUAJWPUqX/D8pBnhfrIEikrvcBP0PbkqDD4
-K7VAr2hWbwKBgQCD2RwoZpWk9uSlOZJmPTsHPv628wDH9mT6olJuVzfJwz1apOYM
-XC/ZPZFyD7172KtkeqJLFH7V+LvinRm99op+3ySnsIn+VuJwB6xca/pc5+5XTIuy
-cXQYuQRBfAiiUeuBT03u3sk0Fsc/hIvtQYpbiqEKXL4hn9azu7S9mtliDQKBgE5i
-zJa0Za+CY4jFObVkNw6LTJoexZ4YIV0QjN3bFArvBl5Sz1wZU1JKtqPbfEwHc7tk
-cvSPrArOuUI92h0jwKsGVYqCzcuuW1fPxDW9RZh3Pq6X30GpAR1Y647FB5wRlSjk
-ahrQp1lVFfUpB3aCqgeAODSG0/AWZp9YhVz2Mz2dAoGBANLeln9LyZQXY5NEoF1Z
-12NU8+xzXdepqt6kBdoSIt+2eAucBNZHCfu1DrEMSIQENr/dLFzCiC2GrcczRFsW
-2oQ0t3S5tR5B7zHaDSEPFf+DBIgNbPxIerYnJ9m1TuQPIHyDlG3du3C5dVyVqNed
-fiaGzQ2FmjqPCk39iWEjBL5T
------END PRIVATE KEY-----
diff --git a/docker/test/integration/resources/elasticsearch/certs/elastic_transport.crt b/docker/test/integration/resources/elasticsearch/certs/elastic_transport.crt
deleted file mode 100644
index 4367809d3..000000000
--- a/docker/test/integration/resources/elasticsearch/certs/elastic_transport.crt
+++ /dev/null
@@ -1,19 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIC/jCCAeagAwIBAgIEBHOyVjANBgkqhkiG9w0BAQsFADASMRAwDgYDVQQDDAdy
-b290IENBMB4XDTIyMDYwODE1MjAxNVoXDTMyMDYwNTE1MjAxNVowFDESMBAGA1UE
-AwwJMTI3LjAuMC4xMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtR/I
-wYDdyZy1iW2y9bi+14p7roO6o3EXWNxCPNlEfCbEQlLWaK3Nq45+bTzvMCN9YZvi
-OjjqisIVsiL1oV6MaIBbGaIdtIvDcUF+wGcqTorzVIHOttAL43zRW0RrJLFeMW7E
-7RydntDDeAq5jcjP+DwCRFWBRHYFZhcMbNSyqMHPghRk704MxRp0bFuJCeadIy8/
-oBluiEa0J7ajizAL+IQYOdCdteUI73PCUFF33WvueIC+REz6CFV+Hzgmn6F2e04W
-Y/h0gbXnyjR2tEqCcK68O7eOujQGqz6Ai/6yE+1xRRDAESUIrNAep3qXecnrguWj
-l9LX28BVO/epWaaR8QIDAQABo1owWDAJBgNVHRMEAjAAMB0GA1UdDgQWBBTaOaPu
-XmtLDTJVv++VYBiQr9gHCTAfBgNVHSMEGDAWgBTaOaPuXmtLDTJVv++VYBiQr9gH
-CTALBgNVHQ8EBAMCB4AwDQYJKoZIhvcNAQELBQADggEBABTIcrRsCbtyat2Cdyl2
-ei9hd+Bow6SBR3fJiuwIWQi4UXs9Fg9ts015dRNZf5i4H3POaZYkI69fCIObXzIq
-YwVKS2p5nC5RbY4icgsLkPRXSts2HhuZKbbuuxw4nM7v4iHpj394lujNcX2Ky5zc
-m8jAQSQvh/HnvLWb8DhWCV8yp8lYF8u7PLefm8TixA/fPyfZsT+BYLlIbQ6r9ry/
-kXOYaQ0JGCSMrMGAvL4nxugoNF2LwNTqxxC1/UOCzn52A8BwCb5tvU1B2+k3oYKq
-on8KH4NdGq1kX+gf3/BZ6vOFsf/UmpTrWPdW/3Fow9MzIUkUPf7QgsBlM0m95+Fm
-cFM=
------END CERTIFICATE-----
diff --git a/docker/test/integration/resources/elasticsearch/certs/elastic_transport.key b/docker/test/integration/resources/elasticsearch/certs/elastic_transport.key
deleted file mode 100644
index d7b99d97b..000000000
--- a/docker/test/integration/resources/elasticsearch/certs/elastic_transport.key
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC1H8jBgN3JnLWJ
-bbL1uL7Xinuug7qjcRdY3EI82UR8JsRCUtZorc2rjn5tPO8wI31hm+I6OOqKwhWy
-IvWhXoxogFsZoh20i8NxQX7AZypOivNUgc620AvjfNFbRGsksV4xbsTtHJ2e0MN4
-CrmNyM/4PAJEVYFEdgVmFwxs1LKowc+CFGTvTgzFGnRsW4kJ5p0jLz+gGW6IRrQn
-tqOLMAv4hBg50J215Qjvc8JQUXfda+54gL5ETPoIVX4fOCafoXZ7ThZj+HSBtefK
-NHa0SoJwrrw7t466NAarPoCL/rIT7XFFEMARJQis0B6nepd5yeuC5aOX0tfbwFU7
-96lZppHxAgMBAAECggEABQHQBVZcg7hIlvCQNGSdCcHV3AAqq20FyKXxM6bIcOu4
-I98Aq/oXdRKEdTdG0OQDNv+pHcD8wMUxMmj94j0rzQo8SYwXnZrrKRkYvY2tf2hq
-XolA8lt7ehD5MT/6oXXmnK7Q29lSDKTQYhETexr4RJ165YEfCKC8YgBS0jpTaEv5
-DalYBxgaJOU6pBX/8xRk8rgwuRad0tYcWuHGTWnB3pCzw6+4QlIohhlaXVn1RG2p
-4B5hRf66y7SuAA8S1F+tlmZQogZ5XTPGeEequzqMOy4/uWaALRlZN+tVG8H7Q6jM
-SaSqLgDDMiXzCX3s0jmlGx89sF3+7Qe/YkEXCujZjwKBgQDQZc5Gs+ioeA0Rnr0Z
-MuI4hSxF9LFAxp6qxT5xcVOTUyf/ZeoVgUpMjQWjvQoNyub1KPWSBorumWxKzXWN
-UUNUFn/Al7tpNC2U2YQ+KD+SHjdPC5A8DFzA95qITauXUZumIdpdWyeLpx840gxy
-rSt0Uizb9uN5MFgoydjp5e8TBwKBgQDefyQUUhPKQRBTzTM1l4ZAxvYGd8FJHgoD
-qsn7oc4cLmL8kzDo6GWNx/jqVwGB8o25j4q1y22tDJTV34ynfUhgFNoKuip4a6W4
-KG3MrKCPuAxRMp2JDasoeyeLzY/hsfLpyIIRqM2noghR/tY3sh4NDmt06SWhKRP1
-OMWwwKudRwKBgQC2AuS5XbJrSYboh8LSul1YTf/QDZmuYIr9zY3vGLKM7sj1SHa9
-V5Gj8Fsnmhio2TkEkESCyFIZeEhbwLzVs07hp1Dmzd7yUI8q/RuCCjcTkG5wDEJq
-H8X3B4qhV2zxmWYfIFwOw2lzkIA1aUdT9SkJzCe/6g7crrugzEI2bwtyiwKBgQDH
-t2JgZyhidTK9tkTQRzn4LV+rgJVcF32vuv9hQuVqwGyUAV92wRxfPu/9EapFxb5X
-oTRGaNsC6Xcz3102/qsXq7/ovyDeROpVOt0MSqw5NWDBBekIm99WD+34fMU5Nu0v
-0o6GYXpwjW5gg5owFk8JMzdbkuf3GJNUm+g/90JUgwKBgDKtFlNjhqIxNPcjgGpS
-7ElwDs51VG0gPSaJqaIIy+/Cq8k4pPBwy2Cgyr1kvX8rGuadeC4rN/8UFEw1A79/
-wZj5RD+zTfa6pxtiK2JFN7mubQ+eygkngbvdB8sp8bqcoSt8N3iqnMRGcyaIBwzE
-6GN8QZtQiCJ17Nd9I/akONbS
------END PRIVATE KEY-----
diff --git a/docker/test/integration/resources/elasticsearch/certs/minifi_client.crt b/docker/test/integration/resources/elasticsearch/certs/minifi_client.crt
deleted file mode 100644
index 17e182158..000000000
--- a/docker/test/integration/resources/elasticsearch/certs/minifi_client.crt
+++ /dev/null
@@ -1,19 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDGTCCAgGgAwIBAgIEAzJaSjANBgkqhkiG9w0BAQsFADASMRAwDgYDVQQDDAdy
-b290IENBMB4XDTIyMDYwODE1MjAxNVoXDTMyMDYwNTE1MjAxNVowGjEYMBYGA1UE
-AwwPbWluaWZpLWNwcC1mbG93MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC
-AQEAioSZCHxWY9RWIwdLG+OLWAOdZF5HRKjD+41oA11zT6O/702AvYI7e9oGAb0S
-8z80C92UMWEtJapPfG0eDXPH9F6KPxgT2sMhbY46Lwd8Sdv1HowcR95ULIwNsn06
-boIIFF7/0/8paE9XWOO9ZlhATO/cgsG6EHeMfZr6J3KwC2kfJQSRW9wnBS67StCi
-9vJf0bFKjuhHDTOWtqA8P3Mv6ey/44tOpMAO2hW4r/wif/n+qljEGjllDWFYoBMQ
-pZeGUUb+N8HfgQggWg017yMVQXRSadH4bF1HDaDYOL+E/0spg3bSN0gYJGWnAtqt
-mIx4A7fHzqJNJxwQF+RKL+lo9QIDAQABo28wbTAJBgNVHRMEAjAAMB0GA1UdDgQW
-BBTaOaPuXmtLDTJVv++VYBiQr9gHCTAfBgNVHSMEGDAWgBTaOaPuXmtLDTJVv++V
-YBiQr9gHCTATBgNVHSUEDDAKBggrBgEFBQcDAjALBgNVHQ8EBAMCB4AwDQYJKoZI
-hvcNAQELBQADggEBALYUrK+h1yJNSDLvlV1lPENelLGFj7XyQqM3IPK1CMpYqVlR
-NfeYt5/gHv96b4XijOMNr1vgJz1T10x+LbqQaDMzOWGFg8uyCe64xOH39UUWkF72
-mTftCOaszCO/stkT6jffa492lV0bY6+q3OpJKCbxJbHQQQkftpSyivXvblnXRO1B
-WXAg7YvhMsca+xyImcJQ5nmqEilpbB4gH2Kena2NMdFy/FTGqwRSJFQNevPYxvc8
-Ok9VcphPmn6u8vZG5zb0KeaknitUDg+PlJKv7OPR7QC99KfgVKAzlyZ4GFcHkMVN
-ahH8dxTCk7sOcF5MBecLtUBhRm0u4m66cQH/gXo=
------END CERTIFICATE-----
diff --git a/docker/test/integration/resources/elasticsearch/certs/minifi_client.key b/docker/test/integration/resources/elasticsearch/certs/minifi_client.key
deleted file mode 100644
index 52e32e335..000000000
--- a/docker/test/integration/resources/elasticsearch/certs/minifi_client.key
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCKhJkIfFZj1FYj
-B0sb44tYA51kXkdEqMP7jWgDXXNPo7/vTYC9gjt72gYBvRLzPzQL3ZQxYS0lqk98
-bR4Nc8f0Xoo/GBPawyFtjjovB3xJ2/UejBxH3lQsjA2yfTpugggUXv/T/yloT1dY
-471mWEBM79yCwboQd4x9mvoncrALaR8lBJFb3CcFLrtK0KL28l/RsUqO6EcNM5a2
-oDw/cy/p7L/ji06kwA7aFbiv/CJ/+f6qWMQaOWUNYVigExCll4ZRRv43wd+BCCBa
-DTXvIxVBdFJp0fhsXUcNoNg4v4T/SymDdtI3SBgkZacC2q2YjHgDt8fOok0nHBAX
-5Eov6Wj1AgMBAAECggEAFKH/sZXjwGcGlZ0mgPb58JhJP4QTlZSV7Mh5b9bhsEEx
-+9FNlezqM16cenHRes41UHcm9dwFv5CI2n3edh6C8FGMwi0x9+oM9qay4AJ7bv0G
-nBA+vi3N1/zcpXOl2oQf8/janpdpcZIv7putwKrwmQU5kXiN4JoOIT+Od4XL10cP
-KxLMJSqBae5U4nHmYc09rLm059uLuClnb6nb493vk0WrQ8JYx0PhFWo/qsUbidKz
-ZJiMHMLD1ygCK6XDZ9dXjm3s4J6L8w24pc4AnCDpEHedb6COWJCCGxTVRXH5w2TW
-gPF8dXZKOWMhlYrrms6kFFCEbwyN1MadXixmDLf7mQKBgQC1b59GlMiVB7HZ64gU
-rVwpcja2kVCITyGe3T9nsfCiHK+eR3OH8EzGY3jHPoAsZVFQXFuxgnoIQ9URsIYO
-hBX6+6KKvYzQqW16bZSYrH5Xvkrka2Sgew2sexoolauHbqws2nD8qqhL5Iafcywt
-YAJN9j86ENddxb3e5HbXRxwc1wKBgQDDca+UT6J6AoeO9KKxVKnKEXO3j9vwXb7m
-kM7uT+NIfGIlS84FE3gmz19/ck2Wx/6BvfeZdJWLoCuUhAiZ7TIMamztMg209NOP
-9dejwq6sln9Xq7dmtCJFWs0q1KlU2voXn9g4SNdmt9ZZoAhDIDJ//YLQXK9gbIcZ
-G0FIFRFDEwKBgFG2r/lN0Pitun/3ABav7S4Nbm7TC34YhX/TDVdaYJHicYXLkFDM
-/QbsjdzOXVOhXfnfXryXkvZd9nw9EGNBQdUDt7TcoaroY5IcACyyo0RJQ9mUNIHC
-aeoglytoHRSl141r5C8nfRyRE8CEUoUtFFygZuFz7EsjX9we8RcLw7GJAoGAMpM1
-qzxcQBRtfcnCW/8bnMWdWy/n6zBnGQT592NG3lgJkbsnCds66VgaT0GF4HHKY0qE
-SvqHREVjYaA1FZfyrpHPtCWStHPyVzt33NGhH9omA8rUv8f3YlkX5HB5jRLJStm7
-Ov1meBc0aabm3G/1gXdZpY+Gdn4/C3kpEH0bJW8CgYBS0ZOmLm70axiQFsxj6zgr
-zYzI637Q0AehUif+7PUr5fc4jIAsM2mAGg1lMICvthf69GkIRs4JUywmRK+VmiOc
-JuVHFTH3XmyklNuhT7OJ8srV+K81V64gBo0I+Mp2S5dvQtePBKManUN2r46CUFM+
-ScBnD0bENAWY9GX7TTmCqQ==
------END PRIVATE KEY-----
diff --git a/docker/test/integration/resources/elasticsearch/certs/root_ca.crt b/docker/test/integration/resources/elasticsearch/certs/root_ca.crt
deleted file mode 100644
index a2cfdb6e3..000000000
--- a/docker/test/integration/resources/elasticsearch/certs/root_ca.crt
+++ /dev/null
@@ -1,19 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIC/zCCAeegAwIBAgIEA9yGAzANBgkqhkiG9w0BAQsFADASMRAwDgYDVQQDDAdy
-b290IENBMB4XDTIyMDYwODE1MjAxNVoXDTMyMDYwNTE1MjAxNVowEjEQMA4GA1UE
-AwwHcm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANncdhc0
-ZKY8PCUo3KZDHS0N+KE0IHzzB90LnzSa5fHzDjAQJxbJwcw7ukr+OlfH2e5rMSVm
-koMpxIQKcNIF6QrT+nD5WZZdIQGdnKBWXYMcT9ao1wna+YhS1HdU2jI3s0WeRuMf
-0OeoWXq+reZkxK06g6CJDqsFfFW/lzuIJPJTRmv3o2kjmhr8nL9xEp17cEUNFsdx
-UfYGUK74OgYrttQkXglaiMjDvEVrC4E0yWsHXukA6ypmb1j0D4Q4bNXnQ2j/YjPs
-vkZxQ//RkXPWfuPum9QpLcqsedYUBwwXuFLqx85ybE6oLI4H4X0TXNcQ54CeYeQn
-MvfrGi+3j5IbJ2cCAwEAAaNdMFswHQYDVR0OBBYEFNo5o+5ea0sNMlW/75VgGJCv
-2AcJMB8GA1UdIwQYMBaAFNo5o+5ea0sNMlW/75VgGJCv2AcJMAwGA1UdEwQFMAMB
-Af8wCwYDVR0PBAQDAgEGMA0GCSqGSIb3DQEBCwUAA4IBAQDFrcL4kBz/hAP6InxR
-AShK6PCLo/UX9VAICbYf5/ZTze04Do9Ly0JjbmP+uYH1jSInppADmytuXYl4FWk6
-z6WUU7qn1hgdKL/TWqODRU23/gVojVy+JEeBrU1qMlKeqOMfI5EaEkQ1ymGERV0N
-8U+w5OYZQhhlx4kbO50RVxrtk+s+MLL56+Io2xzVQ2P9r+4jwBFQ3NB8GayWS30K
-R4T033AkpU0rruE+mDcHCRfjeOPeCyugujSFDXPzprK5THurXQiSS98Ny/Y7Btcs
-s3PoIvjS2197RLZTFhK6oio/lAourAmyIh/1DxmR/736t03XGrhX9Nm8//b6kSDE
-9FFY
------END CERTIFICATE-----
diff --git a/docker/test/integration/resources/kafka_broker/conf/certs/ca-cert b/docker/test/integration/resources/kafka_broker/conf/certs/ca-cert
deleted file mode 100644
index 43a9f42aa..000000000
--- a/docker/test/integration/resources/kafka_broker/conf/certs/ca-cert
+++ /dev/null
@@ -1,21 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDYjCCAkoCCQDS47Sky1hfVzANBgkqhkiG9w0BAQsFADBzMQswCQYDVQQGEwJO
-TjELMAkGA1UECAwCTk4xCzAJBgNVBAcMAk5OMQswCQYDVQQKDAJOTjELMAkGA1UE
-CwwCTk4xDjAMBgNVBAMMBU5naGlhMSAwHgYJKoZIhvcNAQkBFhFsZW1pbmhuZ2hp
-YUBOZ2hpYTAeFw0xOTExMTkwOTIwNDhaFw00NzA0MDYwOTIwNDhaMHMxCzAJBgNV
-BAYTAk5OMQswCQYDVQQIDAJOTjELMAkGA1UEBwwCTk4xCzAJBgNVBAoMAk5OMQsw
-CQYDVQQLDAJOTjEOMAwGA1UEAwwFTmdoaWExIDAeBgkqhkiG9w0BCQEWEWxlbWlu
-aG5naGlhQE5naGlhMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuO7f
-uTupDTk/qhCTvjotdvmaWuJoleEZf9Orl3eD+fJd86sxdwfe5OxJMwSBZcmRWoSL
-8IZlzugsvNVadXxdIeHsqTudIZs5sn4MiwPzwfsjVfkhzW7qrIbQlr4yr6oh+iya
-0fK2vaYtfD/TMvxQ/tjSwmRESIZlDG8521AXq4qHNwgYb1S7oLn+RY2qPeamH0l/
-4vJghRWIMtyOG6Ezb1semyRjTr692fAdqoVK5xwi681taxbhtleL/qYqiqHN7v+X
-rOTIZhdNSyGqgCUPPY5CQCtT33G8QOiZxOHzOI4evd4CIiDmqLHe4St7ES3kav1z
-RDRooeZUWjxDJDrOfwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQBYjQM7CWnpn7Wl
-t6yqwxTvqKoPyjPUfp6NgXH73WbC2XIUr+RtI8UooA0RKcb37L4iVuI7xqYbhLbR
-Ad38JeU9DZfufo+PMyFa+AT9qVW5G7rwYSVD9AZeD2ksPsIXrFyQUAp+mBD1vrn7
-N0iT7MbKBG1CvC3QrPvf5Kv2+DUTdb0GDCxzqM30f0BgckA78irTjYdo3c9gpP7R
-XU2erda5Ec5cStKLcpWk9Rntbqx72btxW3j+xRGDmh+4lcTYALeio0g7FlXot+qs
-fk6/ohKP7BCYwXuFeNeDk7CBfzN9M2lr/zZa/POEUUc1uC5XoEDEf9e+Ff+dCmiA
-+wJWpjTz
------END CERTIFICATE-----
diff --git a/docker/test/integration/resources/kafka_broker/conf/certs/ca-cert.key b/docker/test/integration/resources/kafka_broker/conf/certs/ca-cert.key
deleted file mode 100644
index 35f20702f..000000000
--- a/docker/test/integration/resources/kafka_broker/conf/certs/ca-cert.key
+++ /dev/null
@@ -1,30 +0,0 @@
------BEGIN ENCRYPTED PRIVATE KEY-----
-MIIFHzBJBgkqhkiG9w0BBQ0wPDAbBgkqhkiG9w0BBQwwDgQIwWGIq3k/AtsCAggA
-MB0GCWCGSAFlAwQBKgQQ5GwoeX1PJFk4Tzdd9LeqvgSCBNDZtUnoDzxV5njiHdW8
-Mn5h2V58jcDLvWzkrhsfjqevWWbdMtnvK9IzY3wGndxk00hSuGxz9awC0GeE1OLC
-I05e3PjFgpLdYx/6GltDijnoozVNo+pXgjMyEnGl5DsKsJcxOF3Bf9EvC0lwRB+y
-4CnHKWZl0bPddpRR/IDFbSie9+7X2EnSRm5KaB5q1uTXGTI8KZTYuzxID4CxxTff
-s2GvTiqZ7PSVLXjyMTSuLLrfQxuPFNfRWJG1A3LSkxzAoRfv2wbfbqPAmIPHIQCL
-yKxUK/H5RqiJkWhM/wQFXnKieBftjtcS4yQ+JOW4LDwsHI05sYryhl5X4zVwhe7b
-SlotRCQzgOVCO6SRW1WvlJA+oz426kpOSnH3LNUaOxYxPwL3NYGZTUsR9+Bn1qRc
-XAoSdceuxPkmweNSWI0vDLR8neP548IoXsPD5pDM/O8VbjhBO67huYc2JfwfixZ7
-BlZGsoCPu7W0pfn1lXumgcBDotUPdumpiQNo4pMmcvyFU2kKzMaKg27Xf/FqZkQT
-uOBk760iamUEM7PrFNAOu6fxJ9MUGNDx5FgsgzgC2VKOyshysRopQgeWNoNmkY8j
-6ItkJdFGnbc9r1b93mpDT6yuawKOcc8KCnNUAVfoa9ZSEc6hu8W9j0rRfsfJDqH3
-OxczxlCYs4gAhOOVMsNrUBeEmT8Rq88kCSo1EvdyNSRPXwJ/6YdQPNEHEFFlMU/G
-1BPRZwDfe/aKNb1Y+OTTT8ix9L1fbGxShgcMG3zrqZTqJ1pGkN9iZqDXEHY1HTwq
-oqX0QfpgSD5p+Ja7d1KmhDYeM5uXAHS0O/lC3AQyH4PTk4DeTRM9JVpi6WuIxBcP
-T16ZTYzAkdmeqtbDYiRIUZxWLS4hs/UXKlMIm03nNiURMPul/9azJ2bahN34++NY
-DSBWy3X9nCRmlNJb8vdZl1W5MzOg8XlHzJYmFZ4Qip4C8Ec6fZ+yGjqrPRzbz1NV
-nDoOpu/0DQAlka68pZQyUBYbBZf8MmIoYvBmzvQVP50elM5TB/BlX6s92K+mDLfI
-l4JdtVMoCaszvuCHcqph869WbRMdmq39AgacteqH+xMOe9F2DblTpKCoJva6rk0V
-yxisBbG0ycI3KNcpYPiZZ5mzu6bzzuQIH72/LAGizS52A+evW3QK+OKx5C6t727r
-BQ1314VxkvJ4JRgKdIUiGzZWprhC6X59YyFYG1PUpaAq4mFQqonB81pTpjbTjjW6
-87D+pKEmY0GcTxjdqyuqL5tns6H4MjqByEgSGhfxaMWausKYbIANhrfB4Lk9YLXm
-icGESQz+spXKGeCRUbsFkYA5CLQpdXQoFKCJyzHLvjXZAFrE+Y0dfTdnxqO93+cr
-VvpxMjE5XYsLduBwLr47wXT5XZMbKkoVJiQPqo3dl3OHFJVyZRyI3Wc+6Xq20B+F
-HDb1cXDXHXof1wGswP5HPSJlUhPkHRGxtuk3r6muf44jLNH8cvfAG9bq+dAUdMJK
-rjcwrFmZQnvO14jSIY5hh5khugT+3/LPJWHpXpjD6uj36ddxKllZfEA3SX2uXdRy
-HtF+XdC7RhXfFOiNpLVK7Dj8w5lNXBpMK3107u6pnsjbNE/Z3IggW6ESSgjN8VGq
-ZVKnlFJAdMoNTsjyrDuO9nZDPw==
------END ENCRYPTED PRIVATE KEY-----
diff --git a/docker/test/integration/resources/kafka_broker/conf/certs/ca-cert.srl b/docker/test/integration/resources/kafka_broker/conf/certs/ca-cert.srl
deleted file mode 100644
index 3349fd991..000000000
--- a/docker/test/integration/resources/kafka_broker/conf/certs/ca-cert.srl
+++ /dev/null
@@ -1 +0,0 @@
-B07417C56D5A7969
diff --git a/docker/test/integration/resources/kafka_broker/conf/certs/client_cert.key b/docker/test/integration/resources/kafka_broker/conf/certs/client_cert.key
deleted file mode 100644
index cd2613613..000000000
--- a/docker/test/integration/resources/kafka_broker/conf/certs/client_cert.key
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCvyFiPYJvayOq7
-Y2aFFyQIKNUTK7VFK15rzEATl5aSvQbLOmNK2jeD3qKlC2E5wc/EpggZOfo16n6l
-Nf4rBp9RSaVnnMVeAzqhhS7Wr2W/UUX+NKXs6r05ROLQMmQpIq6O4UZI51oXJIEL
-i8Nw8ZY5MpMesWBYPQjJK3GgtuFh/W/O1l51y4gmKyDO+TyPp16tVIgUvkm3dVAh
-VXKoNG4y53uvQ0u3S9vWaghg4eRD9ouRxOfvpzWht1XcgLABU0R1vCYDHYxW40to
-SD+hSI2gSLPiYEeYw88XzQcApl6DhSUBQ+No59Ieag6Fv5xPL6X43f1HMloosVOe
-Jbxv/w2VAgMBAAECggEAAUfe6VCFYi8zkqDzRsX3471UbBrV68aYngVzdBEXLkOF
-UKbAwXNcky/klHpEP5T7bA3fojtraMgoGxENJIgcUg3M0DfysjAo1bNwN204uw+f
-CG6exN6i+U9YrbaoRypzBiR7DB3vaJrB+/ENP/3qn8kQDFmzf4EWVszju42Jj9qz
-RKXKaUubRXSgH+mwb7F2jHi6WXOg1L8Tb6lnXYwH7UDHimi2RkM49sT9QuvRf4Sv
-R+iDnOkLK+BlQ685uBZiagp4sx9CtMXnD3vmwHuxqqyrA1d6Lei4AZ9Mii7rs25X
-JP6EXMs8K26Sl2LN7QprOoRcN1xIYcbJdNDbB34gUQKBgQDg3hjX4uiKWzDMGHA1
-WzbTZm8BgiFymiuZsq+Pr5pVovAw53IRF328sPYWLSF49yRSesjIsPB1NZGW3XRF
-//fEp5ak/wUEA0cmjJbQbzWhdWWsptU9rk6FIbNRRPa2avB+dBQPGDUlHYSG7t/n
-skXXWPlUGw3Daa/t+lA1cucGqQKBgQDIHowrp4SY98y3m4+K4wCh1EkSrFvRSMOV
-VUSQs8PthSIGUs5zMvkoiGSmY2yPCj5cJ/uTFrBGOoUZ06C0W+g7OKDzsluj60Zz
-/12sG5Xsxi8Ou66uB//sOX0TB00SVBGHAj5hr4o3xUG7FGIVvBeAYnWOyLExHlhk
-gzs0xlNfDQKBgQClILNhpZFaOIjm93M+21KxfuE5XFQvkiM+ZMcjXrKY/GQhpXRQ
-NM3yyIiN1GpmKCaWIvBa4ARULhhRjyIZXVSXJvIrC9KlFbskBc2iFmhtvoYBeSnS
-d7PVc9/4aAo7/t/IHHm2c/Ji5QAzDiuCO5mp//5n0s4SmAPHYC+ScGRSsQKBgDP+
-E8bnwACKKQrvBK096xUgdVe/ph9Ku13aCHVb7ySPuVZRP0gYzzkjrpq4LCVBsQ/6
-Z1dAxs2IOLqMDC2vzzdLsC2AgmSlJOnQSAMGK76XgvRd+LVIaMpz6TX/aJvBcchU
-4eY5WDzDKB+13QHum2CEtCxDoh2z151Npx1ABbslAoGBAL286PjyATiRtEvf+klS
-o7jMkO7ww1zmR0cYcK+SDDCkiD7eTPLOyFSfk01Yvm+xuDSIcaQVipB6l8oHk18i
-IMUbaIXdjxjv29EHffUsJTRmDk1w/6HQfAqGNNpj3oeZDady7HDqqkGWXWKxI7yo
-NOJXFARfQ8yC/z+YxF7wptYZ
------END PRIVATE KEY-----
diff --git a/docker/test/integration/resources/kafka_broker/conf/certs/client_cert.pem b/docker/test/integration/resources/kafka_broker/conf/certs/client_cert.pem
deleted file mode 100644
index d7b438233..000000000
--- a/docker/test/integration/resources/kafka_broker/conf/certs/client_cert.pem
+++ /dev/null
@@ -1,19 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDEzCCAfsCFC1rL3aQfqwrv4UcotoV7y7UrfXNMA0GCSqGSIb3DQEBCwUAMHMx
-CzAJBgNVBAYTAk5OMQswCQYDVQQIDAJOTjELMAkGA1UEBwwCTk4xCzAJBgNVBAoM
-Ak5OMQswCQYDVQQLDAJOTjEOMAwGA1UEAwwFTmdoaWExIDAeBgkqhkiG9w0BCQEW
-EWxlbWluaG5naGlhQE5naGlhMCAXDTIyMDgxODE0MTA0NloYDzMwMjExMjE5MTQx
-MDQ2WjAXMRUwEwYDVQQDDAxrYWZrYS1jbGllbnQwggEiMA0GCSqGSIb3DQEBAQUA
-A4IBDwAwggEKAoIBAQCvyFiPYJvayOq7Y2aFFyQIKNUTK7VFK15rzEATl5aSvQbL
-OmNK2jeD3qKlC2E5wc/EpggZOfo16n6lNf4rBp9RSaVnnMVeAzqhhS7Wr2W/UUX+
-NKXs6r05ROLQMmQpIq6O4UZI51oXJIELi8Nw8ZY5MpMesWBYPQjJK3GgtuFh/W/O
-1l51y4gmKyDO+TyPp16tVIgUvkm3dVAhVXKoNG4y53uvQ0u3S9vWaghg4eRD9ouR
-xOfvpzWht1XcgLABU0R1vCYDHYxW40toSD+hSI2gSLPiYEeYw88XzQcApl6DhSUB
-Q+No59Ieag6Fv5xPL6X43f1HMloosVOeJbxv/w2VAgMBAAEwDQYJKoZIhvcNAQEL
-BQADggEBAB5XS9mMiZ3/JtdpdXnFparqdJ/6ZstNHCB6wIKRhtMA+Kmdab3gQb5w
-UTpBxSJjyBHZKbrytzwJ+Hvlj2daOEDWSar7oE5gdZdxRfFUTxLJucKLptrkDjI3
-GirYq7/q7bP/d4hl/WaDsAIlSB3B4pbAzwwLVbBZysIFLjVFGhTFDf7Ko7JjGIbF
-4ZRLkJ3y1HEI/1lL1L9zw4rFOIRoNOp6lLoLaVeyWTSNl2+pA9iXwxCd2OLM/yEI
-p/qE449uebbVlhEPt6cuvUyAzz4vxsVLDvocifeTbkOjYQHOFBWIjwpGVwH+kS1I
-O6WclYAq3N/uYLZ6J4YG9o/P8pCbs+o=
------END CERTIFICATE-----
diff --git a/docker/test/integration/resources/kafka_broker/conf/certs/server_cert.key b/docker/test/integration/resources/kafka_broker/conf/certs/server_cert.key
deleted file mode 100644
index 7fea0e83d..000000000
--- a/docker/test/integration/resources/kafka_broker/conf/certs/server_cert.key
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDOYfl9K7m4cB+O
-fk/3XnofnN9rc50plLGz8PCB8WL50Lk6ur9whqJ98sJBqINzPYIVL+p1xyWsMW2C
-QRpn1uYMYcTvRUJ5afPBafow/wuekZVdL5LubkDzR+Cx/eSLe0kJ/cyp/YmkIWvh
-8VoVlMkQ1DvZNKFpyhSiQ8aOhDxqjK6HswmDjnpdz4nQ5SEIUtKaSrgZnEwMr8o5
-MXoEEyrHyBVEAU9JFeQcALP9bCLo4BCJ308pbXoK3JKVmTeRFGSt1yx2f+SgzWc8
-SdKaOns+toF9g+iufxxPNeCiUfK1ZOrluIB7W6/LbafXOpzOen9U0KQVJP6/vLrR
-2qHDSprNAgMBAAECggEALwaKB6hv8AO0XQ6RAAeKAiwHiH9Wf0P/XcX3t1QePlf9
-7NDikCxNWC95beaM1TRdsKAzZ1TSU7MLhdN5xdFkp8ubSP3aELpArYCNXp6ushnw
-E8SkUSnUKNrm6we0DMgxs1IeUVmRBFINN/V7llFJo0kIvgxM/0LpgrO4yX5+GbS9
-v1U4FhvWQrZEBXi+0bnYAHHehbW0qVe94R7GY+AdrxWH8DxJjxiYLy7HdTrjCgiL
-VXbWIMkq4FDsmwkAnlaFPsrowGuqkIjmzEZysbkqf/hzvGXDQHzhfuzhZjbGdW8N
-BX+A8lt3mZMdJqqr2aUDofhg88mhR1Lg7SR9h2NjIQKBgQDtBZfIlTNmWmSe61y0
-jJMHlJCAViyjEnkZWXQ2ZQw53XGH/AF6HxXC2W5Upq0tjsS0l0qrcaTnDFNBj9eT
-7JEkOZzojQ6hfXE1Q1T8SLNTljub9hQkwPQvoqJs0VCECUyWS2gAMS9nFWL2T0KM
-7iyo6JzQN73Hy9Sz3uoQF9/NhQKBgQDe6Flsy4WjdWTe3dbvUAFjRDvfahTqHvHX
-AL74P5T/XRvAC9/HQRtiA43IwKJBibQiKFbCaSjW2yf6gFH0+0FPEmNzfvEDkUqv
-rksaPK8lcb4WA0DvnylYcJo8ThRl+h9RLo9owSPVSPJ1KAnZC/P8MIJuQQebDYzJ
-WluVL6CWqQKBgEowAvc8hA8YYgBACQ9WJLstrCEPV/9ey9q8XhycejwubNe0cD3q
-NuR7LAaA+RmWY0lnRW7uLkBHYlPWExLw6+kbw1oitJ6lZebXAWXJ159lGZlSZ7Gt
-V4EFdbN7vKjnRDqStaXBv2WTkxsnadw0vHPOAy9FEzjTOutl7XtGDb4FAoGADV0O
-p2gYLfGGx5mzHgvE2Yeosz8v281ZIFOqH0Xoj9qsJDpSAycTFjAyWtu5noY752RV
-tzgIu8DNx8XTpAVAKBZpIe51UopuWNPxKrSCp5MHG1i8luEK4IN3q6/epzJ0mDBb
-ZqNVm57Q8lchKAMAcJkT6Lo7U39fdl7VaL8/OGkCgYBedkOtab0TyKU5AlFjP1Dq
-vfoRQD95S/cfCeLCdWo1XPI4hyQDC8oLY11bWB7mHHpWZEdWt98uJS9yEBKs3Jcu
-XPsNjqghfoF5I29eJkOoWR0nz0IZTMyRCxKdymcOPyVLvXg/j1MIRtuQso/MZpAg
-Sw5g5qAC27VPvf4KyRtphw==
------END PRIVATE KEY-----
diff --git a/docker/test/integration/resources/kafka_broker/conf/certs/server_cert.pem b/docker/test/integration/resources/kafka_broker/conf/certs/server_cert.pem
deleted file mode 100644
index 01d3b0474..000000000
--- a/docker/test/integration/resources/kafka_broker/conf/certs/server_cert.pem
+++ /dev/null
@@ -1,19 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDEzCCAfsCFBcC4R9vi7MHlWx6fjCZYLb407ukMA0GCSqGSIb3DQEBCwUAMHMx
-CzAJBgNVBAYTAk5OMQswCQYDVQQIDAJOTjELMAkGA1UEBwwCTk4xCzAJBgNVBAoM
-Ak5OMQswCQYDVQQLDAJOTjEOMAwGA1UEAwwFTmdoaWExIDAeBgkqhkiG9w0BCQEW
-EWxlbWluaG5naGlhQE5naGlhMCAXDTIyMDgxODE0MDgzMloYDzMwMjExMjE5MTQw
-ODMyWjAXMRUwEwYDVQQDDAxrYWZrYS1icm9rZXIwggEiMA0GCSqGSIb3DQEBAQUA
-A4IBDwAwggEKAoIBAQDOYfl9K7m4cB+Ofk/3XnofnN9rc50plLGz8PCB8WL50Lk6
-ur9whqJ98sJBqINzPYIVL+p1xyWsMW2CQRpn1uYMYcTvRUJ5afPBafow/wuekZVd
-L5LubkDzR+Cx/eSLe0kJ/cyp/YmkIWvh8VoVlMkQ1DvZNKFpyhSiQ8aOhDxqjK6H
-swmDjnpdz4nQ5SEIUtKaSrgZnEwMr8o5MXoEEyrHyBVEAU9JFeQcALP9bCLo4BCJ
-308pbXoK3JKVmTeRFGSt1yx2f+SgzWc8SdKaOns+toF9g+iufxxPNeCiUfK1ZOrl
-uIB7W6/LbafXOpzOen9U0KQVJP6/vLrR2qHDSprNAgMBAAEwDQYJKoZIhvcNAQEL
-BQADggEBAJbO7Pqnhcq0dXVXUVC7L6Smh52jhER3ICO8KUIV47V3i38qz7hdhOx9
-XOODdDITTB8kWM6RuRtHnjh8RHynFyJWB3Vt+/czaODY+deivy4bFqVfsSHj+vvY
-l1q0fz2p7rLcywCDVAK77v2sRWUn+TwJzon3WZTa16iYsi/UBVDnDp2jc9+y4CS1
-Q2IMXj+LLHPD9na57+LWsPSqOQ+dP3tS/klmm+ahbBJa8HpEsxtLfUPynS7PNbe9
-F7hmgauEjgQriGPFXxXAjDioEiWFEL/C1ostJXTBkhkzkpSjgnH94qyFIILytIP6
-l2MtQn0sPKpsYgpddpkxUSSduewHjB4=
------END CERTIFICATE-----
diff --git a/docker/test/integration/resources/kafka_broker/conf/certs/server_keystore.jks b/docker/test/integration/resources/kafka_broker/conf/certs/server_keystore.jks
deleted file mode 100644
index ddc0c3a0a..000000000
Binary files a/docker/test/integration/resources/kafka_broker/conf/certs/server_keystore.jks and /dev/null differ
diff --git a/docker/test/integration/resources/kafka_broker/conf/certs/server_truststore.jks b/docker/test/integration/resources/kafka_broker/conf/certs/server_truststore.jks
deleted file mode 100644
index 93a013d7f..000000000
Binary files a/docker/test/integration/resources/kafka_broker/conf/certs/server_truststore.jks and /dev/null differ
diff --git a/docker/test/integration/resources/kafka_broker/conf/server.properties b/docker/test/integration/resources/kafka_broker/conf/server.properties
index 61f67ea5b..fd4bf0918 100644
--- a/docker/test/integration/resources/kafka_broker/conf/server.properties
+++ b/docker/test/integration/resources/kafka_broker/conf/server.properties
@@ -174,8 +174,7 @@ ssl.keystore.type = JKS
ssl.keystore.location = /usr/local/etc/kafka/certs/server_keystore.jks
ssl.keystore.password = abcdefgh
ssl.key.password = abcdefgh
-ssl.truststore.type = JKS
-ssl.truststore.location = /usr/local/etc/kafka/certs/server_truststore.jks
-ssl.truststore.password = abcdefgh
+ssl.truststore.type = PEM
+ssl.truststore.location = /usr/local/etc/kafka/certs/server_truststore.pem
# To require authentication of clients use "require", else "none" or "request"
ssl.client.auth = none
diff --git a/docker/test/integration/resources/minifi-c2-server-ssl/Dockerfile b/docker/test/integration/resources/minifi-c2-server-ssl/Dockerfile
deleted file mode 100644
index cb6303347..000000000
--- a/docker/test/integration/resources/minifi-c2-server-ssl/Dockerfile
+++ /dev/null
@@ -1,7 +0,0 @@
-FROM apache/nifi-minifi-c2:1.20.0
-RUN mkdir $MINIFI_C2_HOME/certs/
-COPY --chown=c2:c2 certs/* $MINIFI_C2_HOME/certs/
-COPY --chown=c2:c2 config.yml $MINIFI_C2_HOME/files/minifi-test-class/config.text.yml.v1
-COPY --chown=c2:c2 authorities.yaml $MINIFI_C2_HOME/conf/
-COPY --chown=c2:c2 authorizations.yaml $MINIFI_C2_HOME/conf/
-COPY --chown=c2:c2 c2.properties $MINIFI_C2_HOME/conf/
diff --git a/docker/test/integration/resources/minifi-c2-server-ssl/authorities.yaml b/docker/test/integration/resources/minifi-c2-server-ssl/authorities.yaml
deleted file mode 100644
index 8043a37e6..000000000
--- a/docker/test/integration/resources/minifi-c2-server-ssl/authorities.yaml
+++ /dev/null
@@ -1,2 +0,0 @@
-CN=minifi-cpp-flow:
- - CLASS_MINIFI_CPP
diff --git a/docker/test/integration/resources/minifi-c2-server-ssl/c2.properties b/docker/test/integration/resources/minifi-c2-server-ssl/c2.properties
deleted file mode 100644
index 7d548e8de..000000000
--- a/docker/test/integration/resources/minifi-c2-server-ssl/c2.properties
+++ /dev/null
@@ -1,10 +0,0 @@
-minifi.c2.server.port=10090
-
-minifi.c2.server.secure=true
-minifi.c2.server.keystore=/opt/minifi-c2/minifi-c2-1.20.0/certs/minifi-c2-server-keystore.p12
-minifi.c2.server.keystoreType=PKCS12
-minifi.c2.server.keystorePasswd=abcdefgh
-minifi.c2.server.keyPasswd=abcdefgh
-minifi.c2.server.truststore=/opt/minifi-c2/minifi-c2-1.20.0/certs/minifi-c2-server-truststore.p12
-minifi.c2.server.truststoreType=PKCS12
-minifi.c2.server.truststorePasswd=abcdefgh
diff --git a/docker/test/integration/resources/minifi-c2-server-ssl/certs/minifi-c2-server-keystore.p12 b/docker/test/integration/resources/minifi-c2-server-ssl/certs/minifi-c2-server-keystore.p12
deleted file mode 100644
index 7d280f90f..000000000
Binary files a/docker/test/integration/resources/minifi-c2-server-ssl/certs/minifi-c2-server-keystore.p12 and /dev/null differ
diff --git a/docker/test/integration/resources/minifi-c2-server-ssl/certs/minifi-c2-server-truststore.p12 b/docker/test/integration/resources/minifi-c2-server-ssl/certs/minifi-c2-server-truststore.p12
deleted file mode 100644
index d72e6ebd4..000000000
Binary files a/docker/test/integration/resources/minifi-c2-server-ssl/certs/minifi-c2-server-truststore.p12 and /dev/null differ
diff --git a/docker/test/integration/resources/minifi-c2-server-ssl/certs/minifi-c2-server.crt b/docker/test/integration/resources/minifi-c2-server-ssl/certs/minifi-c2-server.crt
deleted file mode 100644
index 0fcfff9f0..000000000
--- a/docker/test/integration/resources/minifi-c2-server-ssl/certs/minifi-c2-server.crt
+++ /dev/null
@@ -1,20 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDMTCCAhmgAwIBAgIUasTOqNQ87C9VtUiy+N8uLOQTcRUwDQYJKoZIhvcNAQEL
-BQAwDzENMAsGA1UEAwwEbXlDQTAgFw0yMjEwMTQxNTAyNTBaGA8yMDUwMDIyODE1
-MDI1MFowGzEZMBcGA1UEAwwQbWluaWZpLWMyLXNlcnZlcjCCASIwDQYJKoZIhvcN
-AQEBBQADggEPADCCAQoCggEBAKsIVaHbxd9HGI/5Dc9Q75vv7/dDLrlEU7S9/k1S
-lJkBBnF32rBNlwNfZdCgSsdtFzsq++88CQjzbbCdi72hwKA9V60oW8HDI8utDdE6
-QY4qTBhBmrxbwGIE/6TNkSpfz940a18Nhzv0mDTvpjLsYqbDAA6vr1/Od/BSqnXq
-/DLtIEEzX1d1gtRqRonyt44Hhmj58gORka8Kggyh84i9k/Ol9IZdCZoxvgDQOwOK
-XaohByQBPChMKpMDQkpJfkIKRGE1K2Ya7BfGqO0Q+lX6N/zSFE9YIZxrhcT6wjcC
-yjOJGou9vVgnu6Idw6iX6Y0pwOrnWPqpQ/sqIxFKWu2UAy0CAwEAAaN3MHUwHwYD
-VR0jBBgwFoAUbor1/5iMMRzmWD4nVDkQ0IkzMawwCQYDVR0TBAIwADALBgNVHQ8E
-BAMCBPAwGwYDVR0RBBQwEoIQbWluaWZpLWMyLXNlcnZlcjAdBgNVHQ4EFgQUqmzn
-O9XoHVK6afH0IDN3oV4Tv2cwDQYJKoZIhvcNAQELBQADggEBAKdtUTXWcUcSr9U7
-CFaU5DFtJbUIvEkbAs2R8+bK/Y280uOEQ4+9plyCcVPF5/ZvHaRDu0orKwaUWEdZ
-eAjYj2isZACXZdWey6b5kE2s8XAGYrM3W8R6gZzSf0rEdjPyA3TwNb3/RgltBKWV
-nzqNiCjo6Cb4dY2MXKOrmlVtUDKEKDa0zl2FYsrJDJ8KS9chROX01r+kcWXiemXP
-bWWfCdhBCJA6gAQfOHz4Cc/fCLwsMi9xO8ddRNBl/jxJ8Sl9UlImSWlp5BSprSO2
-hu+ucHwoVZbC9As1bkEEwODBUCO5L2XKl5O8MuA/NxgT57L7G1UCxBM9sTWb4bEI
-xd6prvQ=
------END CERTIFICATE-----
diff --git a/docker/test/integration/resources/minifi-c2-server-ssl/certs/minifi-c2-server.key b/docker/test/integration/resources/minifi-c2-server-ssl/certs/minifi-c2-server.key
deleted file mode 100644
index fc18ef48e..000000000
--- a/docker/test/integration/resources/minifi-c2-server-ssl/certs/minifi-c2-server.key
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCrCFWh28XfRxiP
-+Q3PUO+b7+/3Qy65RFO0vf5NUpSZAQZxd9qwTZcDX2XQoErHbRc7KvvvPAkI822w
-nYu9ocCgPVetKFvBwyPLrQ3ROkGOKkwYQZq8W8BiBP+kzZEqX8/eNGtfDYc79Jg0
-76Yy7GKmwwAOr69fznfwUqp16vwy7SBBM19XdYLUakaJ8reOB4Zo+fIDkZGvCoIM
-ofOIvZPzpfSGXQmaMb4A0DsDil2qIQckATwoTCqTA0JKSX5CCkRhNStmGuwXxqjt
-EPpV+jf80hRPWCGca4XE+sI3AsoziRqLvb1YJ7uiHcOol+mNKcDq51j6qUP7KiMR
-SlrtlAMtAgMBAAECggEAReBUWBo52BSseNnopfxrwLqBQHTeyIudZVlAZifolTBh
-iQdOPjydB6A4sUlj8+FineZcYuwUxubpuEBNwO6ui+k0AodcIahP3h14aTSTZvlp
-+HkJNo6H5aQkLBleh0D45NBm08FrsHeonewRa3m/fmFqCxYFIS/yOaoUgbO9UTJ7
-7nQZfurke8a3dxk6zQqrRkAp7MdXiVRYkOv3NOBk6efM8DjvYJSCpC1I0h2XRxJM
-9ZvCvGbmDHpGWnFkSU3Sc19H7latDp+Tef6Oi31tXVDTjmn1Dgyp3fXw6y1qrj99
-7V+lBQO0TEXW3wrqhMfTvJG9VHASEBaeP1cmXiLj4wKBgQDeiwRaw6/APc9bE+rz
-wer9MQ4UzfmZ3nZH0uYggZJhikCQqFZLyuGT041TX3YrNkmqvDD2ue29GA0H0LBa
-GSQkmw0CtBzT/hxjBO5pib1GjdsTRaB82VIXOmSrjgMR2gND/AguOnK4R2EgakXB
-Ah25+7HZ91aHfbeyesrjUneiSwKBgQDEvtdA2+p64Ryo8nR+RaVpqcdjJRa9Iywz
-QMUwJg8uM7bBap3xThtt1otB8MhetqhL916bZHBBK9AdfSIoTq7Q8lrlbE+lQDub
-ytlz0qOv0GsIVKf7pvgGy0IyZTo8ExNrk0ZXi83tEDBH90VCkPZAdYMkTmgRG6Lw
-frwuperFZwKBgG/X/BtFp8l9Bv5mFznkpp4TDlmkXyJWrKlSM/f4RsIgwmwxPhWf
-ZBlwQ+G342K6SPG23QDS1smnEb1ww4C0i/aduj82mBpu5oNZUhzWbbrMxmJ8Jrk4
-W0pzPW7+00oggG2ld9ML6uX0cbrhzia/UoNLHMpHxUQZCb54egkfRCLbAoGAXq9J
-gJlVu1VjKZulnK9/794ZawmKa/PlbbUaMRXf8GhK58KbyGnCoZXC5zUt+QcG76hZ
-C4fGzlZ7jfWO3r8fOseoHwmFOw4yocN561fQFujC2fuD7IRqkTp43TACWq8DhZ4X
-GELcE97anYfO+T4yhMsJFgv14WXfgMY9YmXPGrkCgYA8BVpuN4rAUsJ+4ouePWRX
-Z0FVKzUcenMy+0whYmEjkUtd72ZwCMh1sNp7rgH+3w8U79RJ52iiOLRtcAl9cNEK
-+w8GUdSRr4JQcvhh024eakQ713bZaNZWBZLtgvzDe4HOAC+eRo7v7coHHCBpJzG1
-ihO8WGGFhArEq0YuGxdKaA==
------END PRIVATE KEY-----
diff --git a/docker/test/integration/resources/minifi-c2-server-ssl/certs/minifi-cpp-flow.crt b/docker/test/integration/resources/minifi-c2-server-ssl/certs/minifi-cpp-flow.crt
deleted file mode 100644
index 7b6ac1056..000000000
--- a/docker/test/integration/resources/minifi-c2-server-ssl/certs/minifi-cpp-flow.crt
+++ /dev/null
@@ -1,20 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDLzCCAhegAwIBAgIUI5l5u5iR4/JsSBS/Vt4UlvwJ0lswDQYJKoZIhvcNAQEL
-BQAwDzENMAsGA1UEAwwEbXlDQTAgFw0yMjEwMTQxNTAyMDhaGA8yMDUwMDIyODE1
-MDIwOFowGjEYMBYGA1UEAwwPbWluaWZpLWNwcC1mbG93MIIBIjANBgkqhkiG9w0B
-AQEFAAOCAQ8AMIIBCgKCAQEAqFEBLXiOkjDQ8qt6VOHKrCfQmh0a9FL5VNlfYUvV
-IyuVFpL1+6wc9s3/7yd+M8MwkhbzXHQbh/Hh1VHIrmmcNWVesmCs8vU7f9vueBtX
-mqKEzTN+/apNRSABznkkJoHRJFwQjXq7iznmBh9AHyvBIU/O330b7aKdNWHJsYaK
-lX7HGXYti9nG7dmCxDbZFIWu73JU4fh6PJd/lbRYngHcZ6XoZtsX26bAvWRGqfym
-XQ+RK7yURb87iXEUmToCVlYQRoKTme7uDcJowej/6yjNuru9/VQjDnSG930cY+Ak
-Ssxs73ivn8dnNOvcDOiNeEwjkUONBPFWe8GYiSJhZ6m89QIDAQABo3YwdDAfBgNV
-HSMEGDAWgBRuivX/mIwxHOZYPidUORDQiTMxrDAJBgNVHRMEAjAAMAsGA1UdDwQE
-AwIE8DAaBgNVHREEEzARgg9taW5pZmktY3BwLWZsb3cwHQYDVR0OBBYEFCxfop38
-h8apLKD1KQT5iqbzbOTKMA0GCSqGSIb3DQEBCwUAA4IBAQBbsjR5cdOUWEpGi81E
-jNZ8Wtcxl/ebnl4RRwI2tACn1WHLEj65w0dm2BHzs8e6a3JnA8NDwTDctCsUHV2m
-DtxriBxtcPNcaGtzh2IMwX26a+sVfLR1pP1HJi77AGyLA3Rhm1hBws//KZ6seEbc
-qKoQR0o99UcfASQP1uqBC4LF9k4gzhvE8n+KlTJ5neEwZ4jpD505OiCsN5ZMLa+3
-mCyTGdrO4EXSl2ElvaSSxzf5+FfGE2LT+oSrdwQYFuB+9/qr+c5nU+M8e2E0QLvN
-UusnNOw8wp8rEHiSE6DDhp0PhfxhiOVz3l1oJQduzr7L5aO6kgSlT3nWn3PqPaJo
-G0Vd
------END CERTIFICATE-----
diff --git a/docker/test/integration/resources/minifi-c2-server-ssl/certs/minifi-cpp-flow.key b/docker/test/integration/resources/minifi-c2-server-ssl/certs/minifi-cpp-flow.key
deleted file mode 100644
index c73b44e39..000000000
--- a/docker/test/integration/resources/minifi-c2-server-ssl/certs/minifi-cpp-flow.key
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCoUQEteI6SMNDy
-q3pU4cqsJ9CaHRr0UvlU2V9hS9UjK5UWkvX7rBz2zf/vJ34zwzCSFvNcdBuH8eHV
-UciuaZw1ZV6yYKzy9Tt/2+54G1eaooTNM379qk1FIAHOeSQmgdEkXBCNeruLOeYG
-H0AfK8EhT87ffRvtop01YcmxhoqVfscZdi2L2cbt2YLENtkUha7vclTh+Ho8l3+V
-tFieAdxnpehm2xfbpsC9ZEap/KZdD5ErvJRFvzuJcRSZOgJWVhBGgpOZ7u4NwmjB
-6P/rKM26u739VCMOdIb3fRxj4CRKzGzveK+fx2c069wM6I14TCORQ40E8VZ7wZiJ
-ImFnqbz1AgMBAAECggEAKYizrbDOHa0GIpvF+CQviwPYKe98s0W2WQW6z5uS4Lbk
-d0mUgaIbE5wJx84LCmLkHWikbPAJyyYZADbKOp+8+EAnegT5KIrzP73ZvrGgkHwC
-IVDPyXC42JHpYDXsgcQPA9XkD8V1egmzhVc4z3hQlBPJjMSmm6FBAec7ih8VG4Zi
-fxZ3b4fLmezQvqzC+MBRTMqkWU0A4Q3+IIqe5hTUReglTD8yeu9rzyM8xPYksFV1
-qOatWB22IS+fdU/0xzKp3at/yjU8a4DiTnyosNN/2txazQdU1LdSlNxwDqlTDPbi
-swybSc1a6KLO+ZeD4yGB1NLvPCQuIwR4ugw6dQoMkQKBgQC/05rcrGgju4QBcfSe
-SHagt5nypeGUy/yXIlUbWoetETHx/NXaXtWZ7vxN7S0u09HTr7F4+wu00td3etMA
-dOTuqGU/YEHFoZutDpg+oauZIL+MecM1uZeKKTHN3wQ0qmLsYOS172fNUcZ/gcRt
-4JylyK45c10oE0SvXKeC9bWsSwKBgQDgn/Hhhp57NwNWdlOuA6oXkmSN/UFZWE7w
-EntMcviYqGGtBVW9O3D9SqdbEb0kO2NByNgpjsjUTAGCsyYSa50O3q28vQw4z3qK
-EWyg6Ep51eHDUQDo1cdoIRFrFxrZLOMUoVTfT1uqAI/vHxQ089s60W+WUmU5wUic
-9fJoZdnzvwKBgHEwnI2gEecby73Kjywi2BTnoaiDZ0OUxlwrvwpf9fUSU2VV6p5r
-HSEy2p/k1qduB78gSdl4USUG0GtJB16am0eUCAJIeybxwFlyZjV20jmOEFkEtEJs
-W9YDjsbK1MF61NpkJjCQrrCBk15DpTOsuOI+M0flIc/25q2PP6zP7b5XAoGAO/d3
-Q5YEyTAum+6K+HHR/uj+H0n1ID0LFdxZPleTNm39ZYt/ED3GNFixxQY/UGTqYq2T
-x8RuqP6BiLr69v/ztfyMtU5i7Oe29xUfwvVArLYEx3fgnkg0LABn/gb1C/WHygIn
-/lXZStFLm7LYWiqf5Fv1RlRI4dpP4Fdol6ZZQVECgYBWCtKu2CmRb4cYbYo8092E
-72Jh5B9fuBqcUFELjSgUzxgL/L7P6mx6evCCUj4o4Zzy+U45tm2AYpKAXhkQh33h
-xDFs1t0X2CHSTvES65CCEniN/RKoOBu59fEbZu6hV1tzoz9iU6lgKNkndDeeeYip
-hIGI5QK6mK7sNyOnfE+b4g==
------END PRIVATE KEY-----
diff --git a/docker/test/integration/resources/minifi-c2-server-ssl/certs/root-ca.key b/docker/test/integration/resources/minifi-c2-server-ssl/certs/root-ca.key
deleted file mode 100644
index fbb74cc67..000000000
--- a/docker/test/integration/resources/minifi-c2-server-ssl/certs/root-ca.key
+++ /dev/null
@@ -1,30 +0,0 @@
------BEGIN ENCRYPTED PRIVATE KEY-----
-MIIFHDBOBgkqhkiG9w0BBQ0wQTApBgkqhkiG9w0BBQwwHAQIgbRh5/zItgICAggA
-MAwGCCqGSIb3DQIJBQAwFAYIKoZIhvcNAwcECKS8TpnvoUisBIIEyBOQzTn7rMTg
-Igkc79mdZZsU0XX8Z9USUAEVRnKylsWxnt0BX+DlsgnX6bP33gHUXdCSllVRlyKu
-vpNfJadLUNXGU0ZWpYmnD+jtd0J2j6qVqQGQgSuZ0UpPe/6cBaiZcM2BCeWNr0J6
-f7/HDc8AgyeHyP3v+ZqfO0c9TC6N9BMVBreKLtejI3jbNMJOPMdT3sIdyaKLJonO
-ozIZES3ZhPR9kKDRAgk4NUEaPF41SBOYnlqIR4zfBVbtzn2CHbejvAt32KHlfJMj
-D53GrdKDZdOGTAal90rHw127F75ScF5dyskst1PBO4VxufYPzmo/KgMoZ0TAPDzh
-DUDI0Glff9jep8mrACp8XmEMD9uqd3a/E75KWZWQ5eyUrBrzr1qIdLsGsD0UO00o
-MEEIpkz2Is6VTJMGcNCUXDHbKUgHagSs5T35OHESC/JqWM02i/GCiubPPeNfDEjJ
-wGCwdilMhNcuJR8D3Rb9xyv/NOxsECuO+0rDgV9kCe3WF4p3qey4Pcf3NC9BwmOZ
-QTVpa59rlLqGSbrB/EgETucvBFGy9ZhCs/A3Wuc2yirtzWiAsky95643scQbrUrD
-P+QtrfaDLi3Gj9xEjGzUSoui7MogDaQtIBVa81tioQ7TKzQl1cPBToAes/8vPQ/5
-Jrfc04wVH8jz1YSlP37ho63K7YNVINe+Rplwmf/6S0vQqqEKH0EhPiyqRflJ5Xy5
-HyM1p5vQujg45TsltL9DG/fxiO7uR9D0D/1dz/kMZszAPFeP3yQRaKMCJqMc0hXE
-SjWOpblm8y+RxPwehY7OTCKJtH9acURLsFAeHDqHKcobyPOCv2D3ekhx7ujuzyyb
-BrdGKMsTu7e+sEFZrNAvqBDr7KJuS3YnRX6y9lNySh+WzSBf+AW6Tzpabz5GBMEz
-UdOaq0MvtsYD3GOJvDBHHJS8AAPkqUvZnsztJSl5soKkbPJdUpZF2B3PH93eREb1
-sjUkTG8V7JrdpGrJvqkuVnzDFT2xVlt4dDByEj7wFkRtjX/WEDPX8HvNI34hPGSN
-5m88kTWQncd0h6+lbxKDz5dFZ06uDt3m44YFjVBZUpKhTNKgKnBFh1yEXN/LczVz
-DtNtTVlY+zK2MFdfZ3GB8aGsTajGmXreoom8BbigH0QXjp4t9cZHMbPHz0BfjQ9O
-eEukx+2LyH4+DovqKGUN9iZrqe5VAin9wsgyGDt2N5qJfZj++jmF9r5m9JNhOKLm
-RcZwjnayRfVXILGqoRI/JOlwfximJReOzAvYiVtEhCdjn3J2QJsNAMOlc5oUpwdM
-8MVzXc5/ccsKVzNbsAfBaG5IbTLP+iH2pbCmBhaZeX7X5eB4Fsx5ZKcDqGXHju8Y
-xJNIlgV9ALU3Oqnv1yOGGb7BStgsGVAD8lUfycXZjCTwiXiOMx4/KUWtnBkiOz4x
-R/9FZEGIupfRY2MqkFVbaMpYH3u1pXTVt+hDzEak6NboXbH2hIdQndAA5plxcU5y
-SD+xEiTCHWKxkH/W8mzzzPYXKpr17MjCwqok0+FiR2tT9NkPj7G2Ob0vKfAcBRBD
-Ou4COgYTK2AORxgzgs0fM8Par+sa5p1wuuUILIvAxkqNqh/e3G70xj5OMtdaUrT+
-pIPFUG0yfz/lqoTxfYdtUw==
------END ENCRYPTED PRIVATE KEY-----
diff --git a/docker/test/integration/resources/minifi-c2-server-ssl/certs/root-ca.pem b/docker/test/integration/resources/minifi-c2-server-ssl/certs/root-ca.pem
deleted file mode 100644
index 948ebd5b2..000000000
--- a/docker/test/integration/resources/minifi-c2-server-ssl/certs/root-ca.pem
+++ /dev/null
@@ -1,19 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIC/zCCAeegAwIBAgIUOzrVvbxYHge08GMeuuJmOPgWOtswDQYJKoZIhvcNAQEL
-BQAwDzENMAsGA1UEAwwEbXlDQTAeFw0yMjEwMTQxNDQ0MzlaFw0yNzEwMTMxNDQ0
-MzlaMA8xDTALBgNVBAMMBG15Q0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
-AoIBAQCwXjMjfLWgn8YVWkOCarKc40tGCNrRKITgiKZSQvM44hQw9wbYk/yRj67S
-5bZIr5djvnbmJC5XQSj/vAYL92B37qtualPlw/1ziUURSN1MQKUMgbirKAcGFDMJ
-7g6yrCXrq94A/66cQl1fksXfFk8RzSBPMWfzLNTzy96A/2HriYn3J9GFQV3ggEAg
-VcVJhFiyRBBreZEIar+sasDeR5T1lOfn2Ev7QbiWXTr9IqR1mv4dtqKOv9oYld5Y
-AfoQ0P6yis+RLuNcokNIHgUO+ORpy+CUmIOrCbZgVjSetSlZ8SNQkCXwWhKcObaI
-PK15veDo4SyPv2GDEEBbD1Ccqe/nAgMBAAGjUzBRMB0GA1UdDgQWBBRuivX/mIwx
-HOZYPidUORDQiTMxrDAfBgNVHSMEGDAWgBRuivX/mIwxHOZYPidUORDQiTMxrDAP
-BgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAtnKrcaSoSVgHtFBe8
-HBEbK1tzTXrBeFEEQqNZpRoJ6JSq1oY54UlAlBJdNQJFZOPoeaf9vheevFGFhBDJ
-PwRqE5T6wSp9aohUcnKiBzL8VbTIuUgyWM9gwyg/lipTYK6U5sDROUzfEppBJb0E
-gBp7eBoiEcI4tuih3zncbKKvTNR3b/twAL5yI4MEUNcJsi80xefQt0m39RAwpMIu
-khQNH2YLNOl6hl6wRTylCGB6eWZAtB4qGEDtdCXG9uyCnoUO5MFgZZGFiaqN+avF
-cUU2mOwaoPDbWQ7Q+2m9c1fo/iJ8RThAnFhH5/GQe+OaecW8vHmfCbivj7wkRgTW
-nXzz
------END CERTIFICATE-----
diff --git a/docker/test/integration/resources/minifi-c2-server/Dockerfile b/docker/test/integration/resources/minifi-c2-server/Dockerfile
deleted file mode 100644
index 55f5d18bc..000000000
--- a/docker/test/integration/resources/minifi-c2-server/Dockerfile
+++ /dev/null
@@ -1,2 +0,0 @@
-FROM apache/nifi-minifi-c2:1.20.0
-ADD config.yml $MINIFI_C2_HOME/files/minifi-test-class/config.text.yml.v1
diff --git a/docker/test/integration/resources/minifi-c2-server-ssl/authorizations.yaml b/docker/test/integration/resources/minifi-c2-server/authorizations.yaml
similarity index 100%
rename from docker/test/integration/resources/minifi-c2-server-ssl/authorizations.yaml
rename to docker/test/integration/resources/minifi-c2-server/authorizations.yaml
diff --git a/docker/test/integration/resources/minifi-c2-server/c2.properties b/docker/test/integration/resources/minifi-c2-server/c2.properties
new file mode 100644
index 000000000..01e80a364
--- /dev/null
+++ b/docker/test/integration/resources/minifi-c2-server/c2.properties
@@ -0,0 +1,10 @@
+minifi.c2.server.port=10090
+
+minifi.c2.server.secure=true
+minifi.c2.server.keystore=/opt/minifi-c2/minifi-c2-current/certs/minifi-c2-server-keystore.jks
+minifi.c2.server.keystoreType=JKS
+minifi.c2.server.keystorePasswd=abcdefgh
+minifi.c2.server.keyPasswd=abcdefgh
+minifi.c2.server.truststore=/opt/minifi-c2/minifi-c2-current/certs/minifi-c2-server-truststore.p12
+minifi.c2.server.truststoreType=PKCS12
+minifi.c2.server.truststorePasswd=abcdefgh
diff --git a/docker/test/integration/resources/minifi-c2-server-ssl/config.yml b/docker/test/integration/resources/minifi-c2-server/config-ssl.yml
similarity index 84%
rename from docker/test/integration/resources/minifi-c2-server-ssl/config.yml
rename to docker/test/integration/resources/minifi-c2-server/config-ssl.yml
index 239541288..5167223d0 100644
--- a/docker/test/integration/resources/minifi-c2-server-ssl/config.yml
+++ b/docker/test/integration/resources/minifi-c2-server/config-ssl.yml
@@ -33,11 +33,11 @@ Controller Services:
class: SSLContextService
Properties:
Client Certificate:
- - value: /tmp/resources/minifi-c2-server-ssl/minifi-cpp-flow.crt
+ - value: /tmp/resources/minifi-cpp-flow.crt
Private Key:
- - value: /tmp/resources/minifi-c2-server-ssl/minifi-cpp-flow.key
+ - value: /tmp/resources/minifi-cpp-flow.key
Passphrase:
- value: abcdefgh
CA Certificate:
- - value: /tmp/resources/minifi-c2-server-ssl/root-ca.pem
+ - value: /tmp/resources/root_ca.crt
Remote Process Groups: []
diff --git a/docker/test/integration/resources/opensearch/Dockerfile b/docker/test/integration/resources/opensearch/Dockerfile
deleted file mode 100644
index f9a20682c..000000000
--- a/docker/test/integration/resources/opensearch/Dockerfile
+++ /dev/null
@@ -1,5 +0,0 @@
-FROM opensearchproject/opensearch:2.6.0
-COPY opensearch.yml /usr/share/opensearch/config/opensearch.yml
-COPY certs/admin.pem /usr/share/opensearch/config/admin.pem
-COPY certs/admin-key.pem /usr/share/opensearch/config/admin-key.pem
-COPY certs/root-ca.pem /usr/share/opensearch/config/root-ca.pem
diff --git a/docker/test/integration/resources/opensearch/certs/admin-key.pem b/docker/test/integration/resources/opensearch/certs/admin-key.pem
deleted file mode 100644
index 37ae977a4..000000000
--- a/docker/test/integration/resources/opensearch/certs/admin-key.pem
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC0Gz3ZI/Vwk8nR
-D8w49lH/yC1LAXTh56wx2zqfgKbC/GOBSoZqYKQG9Ss6W0Y/Tt+witKDiuE6HP4s
-d73nbriqhsv41CWIsDpaHDFPsj20r+NzVlAFBgUY5oYl0xb8uqKyVvPAekn/s9T5
-1HXknLoFGzS67tK/HXRhglWCf/3/8O2R3rHB6Qo2UORnNUpdf8PaJuuTfhwzZWYv
-roJB5CJPu27GlMOQ/g2u5R9F2zKBrXw9ud2CZdX+BukQcoDvPAbPq9OVQ7ODc3gW
-YdRj7zzWPrnU0D/rwjH81ZFG801/BIpetpakvvPoz4gZrsLD4fd7vP2otVf6kuuw
-OGoUHPifAgMBAAECggEATm2wbI6Ski/hCddZxJJK2dnXE1ryhW8HlnxTVmLE+5LB
-V+tWhhtIQjlf1h0qBWZ41R1Wl26SLmayrjWEbjPSWXuydcwX/iJTfqrZNE23Pif7
-oV18Ifvhm1rDHtSBpZM2RvfDVFVMRPuP6izBG8i93nxe5WobPpr6c324c8f9mQYa
-3mD1jJm97wFOV8Ccs253GImkW5DXpPZWHOaooa8vI+UOb4cd0miHBUQ5Ui0tjMnW
-A3ivLu8Rh5viPWZ44QtmTTpDy74UxxwMlxr6IXBzTSxTzWAu9hAo9BFN73ObKpsh
-8R8emUDPB8Qzmu+Jqszn7vudCuIHLRwUcUXbWDNBkQKBgQDsxPAmsrZhHif3PvqL
-mOFUqfgbkzJulgBsgruRRrmE5TqpSHnj76LgHaXxDH8JHOvRY/aABg0VWGnLurVN
-sksqvfNZIyD47upMCG3gCS8CikSWbB+Sdy88LsKWK//mf6Vszzb/h8LGRYyuWGpE
-bY/Hkaf+cVQd4HXvO5oXT94KMQKBgQDCvCHE2s76Mo2NcpXDXW1W5Fvjr0he6AX5
-aKjsG24QgBIHmCHHD3EGoQTZo59llt7OCLZNj/QefjZY5l/5MNr1vuR0hn2OzHxB
-6xbQy/y7Aa8Aw4Bh+grqduTC3QqoKSFZivxbU7o2XFgTWQfrnYSZLqs2yHqXDk95
-qQ1H5gSrzwKBgC/ra+9pHCBrygrwZCT8vB42iFCMahiXo15S10N/3OxYuJaqZIEN
-bxmM5pjyvoNJpzCp2qnuj5dSXZcuIqOnAi+UYjwYMAQx1pms2xAMy8bn1RncqEcs
-eBLmI9vg69nq28A58BenknQuf1qJ6ngO9JqfkDCfHzcsxrG0jmtJbtcBAoGAFnat
-TpAPID2EprR5Ijg8zLE7CGDH8GrDhanVEqd3Yt/jNI0QGG5nk2Qvswt077sfqJTI
-stnQR1Q34LlyiMRh/ccvg2GgSkC8dL4xu5RMayYeke/7d1HowHNGoFPvTsav2ix2
-P/LAoFS2tESo7T0WdrUzevZVbedws5AUwB6Am3ECgYEA6+Ka9zgFdwRQWnFZnzo8
-AHVgEfrGySnPhk1TIZaNCCQfW0PbWPat4de7pdmQ8b25Xkmtv5Dvx1DZt7nEXGvi
-CFlrJan7DOXsLCFU4wrOKrAzjX5dyxmOlMQ6yDcUfXNo56nlmu0OzMdXSkeE1NF4
-E178foT4/NXi3o1QPsF4ZOU=
------END PRIVATE KEY-----
diff --git a/docker/test/integration/resources/opensearch/certs/admin.pem b/docker/test/integration/resources/opensearch/certs/admin.pem
deleted file mode 100644
index 615cca17e..000000000
--- a/docker/test/integration/resources/opensearch/certs/admin.pem
+++ /dev/null
@@ -1,19 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDJjCCAg4CFBL2toHOSVHiwhbu9ZZF9GNSmP3BMA0GCSqGSIb3DQEBCwUAMEUx
-CzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl
-cm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjIwNjE0MTQzOTM4WhcNMzIwNDIyMTQz
-OTM4WjBaMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE
-CgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMRMwEQYDVQQDDApvcGVuc2VhcmNo
-MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtBs92SP1cJPJ0Q/MOPZR
-/8gtSwF04eesMds6n4CmwvxjgUqGamCkBvUrOltGP07fsIrSg4rhOhz+LHe95264
-qobL+NQliLA6WhwxT7I9tK/jc1ZQBQYFGOaGJdMW/LqislbzwHpJ/7PU+dR15Jy6
-BRs0uu7Svx10YYJVgn/9//Dtkd6xwekKNlDkZzVKXX/D2ibrk34cM2VmL66CQeQi
-T7tuxpTDkP4NruUfRdsyga18PbndgmXV/gbpEHKA7zwGz6vTlUOzg3N4FmHUY+88
-1j651NA/68Ix/NWRRvNNfwSKXraWpL7z6M+IGa7Cw+H3e7z9qLVX+pLrsDhqFBz4
-nwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQCrKip0y1U4Op0GlOCFn742PXz7jvmX
-RiZ5iJbi0JO9MfVhKvdJH7aMzwzJy5+vqv9tvgbWUBgQrIzINi9+rHJi16oFYsxO
-/41+pFB1ePwlbq8YeEVj79jve+/N/2gJU6Ax5kRgEmJ56ECoM+fhvn9qihPMfViq
-QnvCkhXXDgFrXs5zYgCxRqMKSZOIO3ARwGP9I/SqAV4KSoL+nn9n2WIV19kSOrhV
-BIokJ6ghG1y/eFHs6/YuQJq8Xv0dsSYS8AhPB2ScmpODnzeENbjs9XxvbhWn6mUV
-03TqUwVUi8waNAIosH5ZWOAMcJY9BZt+nHrwdFvPGOwlDlTolluBK4NW
------END CERTIFICATE-----
diff --git a/docker/test/integration/resources/opensearch/certs/root-ca.pem b/docker/test/integration/resources/opensearch/certs/root-ca.pem
deleted file mode 100644
index d46147394..000000000
--- a/docker/test/integration/resources/opensearch/certs/root-ca.pem
+++ /dev/null
@@ -1,21 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDazCCAlOgAwIBAgIUOIX81x7AWgLdLCJp8yU1D/oYRXgwDQYJKoZIhvcNAQEL
-BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
-GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMjA2MTQxNDM4MTRaFw0zMjA0
-MjIxNDM4MTRaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw
-HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB
-AQUAA4IBDwAwggEKAoIBAQC4cE+oIKlQZ6APRyTq/hUhpA2epXpMznwRr8u2hdQF
-fuc2MFhc1IQP4UT6HPpBuVOPNwM+eUIRfL3SFMwb4pemTEm9PoUe9yHFrWMqvojf
-H0IFtXymSjAoaBcNqELELgcb3JMKtDVZsZ5PUEkacvUXj2q4hnkIxbGz1lwQhCNw
-gEvoiITyQHsfSgrjYO9gUmuM434UKLR4V3Al+Z/C6UjGMSYpoVbOKhdjMJ2Q8vtp
-NAurC/K/bsc67FyWZNURAirOCom1Bs/Qs/z1rtT5gQ/4HxPKrG3ognJAg2n+IV6g
-R0GtBeImWUuR8sSyHKkgL6lblPb0hwveyTwYq3mKInh5AgMBAAGjUzBRMB0GA1Ud
-DgQWBBRAcV/RuVoJUrbakhKEi4RG7Scz1TAfBgNVHSMEGDAWgBRAcV/RuVoJUrba
-khKEi4RG7Scz1TAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQB/
-J4ecHjsRkfxaqMOULbXzyrHvgg/pB/5N4u2g30E4W7iNj+7t9sEiMbeB3fML1dk9
-aeVJR8kh6OsFSG6MMIUATwaE3sGQfW54wgnfc9rLAIN/uo735secF8SjE6UgqZjE
-rPLnT/J9M3qmhUexo3YJZy4nzbR18wgIBaqynC3/WCbljt+FgIRXd+LUs8525XiI
-u7zbeBUErDD+tOnEUfHAjfJ49+GWWcM9Hg7s27iLTykHw0pWgMTzJY13t2ycBz6E
-dMx6KTGIAOG6kJeWM6eGiT/q7A++f2ZNZ7rJLg9is7rGRTUf4oJk2rlaiYxH7Jwo
-IfO0VUGkZSaBJjqB/Svg
------END CERTIFICATE-----
diff --git a/docker/test/integration/resources/prometheus/Dockerfile b/docker/test/integration/resources/prometheus/Dockerfile
deleted file mode 100644
index 566dbb25f..000000000
--- a/docker/test/integration/resources/prometheus/Dockerfile
+++ /dev/null
@@ -1,2 +0,0 @@
-FROM prom/prometheus:v2.35.0
-ADD conf/prometheus.yml /etc/prometheus/
diff --git a/docker/test/integration/resources/prometheus/conf/prometheus.yml b/docker/test/integration/resources/prometheus/conf/prometheus.yml
deleted file mode 100644
index 8a75e2aa7..000000000
--- a/docker/test/integration/resources/prometheus/conf/prometheus.yml
+++ /dev/null
@@ -1,7 +0,0 @@
-global:
- scrape_interval: 2s
- evaluation_interval: 15s
-scrape_configs:
- - job_name: "minifi"
- static_configs:
- - targets: ["minifi-cpp-flow:9936"]
diff --git a/docker/test/integration/ssl_utils/SSL_cert_utils.py b/docker/test/integration/ssl_utils/SSL_cert_utils.py
index 59e0498c2..11921d0dc 100644
--- a/docker/test/integration/ssl_utils/SSL_cert_utils.py
+++ b/docker/test/integration/ssl_utils/SSL_cert_utils.py
@@ -110,7 +110,7 @@ def make_ca(common_name):
return ca_cert, ca_key
-def make_cert(common_name, ca_cert, ca_key):
+def _make_cert(common_name, ca_cert, ca_key, extended_key_usage=None):
key = crypto.PKey()
key.generate_key(crypto.TYPE_RSA, 2048)
@@ -126,12 +126,13 @@ def make_cert(common_name, ca_cert, ca_key):
crypto.X509Extension(b"subjectKeyIdentifier", False, b"hash", subject=cert),
])
- cert.add_extensions([
- crypto.X509Extension(b"authorityKeyIdentifier", False, b"keyid:always", issuer=ca_cert),
- crypto.X509Extension(b"extendedKeyUsage", False, b"clientAuth"),
- crypto.X509Extension(b"extendedKeyUsage", False, b"serverAuth"),
- crypto.X509Extension(b"keyUsage", False, b"digitalSignature"),
- ])
+ extensions = [crypto.X509Extension(b"authorityKeyIdentifier", False, b"keyid:always", issuer=ca_cert),
+ crypto.X509Extension(b"keyUsage", False, b"digitalSignature")]
+
+ if extended_key_usage:
+ extensions.append(crypto.X509Extension(b"extendedKeyUsage", False, extended_key_usage))
+
+ cert.add_extensions(extensions)
cert.set_issuer(ca_cert.get_subject())
cert.set_pubkey(key)
@@ -144,9 +145,13 @@ def make_cert(common_name, ca_cert, ca_key):
return cert, key
-def dump_certificate(cert):
- return crypto.dump_certificate(crypto.FILETYPE_PEM, cert)
+def make_client_cert(common_name, ca_cert, ca_key):
+ return _make_cert(common_name=common_name, ca_cert=ca_cert, ca_key=ca_key, extended_key_usage=b"clientAuth")
+
+
+def make_server_cert(common_name, ca_cert, ca_key):
+ return _make_cert(common_name=common_name, ca_cert=ca_cert, ca_key=ca_key, extended_key_usage=b"serverAuth")
-def dump_privatekey(key):
- return crypto.dump_privatekey(crypto.FILETYPE_PEM, key)
+def make_cert_without_extended_usage(common_name, ca_cert, ca_key):
+ return _make_cert(common_name=common_name, ca_cert=ca_cert, ca_key=ca_key, extended_key_usage=None)
diff --git a/win_build_vs.bat b/win_build_vs.bat
index b61db5cfd..e40c8dbd9 100755
--- a/win_build_vs.bat
+++ b/win_build_vs.bat
@@ -118,7 +118,7 @@ echo on
cmake -G %generator% %build_platform_cmd% -DINSTALLER_MERGE_MODULES=%installer_merge_modules% -DTEST_CUSTOM_WEL_PROVIDER=%test_custom_wel_provider% -DENABLE_SQL=%build_SQL% -DUSE_REAL_ODBC_TEST_DRIVER=%real_odbc% ^
-DCMAKE_BUILD_TYPE_INIT=%cmake_build_type% -DCMAKE_BUILD_TYPE=%cmake_build_type% -DWIN32=WIN32 -DENABLE_LIBRDKAFKA=%build_kafka% -DENABLE_JNI=%build_jni% -DOPENSSL_OFF=OFF ^
-DENABLE_COAP=%build_coap% -DENABLE_AWS=%build_AWS% -DENABLE_PDH=%build_PDH% -DENABLE_AZURE=%build_azure% -DENABLE_SFTP=%build_SFTP% -DENABLE_SPLUNK=%build_SPLUNK% -DENABLE_GCP=%build_GCP% ^
- -DENABLE_NANOFI=%build_nanofi% -DENABLE_OPENCV=%build_opencv% -DENABLE_PROMETHEUS=%build_prometheus% -DENABLE_ELASTICSEARCH=%build_ELASTIC% -DUSE_SHARED_LIBS=OFF -DDISABLE_CONTROLLER=ON ^
+ -DENABLE_NANOFI=%build_nanofi% -DENABLE_OPENCV=%build_opencv% -DENABLE_PROMETHEUS=%build_prometheus% -DENABLE_ELASTICSEARCH=%build_ELASTIC% -DUSE_SHARED_LIBS=OFF -DENABLE_CONTROLLER=OFF ^
-DENABLE_BUSTACHE=%enable_bustache% -DENABLE_COAP=%enable_coap% -DENABLE_ENCRYPT_CONFIG=%enable_encrypt_config% -DENABLE_GPS=%enable_gps% -DENABLE_LUA_SCRIPTING=%enable_lua_scripting% ^
-DENABLE_MQTT=%enable_mqtt% -DENABLE_OPC=%enable_opc% -DENABLE_OPENWSMAN=%enable_openwsman% -DENABLE_OPS=%enable_ops% -DENABLE_PCAP=%enable_pcap% ^
-DENABLE_PYTHON_SCRIPTING=%enable_python_scripting% -DENABLE_SENSORS=%enable_sensors% -DENABLE_TENSORFLOW=%enable_tensorflow% -DENABLE_USB_CAMERA=%enable_usb_camera% ^