You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ma...@apache.org on 2026/02/12 15:53:57 UTC
(nifi-minifi-cpp) branch apache-rusty created (now 0e402e4cb)
This is an automated email from the ASF dual-hosted git repository.
martinzink pushed a change to branch apache-rusty
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
at 0e402e4cb MINIFICPP-2718 Windows based docker tests
This branch includes the following new commits:
new 2d943909b MINIFICPP-2708 Controller Service C API
new d1060ebf9 MINIFICPP-2712 Generate modular docs
new 2e3c8f4fd MINIFICPP-2711 Docker tests, missing logs if container fails to start
new 0e402e4cb MINIFICPP-2718 Windows based docker tests
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(nifi-minifi-cpp) 04/04: MINIFICPP-2718 Windows based docker tests
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
martinzink pushed a commit to branch apache-rusty
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 0e402e4cb864b93ac70a66f7af365666d2fa90ea
Author: Martin Zink <ma...@apache.org>
AuthorDate: Thu Feb 12 16:53:10 2026 +0100
MINIFICPP-2718 Windows based docker tests
Closes #2106
Signed-off-by: Martin Zink <ma...@apache.org>
---
.gitignore | 2 +
behave_framework/pyproject.toml | 5 +-
.../{container.py => container_linux.py} | 4 +-
.../containers/container_protocol.py | 37 ++
.../containers/container_windows.py | 371 +++++++++++++++++++++
.../containers/http_proxy_container.py | 4 +-
.../containers/minifi_container.py | 17 +-
.../containers/minifi_controller.py | 79 +++++
.../containers/minifi_fhs_container.py | 119 +++++++
.../containers/minifi_linux_container.py | 119 +++++++
.../containers/minifi_protocol.py | 9 +
.../containers/minifi_win_container.py | 59 ++++
.../containers/nifi_container.py | 4 +-
.../src/minifi_test_framework/core/hooks.py | 12 +-
.../core/minifi_test_context.py | 33 +-
.../src/minifi_test_framework/core/ssl_utils.py | 222 ++++++------
docker/installed/win.Dockerfile | 24 ++
.../features/steps/kinesis_server_container.py | 4 +-
.../tests/features/steps/s3_server_container.py | 4 +-
.../tests/features/steps/azure_server_container.py | 4 +-
.../features/steps/couchbase_server_container.py | 17 +-
.../tests/features/steps/elastic_base_container.py | 4 +-
.../features/steps/elasticsearch_container.py | 13 +-
.../tests/features/steps/opensearch_container.py | 9 +-
.../features/steps/fake_gcs_server_container.py | 4 +-
.../tests/features/steps/grafana_loki_container.py | 13 +-
.../features/steps/reverse_proxy_container.py | 4 +-
.../tests/features/steps/kafka_server_container.py | 13 +-
.../features/steps/opc_ua_server_container.py | 4 +-
.../tests/features/steps/splunk_container.py | 13 +-
.../features/steps/postgress_server_container.py | 4 +-
.../tests/features/core_functionality.feature | 8 +-
.../tests/features/replace_text.feature | 2 +-
.../tests/features/steps/diag_slave_container.py | 4 +-
.../features/steps/minifi_controller_steps.py | 24 +-
.../tests/features/steps/syslog_container.py | 4 +-
.../tests/features/steps/tcp_client_container.py | 4 +-
run_flake8.sh | 2 +-
38 files changed, 1038 insertions(+), 240 deletions(-)
diff --git a/.gitignore b/.gitignore
index d30e23ff6..faaf21d60 100644
--- a/.gitignore
+++ b/.gitignore
@@ -44,6 +44,8 @@ cmake_install.cmake
install_manifest.txt
CTestTestfile.cmake
cmake-build-debug
+venv
+behave_venv
# Generated files
*flowfile_checkpoint
diff --git a/behave_framework/pyproject.toml b/behave_framework/pyproject.toml
index 263750d9f..5bf1f29f2 100644
--- a/behave_framework/pyproject.toml
+++ b/behave_framework/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "minifi-test-framework"
-version = "0.1.0"
+version = "0.2.0"
requires-python = ">= 3.10"
description = "A testing framework for MiNiFi extensions."
dependencies = [
@@ -8,8 +8,7 @@ dependencies = [
"docker==7.1.0",
"PyYAML==6.0.3",
"humanfriendly==10.0",
- "m2crypto==0.46.2",
- "pyopenssl==25.0.0",
+ "cryptography==46.0.5",
"pyjks==20.0.0"
]
diff --git a/behave_framework/src/minifi_test_framework/containers/container.py b/behave_framework/src/minifi_test_framework/containers/container_linux.py
similarity index 99%
rename from behave_framework/src/minifi_test_framework/containers/container.py
rename to behave_framework/src/minifi_test_framework/containers/container_linux.py
index 75660d785..28df65790 100644
--- a/behave_framework/src/minifi_test_framework/containers/container.py
+++ b/behave_framework/src/minifi_test_framework/containers/container_linux.py
@@ -27,6 +27,7 @@ import tarfile
import docker
from docker.models.networks import Network
+from minifi_test_framework.containers.container_protocol import ContainerProtocol
from minifi_test_framework.containers.directory import Directory
from minifi_test_framework.containers.file import File
from minifi_test_framework.containers.host_file import HostFile
@@ -35,8 +36,9 @@ if TYPE_CHECKING:
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
-class Container:
+class LinuxContainer(ContainerProtocol):
def __init__(self, image_name: str, container_name: str, network: Network, command: str | None = None, entrypoint: str | None = None):
+ super().__init__()
self.image_name: str = image_name
self.container_name: str = container_name
self.network: Network = network
diff --git a/behave_framework/src/minifi_test_framework/containers/container_protocol.py b/behave_framework/src/minifi_test_framework/containers/container_protocol.py
new file mode 100644
index 000000000..8713eff9c
--- /dev/null
+++ b/behave_framework/src/minifi_test_framework/containers/container_protocol.py
@@ -0,0 +1,37 @@
+from typing import Protocol
+
+
+class ContainerProtocol(Protocol):
+ def deploy(self, context) -> bool:
+ ...
+
+ def clean_up(self):
+ ...
+
+ def exec_run(self, command):
+ ...
+
+ def directory_contains_file_with_content(self, directory_path: str, expected_content: str) -> bool:
+ ...
+
+ def directory_contains_file_with_regex(self, directory_path: str, regex_str: str) -> bool:
+ ...
+
+ def path_with_content_exists(self, path: str, content: str) -> bool:
+ ...
+
+ def get_logs(self) -> str:
+ ...
+
+ @property
+ def exited(self) -> bool:
+ ...
+
+ def get_number_of_files(self, directory_path: str) -> int:
+ ...
+
+ def verify_file_contents(self, directory_path: str, expected_contents: list[str]) -> bool:
+ ...
+
+ def log_app_output(self) -> bool:
+ ...
diff --git a/behave_framework/src/minifi_test_framework/containers/container_windows.py b/behave_framework/src/minifi_test_framework/containers/container_windows.py
new file mode 100644
index 000000000..2418d3ccf
--- /dev/null
+++ b/behave_framework/src/minifi_test_framework/containers/container_windows.py
@@ -0,0 +1,371 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from __future__ import annotations
+import logging
+import os
+import tempfile
+import base64
+import tarfile
+import io
+from typing import Union, Optional, Tuple, List, Dict, TYPE_CHECKING
+
+import docker
+from docker.models.networks import Network
+from docker.models.containers import Container
+
+from minifi_test_framework.containers.container_protocol import ContainerProtocol
+from minifi_test_framework.containers.directory import Directory
+from minifi_test_framework.containers.file import File
+from minifi_test_framework.containers.host_file import HostFile
+
+if TYPE_CHECKING:
+ from minifi_test_framework.core.minifi_test_context import MinifiTestContext
+
+
+class WindowsContainer(ContainerProtocol):
+ def __init__(self, image_name: str, container_name: str, network: Network, command: str | None = None, entrypoint: str | None = None):
+ super().__init__()
+ self.image_name: str = image_name
+ self.container_name: str = container_name
+ self.network: Network = network
+
+ self.client: docker.DockerClient = docker.from_env()
+ self.container: Optional[Container] = None
+ self.files: List[File] = []
+ self.dirs: List[Directory] = []
+ self.host_files: List[HostFile] = []
+ self.volumes: Dict = {}
+ self.command: str | None = command
+ self.entrypoint: str | None = entrypoint
+ self._temp_dir: Optional[tempfile.TemporaryDirectory] = None
+ self.ports: Optional[Dict[str, int]] = None
+ self.environment: List[str] = []
+
+ def _normalize_path(self, path: str) -> str:
+ """
+ Converts paths to Windows format (C:\\...)
+ Handles leading slashes and ensures backslashes.
+ """
+ clean_path = path.strip().replace("/", "\\")
+ if clean_path.startswith("\\"):
+ clean_path = clean_path[1:]
+
+ # If it doesn't already have a drive letter, assume C:
+ if ":" not in clean_path:
+ return f"C:\\{clean_path}"
+ return clean_path
+
+ def deploy(self, context: MinifiTestContext | None) -> bool:
+ # Cleanup previous temp dir if it exists to prevent leaks
+ if self._temp_dir:
+ self._temp_dir.cleanup()
+ self._temp_dir = tempfile.TemporaryDirectory()
+
+ # 1. Prepare Volumes (Directory objects)
+ for directory in self.dirs:
+ # Mirror directory structure locally in temp
+ # usage of os.sep ensures we use host-native separators for the temp setup
+ rel_path = directory.path.strip("/\\")
+ temp_subdir = os.path.join(self._temp_dir.name, rel_path)
+ os.makedirs(temp_subdir, exist_ok=True)
+
+ for file_name, content in directory.files.items():
+ file_path = os.path.join(temp_subdir, file_name)
+ # Write with UTF-8 to ensure content is preserved on host
+ with open(file_path, "w", encoding="utf-8") as temp_file:
+ logging.info(f"writing content into {temp_file.name}")
+ temp_file.write(content)
+
+ # Mount using Windows path format
+ container_bind_path = self._normalize_path(directory.path)
+ self.volumes[temp_subdir] = {
+ "bind": container_bind_path,
+ "mode": directory.mode
+ }
+
+ for host_file in self.host_files:
+ container_bind_path = self._normalize_path(host_file.container_path)
+ self.volumes[container_bind_path] = {"bind": host_file.host_path, "mode": host_file.mode}
+
+ # 3. Cleanup existing container
+ try:
+ existing_container = self.client.containers.get(self.container_name)
+ logging.warning(f"Found existing container '{self.container_name}'. Removing it first.")
+ existing_container.remove(force=True)
+ except docker.errors.NotFound:
+ pass
+
+ # 4. Create and Start
+ try:
+ print(f"Creating and starting container '{self.container_name}'...")
+ self.container = self.client.containers.create(
+ image=self.image_name,
+ name=self.container_name,
+ ports=self.ports,
+ environment=self.environment,
+ volumes=self.volumes,
+ network=self.network.name,
+ command=self.command,
+ entrypoint=self.entrypoint,
+ detach=True,
+ tty=False
+ )
+
+ self.container.start()
+
+ for file in self.files:
+ self._copy_content_to_container(file.content, file.path)
+
+ except Exception as e:
+ logging.error(f"Error starting container: {e}")
+ self.clean_up()
+ raise
+ return True
+
+ def _copy_content_to_container(self, content: str, target_path: str):
+ if not self.container:
+ return
+
+ win_path = self._normalize_path(target_path)
+ dir_name = os.path.dirname(win_path)
+ file_name = os.path.basename(win_path)
+
+ self._run_powershell(f"New-Item -ItemType Directory -Force -Path '{dir_name}'")
+
+ tar_stream = io.BytesIO()
+ with tarfile.open(fileobj=tar_stream, mode='w') as tar:
+ encoded_data = content.encode('utf-8')
+ tarinfo = tarfile.TarInfo(name=file_name)
+ tarinfo.size = len(encoded_data)
+ tar.addfile(tarinfo, io.BytesIO(encoded_data))
+
+ tar_stream.seek(0)
+
+ self.container.put_archive(path=dir_name, data=tar_stream)
+
+ def clean_up(self):
+ if self._temp_dir:
+ try:
+ self._temp_dir.cleanup()
+ self._temp_dir = None
+ except Exception as e:
+ logging.warning(f"Failed to cleanup temp dir: {e}")
+
+ if self.container:
+ try:
+ self.container.remove(force=True)
+ except docker.errors.NotFound:
+ pass
+ except Exception as e:
+ logging.warning(f"Failed to remove container: {e}")
+ finally:
+ self.container = None
+
+ def exec_run(self, command: Union[str, list]) -> Tuple[int | None, str]:
+ logging.debug(f"Running command: {command}")
+ if self.container:
+ # Passing a list (if provided) bypasses shell parsing
+ (code, output) = self.container.exec_run(command, detach=False)
+ # errors='replace' handles non-utf8 characters often found in Windows logs
+ decoded_output = output.decode("utf-8", errors='replace')
+ logging.debug(f"Result {code}, output: {decoded_output}")
+ return code, decoded_output
+ return None, "Container not running."
+
+ def _run_powershell(self, ps_script: str) -> Tuple[int | None, str]:
+ if not self.container:
+ return None, "Container not running"
+
+ encoded_command = base64.b64encode(ps_script.encode('utf_16_le')).decode('utf-8')
+
+ cmd_parts = ["powershell", "-NonInteractive", "-NoProfile", "-EncodedCommand", encoded_command]
+
+ return self.exec_run(cmd_parts)
+
+ def not_empty_dir_exists(self, directory_path: str) -> bool:
+ if not self.container:
+ return False
+
+ win_path = self._normalize_path(directory_path)
+ ps_script = (
+ f"if (Test-Path -Path '{win_path}' -PathType Container) {{ "
+ f" if (Get-ChildItem -Path '{win_path}') {{ exit 0 }} else {{ exit 1 }} "
+ f"}} else {{ exit 2 }}"
+ )
+
+ exit_code, _ = self._run_powershell(ps_script)
+ return exit_code == 0
+
+ def directory_contains_file_with_content(self, directory_path: str, expected_content: str) -> bool:
+ if not self.container:
+ return False
+
+ win_path = self._normalize_path(directory_path)
+ escaped_content = expected_content.replace("'", "''")
+
+ ps_script = (
+ f"if (-not (Test-Path '{win_path}')) {{ exit 2 }}; "
+ f"$matches = Get-ChildItem -Path '{win_path}' -File -Depth 0 | "
+ f"Select-String -Pattern '{escaped_content}' -SimpleMatch -List; "
+ f"if ($matches) {{ exit 0 }} else {{ exit 1 }}"
+ )
+
+ exit_code, _ = self._run_powershell(ps_script)
+ return exit_code == 0
+
+ def directory_contains_file_with_regex(self, directory_path: str, regex_str: str) -> bool:
+ if not self.container:
+ return False
+
+ win_path = self._normalize_path(directory_path)
+ escaped_regex = regex_str.replace("'", "''")
+
+ ps_script = (
+ f"if (-not (Test-Path '{win_path}')) {{ exit 2 }}; "
+ f"$matches = Get-ChildItem -Path '{win_path}' -File -Depth 0 | "
+ f"Select-String -Pattern '{escaped_regex}' -List; "
+ f"if ($matches) {{ exit 0 }} else {{ exit 1 }}"
+ )
+
+ exit_code, _ = self._run_powershell(ps_script)
+ return exit_code == 0
+
+ def path_with_content_exists(self, path: str, content: str) -> bool:
+ if not self.container:
+ return False
+
+ win_path = self._normalize_path(path)
+ # Handle CRLF and escape quotes
+ escaped_content = content.replace("'", "''").replace("\n", "\r\n")
+
+ ps_script = (
+ f"try {{ "
+ f" $found = Select-String -Path '{win_path}' -Pattern '^{escaped_content}$'; "
+ f" if ($found.Count -eq 1) {{ exit 0 }} else {{ exit 1 }} "
+ f"}} catch {{ exit 2 }}"
+ )
+
+ exit_code, output = self._run_powershell(ps_script)
+ if exit_code != 0:
+ logging.debug(f"path_with_content_exists failed for {win_path}. Output: {output}")
+
+ return exit_code == 0
+
+ def directory_has_single_file_with_content(self, directory_path: str, expected_content: str) -> bool:
+ if not self.container:
+ return False
+
+ win_path = self._normalize_path(directory_path)
+ escaped_content = expected_content.strip().replace("'", "''").replace("\n", "\r\n")
+
+ ps_script = (
+ f"$files = Get-ChildItem -Path '{win_path}' -File -Depth 0; "
+ f"if ($files.Count -ne 1) {{ exit 1 }}; "
+ f"$actual_content = Get-Content -Path $files[0].FullName -Raw; "
+ f"if ($actual_content.Trim() -eq '{escaped_content}') {{ exit 0 }} else {{ exit 2 }};"
+ )
+
+ exit_code, output = self._run_powershell(ps_script)
+ if exit_code != 0:
+ logging.debug(f"Check for single file failed in {win_path}. Output: {output}")
+
+ return exit_code == 0
+
+ def get_logs(self) -> str:
+ logging.debug("Getting logs from container '%s'", self.container_name)
+ if not self.container:
+ return ""
+ logs_as_bytes = self.container.logs()
+ # Windows logs may contain mixed encodings; replace errors to avoid crash
+ return logs_as_bytes.decode('utf-8', errors='replace')
+
+ def log_app_output(self) -> bool:
+ logs = self.get_logs()
+ logging.info("Logs of container '%s':", self.container_name)
+ for line in logs.splitlines():
+ logging.info(line)
+ return False
+
+ @property
+ def exited(self) -> bool:
+ if not self.container:
+ return False
+ try:
+ self.container.reload()
+ return self.container.status == 'exited'
+ except docker.errors.NotFound:
+ self.container = None
+ return False
+ except Exception:
+ return False
+
+ def get_number_of_files(self, directory_path: str) -> int:
+ if not self.container:
+ return -1
+
+ win_path = self._normalize_path(directory_path)
+ ps_script = f"(Get-ChildItem -Path '{win_path}' -File -Depth 0).Count"
+
+ exit_code, output = self._run_powershell(ps_script)
+
+ if exit_code != 0:
+ logging.error(f"Error counting files in '{win_path}': {output}")
+ return -1
+
+ try:
+ return int(output.strip())
+ except (ValueError, IndexError):
+ return -1
+
+ def verify_file_contents(self, directory_path: str, expected_contents: List[str]) -> bool:
+ if not self.container:
+ return False
+
+ win_path = self._normalize_path(directory_path)
+
+ # 1. Get list of files
+ ps_list = f"Get-ChildItem -Path '{win_path}' -File -Depth 0 | Select-Object -ExpandProperty FullName"
+ exit_code, output = self._run_powershell(ps_list)
+
+ if exit_code != 0:
+ logging.error(f"Error listing files in '{win_path}': {output}")
+ return False
+
+ actual_filepaths = [path.strip() for path in output.splitlines() if path.strip()]
+
+ if len(actual_filepaths) != len(expected_contents):
+ return False
+
+ actual_file_contents = []
+ for path in actual_filepaths:
+ # Read, normalize CRLF to LF, and trim
+ ps_read = (
+ f"$c = Get-Content -Path '{path}' -Raw; "
+ f"if ($c) {{ ($c -replace '\\r\\n', '\\n' -replace '\\r', '\\n').Trim() }}"
+ )
+
+ exit_code, content = self._run_powershell(ps_read)
+ if exit_code != 0:
+ return False
+ actual_file_contents.append(content)
+
+ # Normalize expected contents
+ normalized_expected = [
+ s.strip().replace("\r\n", "\n").replace("\r", "\n") for s in expected_contents
+ ]
+
+ return sorted(actual_file_contents) == sorted(normalized_expected)
diff --git a/behave_framework/src/minifi_test_framework/containers/http_proxy_container.py b/behave_framework/src/minifi_test_framework/containers/http_proxy_container.py
index 49ba2e766..c4b4d8823 100644
--- a/behave_framework/src/minifi_test_framework/containers/http_proxy_container.py
+++ b/behave_framework/src/minifi_test_framework/containers/http_proxy_container.py
@@ -17,13 +17,13 @@
from textwrap import dedent
-from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.container_linux import LinuxContainer
from minifi_test_framework.containers.docker_image_builder import DockerImageBuilder
from minifi_test_framework.core.helpers import wait_for_condition
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
-class HttpProxy(Container):
+class HttpProxy(LinuxContainer):
def __init__(self, test_context: MinifiTestContext):
dockerfile = dedent("""\
FROM {base_image}
diff --git a/behave_framework/src/minifi_test_framework/containers/minifi_container.py b/behave_framework/src/minifi_test_framework/containers/minifi_container.py
index 75415017f..32728321f 100644
--- a/behave_framework/src/minifi_test_framework/containers/minifi_container.py
+++ b/behave_framework/src/minifi_test_framework/containers/minifi_container.py
@@ -18,13 +18,12 @@
import logging
import os
from pathlib import Path
-from OpenSSL import crypto
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
from minifi_test_framework.containers.file import File
from minifi_test_framework.containers.host_file import HostFile
from minifi_test_framework.minifi.minifi_flow_definition import MinifiFlowDefinition
-from minifi_test_framework.core.ssl_utils import make_cert_without_extended_usage, make_client_cert, make_server_cert
+from minifi_test_framework.core.ssl_utils import make_cert_without_extended_usage, make_client_cert, make_server_cert, dump_cert, dump_key
from minifi_test_framework.core.helpers import wait_for_condition, retry_check
from .container import Container
@@ -37,18 +36,18 @@ class MinifiContainer(Container):
self.log_properties: dict[str, str] = {}
minifi_client_cert, minifi_client_key = make_cert_without_extended_usage(common_name=self.container_name, ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key)
- self.files.append(File("/usr/local/share/certs/ca-root-nss.crt", crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert)))
- self.files.append(File("/tmp/resources/root_ca.crt", crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert)))
- self.files.append(File("/tmp/resources/minifi_client.crt", crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=minifi_client_cert)))
- self.files.append(File("/tmp/resources/minifi_client.key", crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=minifi_client_key)))
+ self.files.append(File("/usr/local/share/certs/ca-root-nss.crt", dump_cert(test_context.root_ca_cert)))
+ self.files.append(File("/tmp/resources/root_ca.crt", dump_cert(test_context.root_ca_cert)))
+ self.files.append(File("/tmp/resources/minifi_client.crt", dump_cert(minifi_client_cert)))
+ self.files.append(File("/tmp/resources/minifi_client.key", dump_key(minifi_client_key)))
clientuser_cert, clientuser_key = make_client_cert("clientuser", ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key)
- self.files.append(File("/tmp/resources/clientuser.crt", crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=clientuser_cert)))
- self.files.append(File("/tmp/resources/clientuser.key", crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=clientuser_key)))
+ self.files.append(File("/tmp/resources/clientuser.crt", dump_cert(clientuser_cert)))
+ self.files.append(File("/tmp/resources/clientuser.key", dump_key(clientuser_key)))
minifi_server_cert, minifi_server_key = make_server_cert(common_name=f"server-{test_context.scenario_id}", ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key)
self.files.append(File("/tmp/resources/minifi_server.crt",
- crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=minifi_server_cert) + crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=minifi_server_key)))
+ dump_cert(minifi_server_cert) + dump_key(minifi_server_key)))
self.is_fhs = 'MINIFI_INSTALLATION_TYPE=FHS' in str(self.client.images.get(test_context.minifi_container_image).history())
if self.is_fhs:
diff --git a/behave_framework/src/minifi_test_framework/containers/minifi_controller.py b/behave_framework/src/minifi_test_framework/containers/minifi_controller.py
new file mode 100644
index 000000000..0dc5b8330
--- /dev/null
+++ b/behave_framework/src/minifi_test_framework/containers/minifi_controller.py
@@ -0,0 +1,79 @@
+import logging
+from minifi_test_framework.core.helpers import retry_check
+
+
+class MinifiController:
+ def __init__(self, minifi_container, config_path):
+ self.minifi_container = minifi_container
+ self.config_path = config_path
+
+ def set_controller_socket_properties(self):
+ self.minifi_container.properties["controller.socket.enable"] = "true"
+ self.minifi_container.properties["controller.socket.host"] = "localhost"
+ self.minifi_container.properties["controller.socket.port"] = "9998"
+ self.minifi_container.properties["controller.socket.local.any.interface"] = "false"
+
+ def update_flow_config_through_controller(self):
+ self.minifi_container.exec_run([self.minifi_container.minifi_controller_path, "--updateflow", "/tmp/resources/minifi-controller/config.yml"])
+
+ def updated_config_is_persisted(self) -> bool:
+ exit_code, output = self.minifi_container.exec_run(["cat", self.config_path])
+ if exit_code != 0:
+ logging.error("Failed to read MiNiFi config file to check if updated config is persisted")
+ return False
+ return "2f2a3b47-f5ba-49f6-82b5-bc1c86b96f38" in output
+
+ def stop_component_through_controller(self, component: str):
+ self.minifi_container.exec_run([self.minifi_container.minifi_controller_path, "--stop", component])
+
+ def start_component_through_controller(self, component: str):
+ self.minifi_container.exec_run([self.minifi_container.minifi_controller_path, "--start", component])
+
+ @retry_check(10, 1)
+ def is_component_running(self, component: str) -> bool:
+ (code, output) = self.minifi_container.exec_run([self.minifi_container.minifi_controller_path, "--list", "components"])
+ return code == 0 and component + ", running: true" in output
+
+ def get_connections(self):
+ (_, output) = self.minifi_container.exec_run([self.minifi_container.minifi_controller_path, "--list", "connections"])
+ connections = []
+ for line in output.split('\n'):
+ if not line.startswith('[') and not line.startswith('Connection Names'):
+ connections.append(line)
+ return connections
+
+ @retry_check(10, 1)
+ def connection_found_through_controller(self, connection: str) -> bool:
+ return connection in self.get_connections()
+
+ def get_full_connection_count(self) -> int:
+ (_, output) = self.minifi_container.exec_run([self.minifi_container.minifi_controller_path, "--getfull"])
+ for line in output.split('\n'):
+ if "are full" in line:
+ return int(line.split(' ')[0])
+ return -1
+
+ def get_connection_size(self, connection: str):
+ (_, output) = self.minifi_container.exec_run([self.minifi_container.minifi_controller_path, "--getsize", connection])
+ for line in output.split('\n'):
+ if "Size/Max of " + connection in line:
+ size_and_max = line.split(connection)[1].split('/')
+ return (int(size_and_max[0].strip()), int(size_and_max[1].strip()))
+ return (-1, -1)
+
+ def get_manifest(self) -> str:
+ (_, output) = self.minifi_container.exec_run([self.minifi_container.minifi_controller_path, "--manifest"])
+ manifest = ""
+ for line in output.split('\n'):
+ if not line.startswith('['):
+ manifest += line
+ return manifest
+
+ def create_debug_bundle(self) -> bool:
+ (code, _) = self.minifi_container.exec_run([self.minifi_container.minifi_controller_path, "--debug", "/tmp"])
+ if code != 0:
+ logging.error("Minifi controller debug command failed with code: %d", code)
+ return False
+
+ (code, _) = self.minifi_container.exec_run(["test", "-f", "/tmp/debug.tar.gz"])
+ return code == 0
diff --git a/behave_framework/src/minifi_test_framework/containers/minifi_fhs_container.py b/behave_framework/src/minifi_test_framework/containers/minifi_fhs_container.py
new file mode 100644
index 000000000..49c5236f0
--- /dev/null
+++ b/behave_framework/src/minifi_test_framework/containers/minifi_fhs_container.py
@@ -0,0 +1,119 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import logging
+import os
+from pathlib import Path
+
+from .container_linux import LinuxContainer
+
+from minifi_test_framework.core.minifi_test_context import MinifiTestContext
+from minifi_test_framework.containers.file import File
+from minifi_test_framework.containers.host_file import HostFile
+from minifi_test_framework.minifi.minifi_flow_definition import MinifiFlowDefinition
+from minifi_test_framework.core.ssl_utils import make_cert_without_extended_usage, make_client_cert, make_server_cert, dump_cert, dump_key
+from minifi_test_framework.core.helpers import wait_for_condition
+
+from .minifi_protocol import MinifiProtocol
+from .minifi_controller import MinifiController
+
+
+class MinifiFhsContainer(LinuxContainer, MinifiProtocol):
+ def __init__(self, container_name: str, test_context: MinifiTestContext):
+ super().__init__(test_context.minifi_container_image, f"{container_name}-{test_context.scenario_id}", test_context.network)
+ self.flow_definition = MinifiFlowDefinition()
+ self.properties: dict[str, str] = {}
+ self.log_properties: dict[str, str] = {}
+ self.controller = MinifiController(self, "/etc/nifi-minifi-cpp/config.yml")
+
+ minifi_client_cert, minifi_client_key = make_cert_without_extended_usage(common_name=self.container_name, ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key)
+ self.files.append(File("/usr/local/share/certs/ca-root-nss.crt", dump_cert(test_context.root_ca_cert)))
+ self.files.append(File("/tmp/resources/root_ca.crt", dump_cert(test_context.root_ca_cert)))
+ self.files.append(File("/tmp/resources/minifi_client.crt", dump_cert(minifi_client_cert)))
+ self.files.append(File("/tmp/resources/minifi_client.key", dump_key(minifi_client_key)))
+
+ clientuser_cert, clientuser_key = make_client_cert("clientuser", ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key)
+ self.files.append(File("/tmp/resources/clientuser.crt", dump_cert(clientuser_cert)))
+ self.files.append(File("/tmp/resources/clientuser.key", dump_key(clientuser_key)))
+
+ minifi_server_cert, minifi_server_key = make_server_cert(common_name=f"server-{test_context.scenario_id}", ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key)
+ self.files.append(File("/tmp/resources/minifi_server.crt",
+ dump_cert(minifi_server_cert) + dump_key(minifi_server_key)))
+
+ self.minifi_controller_path = '/usr/bin/minifi-controller'
+
+ self._fill_default_properties()
+ self._fill_default_log_properties()
+
+ def deploy(self, context: MinifiTestContext | None) -> bool:
+ flow_config = self.flow_definition.to_yaml()
+ logging.info(f"Deploying MiNiFi container '{self.container_name}' with flow configuration:\n{flow_config}")
+ self.files.append(File("/etc/nifi-minifi-cpp/config.yml", flow_config))
+ self.files.append(File("/etc/nifi-minifi-cpp/minifi.properties", self._get_properties_file_content()))
+ self.files.append(File("/etc/nifi-minifi-cpp/minifi-log.properties", self._get_log_properties_file_content()))
+ resource_dir = Path(__file__).resolve().parent / "resources" / "minifi-controller"
+ self.host_files.append(HostFile("/tmp/resources/minifi-controller/config.yml", os.path.join(resource_dir, "config.yml")))
+
+ if not super().deploy(context):
+ return False
+
+ finished_str = "MiNiFi started"
+ return wait_for_condition(
+ condition=lambda: finished_str in self.get_logs(),
+ timeout_seconds=15,
+ bail_condition=lambda: self.exited,
+ context=context)
+
+ def set_property(self, key: str, value: str):
+ self.properties[key] = value
+
+ def set_log_property(self, key: str, value: str):
+ self.log_properties[key] = value
+
+ def enable_openssl_fips_mode(self):
+ self.properties["nifi.openssl.fips.support.enable"] = "true"
+
+ def _fill_default_properties(self):
+ self.properties["nifi.flow.configuration.file"] = "/etc/nifi-minifi-cpp/config.yml"
+ self.properties["nifi.extension.path"] = "/usr/lib64/nifi-minifi-cpp/extensions/*"
+ self.properties["nifi.administrative.yield.duration"] = "1 sec"
+ self.properties["nifi.bored.yield.duration"] = "100 millis"
+ self.properties["nifi.openssl.fips.support.enable"] = "false"
+ self.properties["nifi.provenance.repository.class.name"] = "NoOpRepository"
+
+ def _fill_default_log_properties(self):
+ self.log_properties["spdlog.pattern"] = "[%Y-%m-%d %H:%M:%S.%e] [%n] [%l] %v"
+
+ self.log_properties["appender.stderr"] = "stderr"
+ self.log_properties["logger.root"] = "DEBUG, stderr"
+ self.log_properties["logger.org::apache::nifi::minifi"] = "DEBUG, stderr"
+
+ def _get_properties_file_content(self):
+ lines = (f"{key}={value}" for key, value in self.properties.items())
+ return "\n".join(lines)
+
+ def _get_log_properties_file_content(self):
+ lines = (f"{key}={value}" for key, value in self.log_properties.items())
+ return "\n".join(lines)
+
+ def get_memory_usage(self) -> int | None:
+ exit_code, output = self.exec_run(["awk", "/VmRSS/ { printf \"%d\\n\", $2 }", "/proc/1/status"])
+ if exit_code != 0:
+ return None
+ memory_usage_in_bytes = int(output.strip()) * 1024
+ logging.info(f"MiNiFi memory usage: {memory_usage_in_bytes} bytes")
+ return memory_usage_in_bytes
diff --git a/behave_framework/src/minifi_test_framework/containers/minifi_linux_container.py b/behave_framework/src/minifi_test_framework/containers/minifi_linux_container.py
new file mode 100644
index 000000000..f0279ebe2
--- /dev/null
+++ b/behave_framework/src/minifi_test_framework/containers/minifi_linux_container.py
@@ -0,0 +1,119 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import logging
+import os
+from pathlib import Path
+
+from .container_linux import LinuxContainer
+
+from minifi_test_framework.core.minifi_test_context import MinifiTestContext
+from minifi_test_framework.containers.file import File
+from minifi_test_framework.containers.host_file import HostFile
+from minifi_test_framework.minifi.minifi_flow_definition import MinifiFlowDefinition
+from minifi_test_framework.core.ssl_utils import make_cert_without_extended_usage, make_client_cert, make_server_cert, dump_cert, dump_key
+from minifi_test_framework.core.helpers import wait_for_condition
+
+from .minifi_protocol import MinifiProtocol
+from .minifi_controller import MinifiController
+
+
+class MinifiLinuxContainer(LinuxContainer, MinifiProtocol):
+ def __init__(self, container_name: str, test_context: MinifiTestContext):
+ super().__init__(test_context.minifi_container_image, f"{container_name}-{test_context.scenario_id}", test_context.network)
+ self.flow_definition = MinifiFlowDefinition()
+ self.properties: dict[str, str] = {}
+ self.log_properties: dict[str, str] = {}
+ self.controller = MinifiController(self, "/opt/minifi/minifi-current/conf/config.yml")
+
+ minifi_client_cert, minifi_client_key = make_cert_without_extended_usage(common_name=self.container_name, ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key)
+ self.files.append(File("/usr/local/share/certs/ca-root-nss.crt", dump_cert(test_context.root_ca_cert)))
+ self.files.append(File("/tmp/resources/root_ca.crt", dump_cert(test_context.root_ca_cert)))
+ self.files.append(File("/tmp/resources/minifi_client.crt", dump_cert(minifi_client_cert)))
+ self.files.append(File("/tmp/resources/minifi_client.key", dump_key(minifi_client_key)))
+
+ clientuser_cert, clientuser_key = make_client_cert("clientuser", ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key)
+ self.files.append(File("/tmp/resources/clientuser.crt", dump_cert(clientuser_cert)))
+ self.files.append(File("/tmp/resources/clientuser.key", dump_key(clientuser_key)))
+
+ minifi_server_cert, minifi_server_key = make_server_cert(common_name=f"server-{test_context.scenario_id}", ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key)
+ self.files.append(File("/tmp/resources/minifi_server.crt",
+ dump_cert(cert=minifi_server_cert) + dump_key(minifi_server_key)))
+
+ self.minifi_controller_path = '/opt/minifi/minifi-current/bin/minifi-controller'
+
+ self._fill_default_properties()
+ self._fill_default_log_properties()
+
+ def deploy(self, context: MinifiTestContext | None) -> bool:
+ flow_config = self.flow_definition.to_yaml()
+ logging.info(f"Deploying MiNiFi container '{self.container_name}' with flow configuration:\n{flow_config}")
+ self.files.append(File("/opt/minifi/minifi-current/conf/config.yml", flow_config))
+ self.files.append(File("/opt/minifi/minifi-current/conf/minifi.properties", self._get_properties_file_content()))
+ self.files.append(File("/opt/minifi/minifi-current/conf/minifi-log.properties", self._get_log_properties_file_content()))
+ resource_dir = Path(__file__).resolve().parent / "resources" / "minifi-controller"
+ self.host_files.append(HostFile("/tmp/resources/minifi-controller/config.yml", os.path.join(resource_dir, "config.yml")))
+
+ if not super().deploy(context):
+ return False
+
+ finished_str = "MiNiFi started"
+ return wait_for_condition(
+ condition=lambda: finished_str in self.get_logs(),
+ timeout_seconds=15,
+ bail_condition=lambda: self.exited,
+ context=context)
+
+ def set_property(self, key: str, value: str):
+ self.properties[key] = value
+
+ def set_log_property(self, key: str, value: str):
+ self.log_properties[key] = value
+
+ def enable_openssl_fips_mode(self):
+ self.properties["nifi.openssl.fips.support.enable"] = "true"
+
+ def _fill_default_properties(self):
+ self.properties["nifi.flow.configuration.file"] = "./conf/config.yml"
+ self.properties["nifi.extension.path"] = "../extensions/*"
+ self.properties["nifi.administrative.yield.duration"] = "1 sec"
+ self.properties["nifi.bored.yield.duration"] = "100 millis"
+ self.properties["nifi.openssl.fips.support.enable"] = "false"
+ self.properties["nifi.provenance.repository.class.name"] = "NoOpRepository"
+
+ def _fill_default_log_properties(self):
+ self.log_properties["spdlog.pattern"] = "[%Y-%m-%d %H:%M:%S.%e] [%n] [%l] %v"
+
+ self.log_properties["appender.stderr"] = "stderr"
+ self.log_properties["logger.root"] = "DEBUG, stderr"
+ self.log_properties["logger.org::apache::nifi::minifi"] = "DEBUG, stderr"
+
+ def _get_properties_file_content(self):
+ lines = (f"{key}={value}" for key, value in self.properties.items())
+ return "\n".join(lines)
+
+ def _get_log_properties_file_content(self):
+ lines = (f"{key}={value}" for key, value in self.log_properties.items())
+ return "\n".join(lines)
+
+ def get_memory_usage(self) -> int | None:
+ exit_code, output = self.exec_run(["awk", "/VmRSS/ { printf \"%d\\n\", $2 }", "/proc/1/status"])
+ if exit_code != 0:
+ return None
+ memory_usage_in_bytes = int(output.strip()) * 1024
+ logging.info(f"MiNiFi memory usage: {memory_usage_in_bytes} bytes")
+ return memory_usage_in_bytes
diff --git a/behave_framework/src/minifi_test_framework/containers/minifi_protocol.py b/behave_framework/src/minifi_test_framework/containers/minifi_protocol.py
new file mode 100644
index 000000000..1c6a110d8
--- /dev/null
+++ b/behave_framework/src/minifi_test_framework/containers/minifi_protocol.py
@@ -0,0 +1,9 @@
+from typing import Protocol
+
+
+class MinifiProtocol(Protocol):
+ def set_property(self, key: str, value: str):
+ ...
+
+ def set_log_property(self, key: str, value: str):
+ ...
diff --git a/behave_framework/src/minifi_test_framework/containers/minifi_win_container.py b/behave_framework/src/minifi_test_framework/containers/minifi_win_container.py
new file mode 100644
index 000000000..2e8952cf3
--- /dev/null
+++ b/behave_framework/src/minifi_test_framework/containers/minifi_win_container.py
@@ -0,0 +1,59 @@
+from typing import Dict
+
+from minifi_test_framework.core.minifi_test_context import MinifiTestContext
+from minifi_test_framework.minifi.minifi_flow_definition import MinifiFlowDefinition
+from minifi_test_framework.containers.directory import Directory
+from .container_windows import WindowsContainer
+from .minifi_protocol import MinifiProtocol
+import logging
+
+
+class MinifiWindowsContainer(WindowsContainer, MinifiProtocol):
+ def __init__(self, container_name: str, test_context: MinifiTestContext):
+ super().__init__(test_context.minifi_container_image, f"{container_name}-{test_context.scenario_id}", test_context.network)
+ self.flow_config_str: str = ""
+ self.flow_definition = MinifiFlowDefinition()
+ self.properties: Dict[str, str] = {}
+ self.log_properties: Dict[str, str] = {}
+
+ self._fill_default_properties()
+ self._fill_default_log_properties()
+
+ def deploy(self, context: MinifiTestContext | None) -> bool:
+ logging.info(self.flow_definition.to_yaml())
+ conf_dir = Directory("\\Program Files\\ApacheNiFiMiNiFi\\nifi-minifi-cpp\\conf")
+ conf_dir.add_file("config.yml", self.flow_definition.to_yaml())
+ conf_dir.add_file("minifi.properties", self._get_properties_file_content())
+ conf_dir.add_file("minifi-log.properties", self._get_log_properties_file_content())
+
+ self.dirs.append(conf_dir)
+ return super().deploy(context)
+
+ def set_property(self, key: str, value: str):
+ self.properties[key] = value
+
+ def set_log_property(self, key: str, value: str):
+ self.log_properties[key] = value
+
+ def _fill_default_properties(self):
+ self.properties["nifi.flow.configuration.file"] = "./conf/config.yml"
+ self.properties["nifi.extension.path"] = "../extensions/*"
+ self.properties["nifi.administrative.yield.duration"] = "1 sec"
+ self.properties["nifi.bored.yield.duration"] = "100 millis"
+ self.properties["nifi.openssl.fips.support.enable"] = "false"
+ self.properties["nifi.provenance.repository.class.name"] = "NoOpRepository"
+
+ def _fill_default_log_properties(self):
+ self.log_properties["spdlog.pattern"] = "[%Y-%m-%d %H:%M:%S.%e] [%n] [%l] %v"
+
+ self.log_properties["appender.stderr"] = "stderr"
+ self.log_properties["logger.root"] = "DEBUG, stderr"
+ self.log_properties["logger.org::apache::nifi::minifi"] = "DEBUG, stderr"
+
+ def _get_properties_file_content(self):
+ lines = (f"{key}={value}" for key, value in self.properties.items())
+ return "\n".join(lines)
+
+ def _get_log_properties_file_content(self):
+ lines = (f"{key}={value}" for key, value in self.log_properties.items())
+ return "\n".join(lines)
diff --git a/behave_framework/src/minifi_test_framework/containers/nifi_container.py b/behave_framework/src/minifi_test_framework/containers/nifi_container.py
index d40724de1..140b5d508 100644
--- a/behave_framework/src/minifi_test_framework/containers/nifi_container.py
+++ b/behave_framework/src/minifi_test_framework/containers/nifi_container.py
@@ -17,13 +17,13 @@ import io
import gzip
from typing import List, Optional
from minifi_test_framework.containers.file import File
-from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.container_linux import LinuxContainer
from minifi_test_framework.core.helpers import wait_for_condition
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
from minifi_test_framework.minifi.nifi_flow_definition import NifiFlowDefinition
-class NifiContainer(Container):
+class NifiContainer(LinuxContainer):
NIFI_VERSION = '2.7.2'
def __init__(self, test_context: MinifiTestContext, command: Optional[List[str]] = None, use_ssl: bool = False):
diff --git a/behave_framework/src/minifi_test_framework/core/hooks.py b/behave_framework/src/minifi_test_framework/core/hooks.py
index 298f27d85..576ac40f8 100644
--- a/behave_framework/src/minifi_test_framework/core/hooks.py
+++ b/behave_framework/src/minifi_test_framework/core/hooks.py
@@ -46,6 +46,10 @@ def inject_scenario_id(context: MinifiTestContext, step):
def common_before_scenario(context: Context, scenario: Scenario):
+ if "SUPPORTS_WINDOWS" not in scenario.effective_tags and os.name == 'nt':
+ scenario.skip("No windows support")
+ return
+
if not hasattr(context, "minifi_container_image"):
context.minifi_container_image = get_minifi_container_image()
@@ -80,6 +84,8 @@ def common_before_scenario(context: Context, scenario: Scenario):
def common_after_scenario(context: MinifiTestContext, scenario: Scenario):
- for container in context.containers.values():
- container.clean_up()
- context.network.remove()
+ if hasattr(context, 'containers'):
+ for container in context.containers.values():
+ container.clean_up()
+ if hasattr(context, 'network'):
+ context.network.remove()
diff --git a/behave_framework/src/minifi_test_framework/core/minifi_test_context.py b/behave_framework/src/minifi_test_framework/core/minifi_test_context.py
index 959f2e1fa..5dd4db9cc 100644
--- a/behave_framework/src/minifi_test_framework/core/minifi_test_context.py
+++ b/behave_framework/src/minifi_test_framework/core/minifi_test_context.py
@@ -16,32 +16,45 @@
#
from __future__ import annotations
-from typing import TYPE_CHECKING
+
+import os
+from typing import Any
+import docker
from behave.runner import Context
from docker.models.networks import Network
-from minifi_test_framework.containers.container import Container
-from OpenSSL import crypto
+from minifi_test_framework.containers.container_protocol import ContainerProtocol
+from minifi_test_framework.containers.minifi_protocol import MinifiProtocol
-if TYPE_CHECKING:
- from minifi_test_framework.containers.minifi_container import MinifiContainer
DEFAULT_MINIFI_CONTAINER_NAME = "minifi-primary"
+class MinifiContainer(ContainerProtocol, MinifiProtocol):
+ pass
+
+
class MinifiTestContext(Context):
- containers: dict[str, Container]
+ containers: dict[str, ContainerProtocol]
scenario_id: str
network: Network
minifi_container_image: str
resource_dir: str | None
- root_ca_key: crypto.PKey
- root_ca_cert: crypto.X509
+ root_ca_key: Any
+ root_ca_cert: Any
def get_or_create_minifi_container(self, container_name: str) -> MinifiContainer:
if container_name not in self.containers:
- from minifi_test_framework.containers.minifi_container import MinifiContainer
- self.containers[container_name] = MinifiContainer(container_name, self)
+ if os.name == 'nt':
+ from minifi_test_framework.containers.minifi_win_container import MinifiWindowsContainer
+ minifi_container = MinifiWindowsContainer(container_name, self)
+ elif 'MINIFI_INSTALLATION_TYPE=FHS' in str(docker.from_env().images.get(self.minifi_container_image).history()):
+ from minifi_test_framework.containers.minifi_fhs_container import MinifiFhsContainer
+ minifi_container = MinifiFhsContainer(container_name, self)
+ else:
+ from minifi_test_framework.containers.minifi_linux_container import MinifiLinuxContainer
+ minifi_container = MinifiLinuxContainer(container_name, self)
+ self.containers[container_name] = minifi_container
return self.containers[container_name]
def get_or_create_default_minifi_container(self) -> MinifiContainer:
diff --git a/behave_framework/src/minifi_test_framework/core/ssl_utils.py b/behave_framework/src/minifi_test_framework/core/ssl_utils.py
index d39cf9646..b11da3443 100644
--- a/behave_framework/src/minifi_test_framework/core/ssl_utils.py
+++ b/behave_framework/src/minifi_test_framework/core/ssl_utils.py
@@ -1,161 +1,125 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-import time
-import logging
-import random
-
-from M2Crypto import X509, EVP, RSA, ASN1
-from OpenSSL import crypto
+import datetime
+from cryptography import x509
+from cryptography.x509.oid import NameOID
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.asymmetric import rsa
+from cryptography.hazmat.primitives import serialization
def gen_cert():
- """
- Generate TLS certificate request for testing
- """
-
- req, key = gen_req()
- pub_key = req.get_pubkey()
- subject = req.get_subject()
- cert = X509.X509()
- # noinspection PyTypeChecker
- cert.set_serial_number(1)
- cert.set_version(2)
- cert.set_subject(subject)
- t = int(time.time())
- now = ASN1.ASN1_UTCTIME()
- now.set_time(t)
- now_plus_year = ASN1.ASN1_UTCTIME()
- now_plus_year.set_time(t + 60 * 60 * 24 * 365)
- cert.set_not_before(now)
- cert.set_not_after(now_plus_year)
- issuer = X509.X509_Name()
- issuer.C = 'US'
- issuer.CN = 'minifi-listen'
- cert.set_issuer(issuer)
- cert.set_pubkey(pub_key)
- cert.sign(key, 'sha256')
+ key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
- return cert, key
-
-
-def rsa_gen_key_callback():
- pass
-
-
-def gen_req():
- """
- Generate TLS certificate request for testing
- """
+ subject = issuer = x509.Name([
+ x509.NameAttribute(NameOID.COUNTRY_NAME, u"US"),
+ x509.NameAttribute(NameOID.COMMON_NAME, u"minifi-listen"),
+ ])
- logging.info('Generating test certificate request')
- key = EVP.PKey()
- req = X509.Request()
- rsa = RSA.gen_key(1024, 65537, rsa_gen_key_callback)
- key.assign_rsa(rsa)
- req.set_pubkey(key)
- name = req.get_subject()
- name.C = 'US'
- name.CN = 'minifi-listen'
- req.sign(key, 'sha256')
+ cert = x509.CertificateBuilder().subject_name(
+ subject
+ ).issuer_name(
+ issuer
+ ).public_key(
+ key.public_key()
+ ).serial_number(
+ x509.random_serial_number()
+ ).not_valid_before(
+ datetime.datetime.utcnow()
+ ).not_valid_after(
+ datetime.datetime.utcnow() + datetime.timedelta(days=365)
+ ).sign(key, hashes.SHA256())
- return req, key
+ return cert, key
def make_self_signed_cert(common_name):
- ca_key = crypto.PKey()
- ca_key.generate_key(crypto.TYPE_RSA, 2048)
+ key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
- ca_cert = crypto.X509()
- ca_cert.set_version(2)
- ca_cert.set_serial_number(random.randint(50000000, 100000000))
-
- ca_subj = ca_cert.get_subject()
- ca_subj.commonName = common_name
-
- ca_cert.add_extensions([
- crypto.X509Extension(b"subjectKeyIdentifier", False, b"hash", subject=ca_cert),
- ])
-
- ca_cert.add_extensions([
- crypto.X509Extension(b"authorityKeyIdentifier", False, b"keyid:always", issuer=ca_cert),
+ subject = issuer = x509.Name([
+ x509.NameAttribute(NameOID.COMMON_NAME, common_name),
])
- ca_cert.add_extensions([
- crypto.X509Extension(b"basicConstraints", False, b"CA:TRUE"),
- crypto.X509Extension(b"keyUsage", False, b"keyCertSign, cRLSign"),
- ])
-
- ca_cert.set_issuer(ca_subj)
- ca_cert.set_pubkey(ca_key)
-
- ca_cert.gmtime_adj_notBefore(0)
- ca_cert.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60)
+ cert = x509.CertificateBuilder().subject_name(
+ subject
+ ).issuer_name(
+ issuer
+ ).public_key(
+ key.public_key()
+ ).serial_number(
+ x509.random_serial_number()
+ ).not_valid_before(
+ datetime.datetime.utcnow()
+ ).not_valid_after(
+ datetime.datetime.utcnow() + datetime.timedelta(days=3650)
+ ).add_extension(
+ x509.SubjectKeyIdentifier.from_public_key(key.public_key()),
+ critical=False,
+ ).add_extension(
+ x509.BasicConstraints(ca=True, path_length=None),
+ critical=True,
+ ).sign(key, hashes.SHA256())
- ca_cert.sign(ca_key, 'sha256')
-
- return ca_cert, ca_key
+ return cert, key
def _make_cert(common_name, ca_cert, ca_key, extended_key_usage=None):
- key = crypto.PKey()
- key.generate_key(crypto.TYPE_RSA, 2048)
-
- cert = crypto.X509()
- cert.set_version(2)
- cert.set_serial_number(random.randint(50000000, 100000000))
-
- client_subj = cert.get_subject()
- client_subj.commonName = common_name
+ key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
- cert.add_extensions([
- crypto.X509Extension(b"basicConstraints", False, b"CA:FALSE"),
- crypto.X509Extension(b"subjectKeyIdentifier", False, b"hash", subject=cert),
+ subject = x509.Name([
+ x509.NameAttribute(NameOID.COMMON_NAME, common_name),
])
- extensions = [crypto.X509Extension(b"authorityKeyIdentifier", False, b"keyid:always", issuer=ca_cert),
- crypto.X509Extension(b"keyUsage", False, b"digitalSignature")]
+ builder = x509.CertificateBuilder().subject_name(
+ subject
+ ).issuer_name(
+ ca_cert.subject
+ ).public_key(
+ key.public_key()
+ ).serial_number(
+ x509.random_serial_number()
+ ).not_valid_before(
+ datetime.datetime.utcnow()
+ ).not_valid_after(
+ datetime.datetime.utcnow() + datetime.timedelta(days=3650)
+ ).add_extension(
+ x509.BasicConstraints(ca=False, path_length=None),
+ critical=True,
+ ).add_extension(
+ x509.SubjectKeyIdentifier.from_public_key(key.public_key()),
+ critical=False,
+ ).add_extension(
+ x509.SubjectAlternativeName([x509.DNSName(common_name)]),
+ critical=False,
+ )
if extended_key_usage:
- extensions.append(crypto.X509Extension(b"extendedKeyUsage", False, extended_key_usage))
-
- cert.add_extensions([
- crypto.X509Extension(b"subjectAltName", False, b"DNS.1:" + common_name.encode())
- ])
-
- cert.add_extensions(extensions)
-
- cert.set_issuer(ca_cert.get_subject())
- cert.set_pubkey(key)
-
- cert.gmtime_adj_notBefore(0)
- cert.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60)
-
- cert.sign(ca_key, 'sha256')
+ builder = builder.add_extension(
+ x509.ExtendedKeyUsage(extended_key_usage),
+ critical=False
+ )
+ cert = builder.sign(ca_key, hashes.SHA256())
return cert, key
def make_client_cert(common_name, ca_cert, ca_key):
- return _make_cert(common_name=common_name, ca_cert=ca_cert, ca_key=ca_key, extended_key_usage=b"clientAuth")
+ return _make_cert(common_name, ca_cert, ca_key, [x509.OID_CLIENT_AUTH])
def make_server_cert(common_name, ca_cert, ca_key):
- return _make_cert(common_name=common_name, ca_cert=ca_cert, ca_key=ca_key, extended_key_usage=b"serverAuth")
+ return _make_cert(common_name, ca_cert, ca_key, [x509.OID_SERVER_AUTH])
def make_cert_without_extended_usage(common_name, ca_cert, ca_key):
- return _make_cert(common_name=common_name, ca_cert=ca_cert, ca_key=ca_key, extended_key_usage=None)
+ return _make_cert(common_name, ca_cert, ca_key, None)
+
+
+def dump_cert(cert, encoding_type=serialization.Encoding.PEM):
+ return cert.public_bytes(encoding_type)
+
+
+def dump_key(key, encoding_type=serialization.Encoding.PEM):
+ return key.private_bytes(
+ encoding=encoding_type,
+ format=serialization.PrivateFormat.TraditionalOpenSSL,
+ encryption_algorithm=serialization.NoEncryption()
+ )
diff --git a/docker/installed/win.Dockerfile b/docker/installed/win.Dockerfile
new file mode 100644
index 000000000..b413b68b9
--- /dev/null
+++ b/docker/installed/win.Dockerfile
@@ -0,0 +1,24 @@
+#escape=`
+
+FROM mcr.microsoft.com/windows/servercore:ltsc2022
+
+LABEL maintainer="Apache NiFi <de...@nifi.apache.org>"
+
+ARG MSI_SOURCE="nifi-minifi-cpp.msi"
+
+ENV MINIFI_HOME="C:\Program Files\ApacheNiFiMiNiFi\nifi-minifi-cpp"
+
+SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
+
+RUN Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))
+
+RUN choco install git -y
+RUN choco install vcredist140 -y
+
+COPY ${MSI_SOURCE} C:\temp\minifi.msi
+
+SHELL ["cmd", "/S", "/C"]
+
+RUN C:\Windows\System32\msiexec.exe /i C:\temp\minifi.msi /qn /norestart /L*V C:\minifi_install.log
+
+CMD ["C:\\Program Files\\ApacheNiFiMiNiFi\\nifi-minifi-cpp\\bin\\minifi.exe"]
\ No newline at end of file
diff --git a/extensions/aws/tests/features/steps/kinesis_server_container.py b/extensions/aws/tests/features/steps/kinesis_server_container.py
index 01b6dced2..d97bb5c83 100644
--- a/extensions/aws/tests/features/steps/kinesis_server_container.py
+++ b/extensions/aws/tests/features/steps/kinesis_server_container.py
@@ -16,13 +16,13 @@
import logging
from pathlib import Path
-from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.container_linux import LinuxContainer
from minifi_test_framework.core.helpers import wait_for_condition, retry_check
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
from minifi_test_framework.containers.docker_image_builder import DockerImageBuilder
-class KinesisServerContainer(Container):
+class KinesisServerContainer(LinuxContainer):
def __init__(self, test_context: MinifiTestContext):
builder = DockerImageBuilder(
image_tag="minifi-kinesis-mock:latest",
diff --git a/extensions/aws/tests/features/steps/s3_server_container.py b/extensions/aws/tests/features/steps/s3_server_container.py
index 699617c71..a2c915743 100644
--- a/extensions/aws/tests/features/steps/s3_server_container.py
+++ b/extensions/aws/tests/features/steps/s3_server_container.py
@@ -16,12 +16,12 @@
import json
import logging
-from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.container_linux import LinuxContainer
from minifi_test_framework.core.helpers import wait_for_condition
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
-class S3ServerContainer(Container):
+class S3ServerContainer(LinuxContainer):
def __init__(self, test_context: MinifiTestContext):
super().__init__("adobe/s3mock:3.12.0", f"s3-server-{test_context.scenario_id}", test_context.network)
self.environment.append("initialBuckets=test_bucket")
diff --git a/extensions/azure/tests/features/steps/azure_server_container.py b/extensions/azure/tests/features/steps/azure_server_container.py
index 4caeb786b..cb0d2e5eb 100644
--- a/extensions/azure/tests/features/steps/azure_server_container.py
+++ b/extensions/azure/tests/features/steps/azure_server_container.py
@@ -18,13 +18,13 @@
import logging
from docker.errors import ContainerError
-from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.container_linux import LinuxContainer
from minifi_test_framework.core.helpers import run_cmd_in_docker_image
from minifi_test_framework.core.helpers import wait_for_condition
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
-class AzureServerContainer(Container):
+class AzureServerContainer(LinuxContainer):
def __init__(self, test_context: MinifiTestContext):
super().__init__("mcr.microsoft.com/azure-storage/azurite:3.35.0",
f"azure-storage-server-{test_context.scenario_id}",
diff --git a/extensions/couchbase/tests/features/steps/couchbase_server_container.py b/extensions/couchbase/tests/features/steps/couchbase_server_container.py
index 52228951d..c0471be50 100644
--- a/extensions/couchbase/tests/features/steps/couchbase_server_container.py
+++ b/extensions/couchbase/tests/features/steps/couchbase_server_container.py
@@ -15,26 +15,25 @@
import logging
-from OpenSSL import crypto
from minifi_test_framework.core.helpers import wait_for_condition, retry_check
-from minifi_test_framework.core.ssl_utils import make_server_cert
-from minifi_test_framework.containers.container import Container
+from minifi_test_framework.core.ssl_utils import make_server_cert, dump_cert, dump_key
+from minifi_test_framework.containers.container_linux import LinuxContainer
from minifi_test_framework.containers.file import File
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
from docker.errors import ContainerError
-class CouchbaseServerContainer(Container):
+class CouchbaseServerContainer(LinuxContainer):
def __init__(self, test_context: MinifiTestContext):
super().__init__("couchbase:enterprise-7.2.5", f"couchbase-server-{test_context.scenario_id}", test_context.network)
couchbase_cert, couchbase_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key)
- root_ca_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert)
+ root_ca_content = dump_cert(test_context.root_ca_cert)
self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/CA/root_ca.crt", root_ca_content, permissions=0o666))
- couchbase_cert_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=couchbase_cert)
+ couchbase_cert_content = dump_cert(couchbase_cert)
self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/chain.pem", couchbase_cert_content, permissions=0o666))
- couchbase_key_content = crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=couchbase_key)
+ couchbase_key_content = dump_key(couchbase_key)
self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/pkey.key", couchbase_key_content, permissions=0o666))
def deploy(self, context: MinifiTestContext | None) -> bool:
@@ -66,7 +65,7 @@ class CouchbaseServerContainer(Container):
return True
- @retry_check(max_tries=12, retry_interval=5)
+ @retry_check(12, 5)
def _run_couchbase_cli_command(self, command):
(code, output) = self.exec_run(command)
if code != 0:
@@ -90,7 +89,7 @@ class CouchbaseServerContainer(Container):
logging.error(f"Unexpected error while running python command '{command}' in couchbase helper docker: '{e}'")
return False
- @retry_check(max_tries=15, retry_interval=2)
+ @retry_check(15, 2)
def _load_couchbase_certs(self):
python_command = f"""
import requests
diff --git a/extensions/elasticsearch/tests/features/steps/elastic_base_container.py b/extensions/elasticsearch/tests/features/steps/elastic_base_container.py
index 5db6e61dc..2287278c4 100644
--- a/extensions/elasticsearch/tests/features/steps/elastic_base_container.py
+++ b/extensions/elasticsearch/tests/features/steps/elastic_base_container.py
@@ -14,11 +14,11 @@
# limitations under the License.
from minifi_test_framework.core.helpers import wait_for_condition, retry_check
-from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.container_linux import LinuxContainer
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
-class ElasticBaseContainer(Container):
+class ElasticBaseContainer(LinuxContainer):
def __init__(self, test_context: MinifiTestContext, image: str, container_name: str):
super().__init__(image, container_name, test_context.network)
self.user = None
diff --git a/extensions/elasticsearch/tests/features/steps/elasticsearch_container.py b/extensions/elasticsearch/tests/features/steps/elasticsearch_container.py
index 037711d75..aec928026 100644
--- a/extensions/elasticsearch/tests/features/steps/elasticsearch_container.py
+++ b/extensions/elasticsearch/tests/features/steps/elasticsearch_container.py
@@ -19,8 +19,7 @@ import os
from elastic_base_container import ElasticBaseContainer
from pathlib import Path
-from OpenSSL import crypto
-from minifi_test_framework.core.ssl_utils import make_server_cert, make_cert_without_extended_usage
+from minifi_test_framework.core.ssl_utils import make_server_cert, make_cert_without_extended_usage, dump_cert, dump_key
from minifi_test_framework.containers.file import File
from minifi_test_framework.containers.host_file import HostFile
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
@@ -33,19 +32,19 @@ class ElasticsearchContainer(ElasticBaseContainer):
http_cert, http_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key)
transport_cert, transport_key = make_cert_without_extended_usage(self.container_name, test_context.root_ca_cert, test_context.root_ca_key)
- root_ca_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert)
+ root_ca_content = dump_cert(test_context.root_ca_cert)
self.files.append(File("/usr/share/elasticsearch/config/certs/root_ca.crt", root_ca_content, permissions=0o644))
- http_cert_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=http_cert)
+ http_cert_content = dump_cert(http_cert)
self.files.append(File("/usr/share/elasticsearch/config/certs/elastic_http.crt", http_cert_content, permissions=0o644))
- http_key_content = crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=http_key)
+ http_key_content = dump_key(http_key)
self.files.append(File("/usr/share/elasticsearch/config/certs/elastic_http.key", http_key_content, permissions=0o644))
- transport_cert_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=transport_cert)
+ transport_cert_content = dump_cert(transport_cert)
self.files.append(File("/usr/share/elasticsearch/config/certs/elastic_transport.crt", transport_cert_content, permissions=0o644))
- transport_key_content = crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=transport_key)
+ transport_key_content = dump_key(transport_key)
self.files.append(File("/usr/share/elasticsearch/config/certs/elastic_transport.key", transport_key_content, permissions=0o644))
features_dir = Path(__file__).resolve().parent.parent
diff --git a/extensions/elasticsearch/tests/features/steps/opensearch_container.py b/extensions/elasticsearch/tests/features/steps/opensearch_container.py
index cdf5008d5..234a26d12 100644
--- a/extensions/elasticsearch/tests/features/steps/opensearch_container.py
+++ b/extensions/elasticsearch/tests/features/steps/opensearch_container.py
@@ -18,8 +18,7 @@ import logging
from elastic_base_container import ElasticBaseContainer
from pathlib import Path
-from OpenSSL import crypto
-from minifi_test_framework.core.ssl_utils import make_server_cert
+from minifi_test_framework.core.ssl_utils import make_server_cert, dump_cert, dump_key
from minifi_test_framework.containers.file import File
from minifi_test_framework.containers.host_file import HostFile
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
@@ -31,13 +30,13 @@ class OpensearchContainer(ElasticBaseContainer):
admin_pem, admin_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key)
- root_ca_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert)
+ root_ca_content = dump_cert(test_context.root_ca_cert)
self.files.append(File("/usr/share/opensearch/config/root-ca.pem", root_ca_content, permissions=0o644))
- admin_pem_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=admin_pem)
+ admin_pem_content = dump_cert(admin_pem)
self.files.append(File("/usr/share/opensearch/config/admin.pem", admin_pem_content, permissions=0o644))
- admin_key_content = crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=admin_key)
+ admin_key_content = dump_key(admin_key)
self.files.append(File("/usr/share/opensearch/config/admin-key.pem", admin_key_content, permissions=0o644))
features_dir = Path(__file__).resolve().parent.parent
diff --git a/extensions/gcp/tests/features/steps/fake_gcs_server_container.py b/extensions/gcp/tests/features/steps/fake_gcs_server_container.py
index a9733ba4e..5689065bb 100644
--- a/extensions/gcp/tests/features/steps/fake_gcs_server_container.py
+++ b/extensions/gcp/tests/features/steps/fake_gcs_server_container.py
@@ -15,12 +15,12 @@
import logging
from minifi_test_framework.core.helpers import wait_for_condition, retry_check
-from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.container_linux import LinuxContainer
from minifi_test_framework.containers.directory import Directory
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
-class FakeGcsServerContainer(Container):
+class FakeGcsServerContainer(LinuxContainer):
def __init__(self, test_context: MinifiTestContext):
super().__init__("fsouza/fake-gcs-server:1.45.1", f"fake-gcs-server-{test_context.scenario_id}", test_context.network,
command=f'-scheme http -host fake-gcs-server-{test_context.scenario_id}')
diff --git a/extensions/grafana-loki/tests/features/steps/grafana_loki_container.py b/extensions/grafana-loki/tests/features/steps/grafana_loki_container.py
index 1284b95b0..10c77defd 100644
--- a/extensions/grafana-loki/tests/features/steps/grafana_loki_container.py
+++ b/extensions/grafana-loki/tests/features/steps/grafana_loki_container.py
@@ -14,13 +14,12 @@
# limitations under the License.
import logging
-from OpenSSL import crypto
from minifi_test_framework.core.helpers import wait_for_condition, retry_check
-from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.container_linux import LinuxContainer
from minifi_test_framework.containers.file import File
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
-from minifi_test_framework.core.ssl_utils import make_server_cert
+from minifi_test_framework.core.ssl_utils import make_server_cert, dump_cert, dump_key
from docker.errors import ContainerError
@@ -30,20 +29,20 @@ class GrafanaLokiOptions:
self.enable_multi_tenancy = enable_multi_tenancy
-class GrafanaLokiContainer(Container):
+class GrafanaLokiContainer(LinuxContainer):
def __init__(self, test_context: MinifiTestContext, options: GrafanaLokiOptions):
super().__init__("grafana/loki:3.2.1", f"grafana-loki-server-{test_context.scenario_id}", test_context.network)
extra_ssl_settings = ""
if options.enable_ssl:
grafana_loki_cert, grafana_loki_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key)
- root_ca_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert)
+ root_ca_content = dump_cert(test_context.root_ca_cert)
self.files.append(File("/etc/loki/root_ca.crt", root_ca_content, permissions=0o644))
- grafana_loki_cert_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=grafana_loki_cert)
+ grafana_loki_cert_content = dump_cert(grafana_loki_cert)
self.files.append(File("/etc/loki/cert.pem", grafana_loki_cert_content, permissions=0o644))
- grafana_loki_key_content = crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=grafana_loki_key)
+ grafana_loki_key_content = dump_key(grafana_loki_key)
self.files.append(File("/etc/loki/key.pem", grafana_loki_key_content, permissions=0o644))
extra_ssl_settings = """
diff --git a/extensions/grafana-loki/tests/features/steps/reverse_proxy_container.py b/extensions/grafana-loki/tests/features/steps/reverse_proxy_container.py
index 409a9b9e6..43efd49c5 100644
--- a/extensions/grafana-loki/tests/features/steps/reverse_proxy_container.py
+++ b/extensions/grafana-loki/tests/features/steps/reverse_proxy_container.py
@@ -14,11 +14,11 @@
# limitations under the License.
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
-from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.container_linux import LinuxContainer
from minifi_test_framework.core.helpers import wait_for_condition
-class ReverseProxyContainer(Container):
+class ReverseProxyContainer(LinuxContainer):
def __init__(self, test_context: MinifiTestContext):
super().__init__("minifi-reverse-proxy:latest", f"reverse-proxy-{test_context.scenario_id}", test_context.network)
self.environment = [
diff --git a/extensions/kafka/tests/features/steps/kafka_server_container.py b/extensions/kafka/tests/features/steps/kafka_server_container.py
index 30073186f..09e35e131 100644
--- a/extensions/kafka/tests/features/steps/kafka_server_container.py
+++ b/extensions/kafka/tests/features/steps/kafka_server_container.py
@@ -15,17 +15,18 @@
import logging
import re
+
+from cryptography.hazmat.primitives import serialization
import jks
-from OpenSSL import crypto
from minifi_test_framework.core.helpers import wait_for_condition
-from minifi_test_framework.core.ssl_utils import make_server_cert
-from minifi_test_framework.containers.container import Container
+from minifi_test_framework.core.ssl_utils import make_server_cert, dump_cert, dump_key
+from minifi_test_framework.containers.container_linux import LinuxContainer
from minifi_test_framework.containers.file import File
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
-class KafkaServer(Container):
+class KafkaServer(LinuxContainer):
def __init__(self, test_context: MinifiTestContext):
super().__init__("apache/kafka:4.1.0", f"kafka-server-{test_context.scenario_id}", test_context.network)
self.environment.append("KAFKA_NODE_ID=1")
@@ -55,7 +56,7 @@ class KafkaServer(Container):
kafka_cert, kafka_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key)
- pke = jks.PrivateKeyEntry.new('kafka-broker-cert', [crypto.dump_certificate(crypto.FILETYPE_ASN1, kafka_cert)], crypto.dump_privatekey(crypto.FILETYPE_ASN1, kafka_key), 'rsa_raw')
+ pke = jks.PrivateKeyEntry.new('kafka-broker-cert', [dump_cert(kafka_cert, encoding_type=serialization.Encoding.DER)], dump_key(kafka_key, encoding_type=serialization.Encoding.DER), 'rsa_raw')
server_keystore = jks.KeyStore.new('jks', [pke])
server_keystore_content = server_keystore.saves('abcdefgh')
self.files.append(File("/etc/kafka/secrets/kafka.keystore.jks", server_keystore_content, permissions=0o644))
@@ -63,7 +64,7 @@ class KafkaServer(Container):
trusted_cert = jks.TrustedCertEntry.new(
'root-ca', # Alias for the certificate
- crypto.dump_certificate(crypto.FILETYPE_ASN1, test_context.root_ca_cert)
+ dump_cert(test_context.root_ca_cert, encoding_type=serialization.Encoding.DER)
)
# Create a JKS keystore that will serve as a truststore with the trusted cert entry.
diff --git a/extensions/opc/tests/features/steps/opc_ua_server_container.py b/extensions/opc/tests/features/steps/opc_ua_server_container.py
index 868e54202..c9ba312ec 100644
--- a/extensions/opc/tests/features/steps/opc_ua_server_container.py
+++ b/extensions/opc/tests/features/steps/opc_ua_server_container.py
@@ -14,12 +14,12 @@
# limitations under the License.
from typing import List, Optional
-from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.container_linux import LinuxContainer
from minifi_test_framework.core.helpers import wait_for_condition
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
-class OPCUAServerContainer(Container):
+class OPCUAServerContainer(LinuxContainer):
def __init__(self, test_context: MinifiTestContext, command: Optional[List[str]] = None):
super().__init__("lordgamez/open62541:1.4.10", f"opcua-server-{test_context.scenario_id}", test_context.network, command=command)
diff --git a/extensions/splunk/tests/features/steps/splunk_container.py b/extensions/splunk/tests/features/steps/splunk_container.py
index 4106bb63e..9a1488e63 100644
--- a/extensions/splunk/tests/features/steps/splunk_container.py
+++ b/extensions/splunk/tests/features/steps/splunk_container.py
@@ -15,15 +15,14 @@
import json
-from OpenSSL import crypto
-from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.container_linux import LinuxContainer
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
from minifi_test_framework.core.helpers import wait_for_condition, retry_check
from minifi_test_framework.containers.file import File
-from minifi_test_framework.core.ssl_utils import make_server_cert
+from minifi_test_framework.core.ssl_utils import make_server_cert, dump_cert, dump_key
-class SplunkContainer(Container):
+class SplunkContainer(LinuxContainer):
def __init__(self, test_context: MinifiTestContext):
super().__init__("splunk/splunk:9.2.1-patch2", f"splunk-{test_context.scenario_id}", test_context.network)
self.user = None
@@ -43,9 +42,9 @@ splunk:
self.files.append(File("/tmp/defaults/default.yml", splunk_config_content, mode="rw", permissions=0o644))
splunk_cert, splunk_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key)
- splunk_cert_content = crypto.dump_certificate(crypto.FILETYPE_PEM, splunk_cert)
- splunk_key_content = crypto.dump_privatekey(crypto.FILETYPE_PEM, splunk_key)
- root_ca_content = crypto.dump_certificate(crypto.FILETYPE_PEM, test_context.root_ca_cert)
+ splunk_cert_content = dump_cert(splunk_cert)
+ splunk_key_content = dump_key(splunk_key)
+ root_ca_content = dump_cert(test_context.root_ca_cert)
self.files.append(File("/opt/splunk/etc/auth/splunk_cert.pem", splunk_cert_content.decode() + splunk_key_content.decode() + root_ca_content.decode(), permissions=0o644))
self.files.append(File("/opt/splunk/etc/auth/root_ca.pem", root_ca_content.decode(), permissions=0o644))
diff --git a/extensions/sql/tests/features/steps/postgress_server_container.py b/extensions/sql/tests/features/steps/postgress_server_container.py
index 467eecc4b..a61209528 100644
--- a/extensions/sql/tests/features/steps/postgress_server_container.py
+++ b/extensions/sql/tests/features/steps/postgress_server_container.py
@@ -17,13 +17,13 @@
import logging
from textwrap import dedent
-from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.container_linux import LinuxContainer
from minifi_test_framework.containers.docker_image_builder import DockerImageBuilder
from minifi_test_framework.core.helpers import wait_for_condition
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
-class PostgresContainer(Container):
+class PostgresContainer(LinuxContainer):
def __init__(self, context):
dockerfile = dedent("""\
FROM {base_image}
diff --git a/extensions/standard-processors/tests/features/core_functionality.feature b/extensions/standard-processors/tests/features/core_functionality.feature
index 4b5f6b38b..0acdbb576 100644
--- a/extensions/standard-processors/tests/features/core_functionality.feature
+++ b/extensions/standard-processors/tests/features/core_functionality.feature
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-@CORE
+@CORE @SUPPORTS_WINDOWS
Feature: Core flow functionalities
Test core flow configuration functionalities
@@ -33,8 +33,8 @@ Feature: Core flow functionalities
When all instances start up
- Then at least one file with the content "first_custom_text" is placed in the "/tmp/output" directory in less than 20 seconds
- And at least one file with the content "second_custom_text" is placed in the "/tmp/output" directory in less than 20 seconds
+ Then at least one file with the content "first_custom_text" is placed in the "/tmp/output" directory in less than 200 seconds
+ And at least one file with the content "second_custom_text" is placed in the "/tmp/output" directory in less than 200 seconds
Scenario: A funnel can be used as a terminator
Given a GenerateFlowFile processor with the "Data Format" property set to "Text"
@@ -77,7 +77,7 @@ Feature: Core flow functionalities
And a non-sensitive parameter in the flow config called 'FILE_INPUT_PATH' with the value '/tmp/input' in the parameter context 'my-context'
And a non-sensitive parameter in the flow config called 'FILE_OUTPUT_UPPER_PATH_ATTR' with the value 'upper_out_path_attr' in the parameter context 'my-context'
And a GetFile processor with the "Input Directory" property set to "#{FILE_INPUT_PATH}"
- And a file with filename "test_file_name" and content "test content" is present in "/tmp/input"
+ And a directory at "/tmp/input" has a file ("test_file_name") with the content "test content"
And a UpdateAttribute processor with the "expr-lang-filename" property set to "#{FILENAME}"
And the "is-upper-correct" property of the UpdateAttribute processor is set to "${#{FILENAME_IN_EXPRESSION}:toUpper():equals('TEST_FILE_NAME')}"
And the "upper_out_path_attr" property of the UpdateAttribute processor is set to "/TMP/OUTPUT"
diff --git a/extensions/standard-processors/tests/features/replace_text.feature b/extensions/standard-processors/tests/features/replace_text.feature
index f8a3fdb05..b24a0e873 100644
--- a/extensions/standard-processors/tests/features/replace_text.feature
+++ b/extensions/standard-processors/tests/features/replace_text.feature
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-@CORE
+@CORE @SUPPORTS_WINDOWS
Feature: Changing flowfile contents using the ReplaceText processor
Scenario Outline: Replace text using Entire text mode
diff --git a/extensions/standard-processors/tests/features/steps/diag_slave_container.py b/extensions/standard-processors/tests/features/steps/diag_slave_container.py
index ba89e75c3..8defe490f 100644
--- a/extensions/standard-processors/tests/features/steps/diag_slave_container.py
+++ b/extensions/standard-processors/tests/features/steps/diag_slave_container.py
@@ -18,13 +18,13 @@
import logging
from textwrap import dedent
-from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.container_linux import LinuxContainer
from minifi_test_framework.containers.docker_image_builder import DockerImageBuilder
from minifi_test_framework.core.helpers import wait_for_condition
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
-class DiagSlave(Container):
+class DiagSlave(LinuxContainer):
def __init__(self, test_context: MinifiTestContext):
dockerfile = dedent("""\
FROM panterdsd/diagslave:latest
diff --git a/extensions/standard-processors/tests/features/steps/minifi_controller_steps.py b/extensions/standard-processors/tests/features/steps/minifi_controller_steps.py
index c65d67e0f..c53df9b02 100644
--- a/extensions/standard-processors/tests/features/steps/minifi_controller_steps.py
+++ b/extensions/standard-processors/tests/features/steps/minifi_controller_steps.py
@@ -22,52 +22,52 @@ from minifi_test_framework.core.minifi_test_context import MinifiTestContext
@given('controller socket properties are set up')
def step_impl(context: MinifiTestContext):
- context.get_or_create_default_minifi_container().set_controller_socket_properties()
+ context.get_or_create_default_minifi_container().controller.set_controller_socket_properties()
@when('MiNiFi config is updated through MiNiFi controller')
def step_impl(context: MinifiTestContext):
- context.get_or_create_default_minifi_container().update_flow_config_through_controller()
+ context.get_or_create_default_minifi_container().controller.update_flow_config_through_controller()
@then('the updated config is persisted')
def step_impl(context: MinifiTestContext):
- assert context.get_or_create_default_minifi_container().updated_config_is_persisted()
+ assert context.get_or_create_default_minifi_container().controller.updated_config_is_persisted()
@when('the {component} component is stopped through MiNiFi controller')
def step_impl(context: MinifiTestContext, component: str):
- context.get_or_create_default_minifi_container().stop_component_through_controller(component)
+ context.get_or_create_default_minifi_container().controller.stop_component_through_controller(component)
@when('the {component} component is started through MiNiFi controller')
def step_impl(context: MinifiTestContext, component: str):
- context.get_or_create_default_minifi_container().start_component_through_controller(component)
+ context.get_or_create_default_minifi_container().controller.start_component_through_controller(component)
@then('the {component} component is not running')
def step_impl(context: MinifiTestContext, component: str):
- assert not context.get_or_create_default_minifi_container().is_component_running(component)
+ assert not context.get_or_create_default_minifi_container().controller.is_component_running(component)
@then('the {component} component is running')
def step_impl(context: MinifiTestContext, component: str):
- assert context.get_or_create_default_minifi_container().is_component_running(component)
+ assert context.get_or_create_default_minifi_container().controller.is_component_running(component)
@then('connection \"{connection}\" can be seen through MiNiFi controller')
def step_impl(context: MinifiTestContext, connection: str):
- assert context.get_or_create_default_minifi_container().connection_found_through_controller(connection)
+ assert context.get_or_create_default_minifi_container().controller.connection_found_through_controller(connection)
@then('{connection_count:d} connections can be seen full through MiNiFi controller')
def step_impl(context: MinifiTestContext, connection_count: int):
- assert context.get_or_create_default_minifi_container().get_full_connection_count() == connection_count
+ assert context.get_or_create_default_minifi_container().controller.get_full_connection_count() == connection_count
@retry_check(10, 1)
def check_connection_size_through_controller(context: MinifiTestContext, connection: str, size: int, max_size: int) -> bool:
- return context.get_or_create_default_minifi_container().get_connection_size(connection) == (size, max_size)
+ return context.get_or_create_default_minifi_container().controller.get_connection_size(connection) == (size, max_size)
@then('connection \"{connection}\" has {size:d} size and {max_size:d} max size through MiNiFi controller')
@@ -77,7 +77,7 @@ def step_impl(context: MinifiTestContext, connection: str, size: int, max_size:
@retry_check(10, 1)
def manifest_can_be_retrieved_through_minifi_controller(context: MinifiTestContext) -> bool:
- manifest = context.get_or_create_default_minifi_container().get_manifest()
+ manifest = context.get_or_create_default_minifi_container().controller.get_manifest()
return '"agentManifest": {' in manifest and '"componentManifest": {' in manifest and '"agentType": "cpp"' in manifest
@@ -88,4 +88,4 @@ def step_impl(context: MinifiTestContext):
@then('debug bundle can be retrieved through MiNiFi controller')
def step_impl(context: MinifiTestContext):
- assert context.get_or_create_default_minifi_container().create_debug_bundle()
+ assert context.get_or_create_default_minifi_container().controller.create_debug_bundle()
diff --git a/extensions/standard-processors/tests/features/steps/syslog_container.py b/extensions/standard-processors/tests/features/steps/syslog_container.py
index f2da956f5..c831af6f1 100644
--- a/extensions/standard-processors/tests/features/steps/syslog_container.py
+++ b/extensions/standard-processors/tests/features/steps/syslog_container.py
@@ -15,10 +15,10 @@
# limitations under the License.
#
-from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.container_linux import LinuxContainer
-class SyslogContainer(Container):
+class SyslogContainer(LinuxContainer):
def __init__(self, protocol, context):
super(SyslogContainer, self).__init__("ubuntu:24.04", f"syslog-{protocol}-{context.scenario_id}", context.network)
self.command = f'/bin/bash -c "echo Syslog {protocol} client started; while true; do logger --{protocol} -n minifi-primary-{context.scenario_id} -P 514 sample_log; sleep 1; done"'
diff --git a/extensions/standard-processors/tests/features/steps/tcp_client_container.py b/extensions/standard-processors/tests/features/steps/tcp_client_container.py
index aca77dfae..0d90f3622 100644
--- a/extensions/standard-processors/tests/features/steps/tcp_client_container.py
+++ b/extensions/standard-processors/tests/features/steps/tcp_client_container.py
@@ -13,12 +13,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.container_linux import LinuxContainer
from minifi_test_framework.core.helpers import wait_for_condition
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
-class TcpClientContainer(Container):
+class TcpClientContainer(LinuxContainer):
def __init__(self, test_context: MinifiTestContext):
cmd = (
"/bin/sh -c 'apk add netcat-openbsd && "
diff --git a/run_flake8.sh b/run_flake8.sh
index 2288893bc..1effd79ae 100755
--- a/run_flake8.sh
+++ b/run_flake8.sh
@@ -19,4 +19,4 @@
set -euo pipefail
directory=${1:-.}
-flake8 --exclude .venv,venv,thirdparty,build,cmake-build-*,github_env --builtins log,REL_SUCCESS,REL_FAILURE,REL_ORIGINAL,raw_input --ignore E501,W503,F811 "${directory}"
+flake8 --exclude .venv,venv,venv2,behave-venv,thirdparty,build,cmake-build-*,github_env --builtins log,REL_SUCCESS,REL_FAILURE,REL_ORIGINAL,raw_input --ignore E501,W503,F811 "${directory}"
(nifi-minifi-cpp) 02/04: MINIFICPP-2712 Generate modular docs
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
martinzink pushed a commit to branch apache-rusty
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit d1060ebf9afb33f7716279c03f58c8efce58819a
Author: Martin Zink <ma...@apache.org>
AuthorDate: Thu Feb 12 16:52:45 2026 +0100
MINIFICPP-2712 Generate modular docs
With this PR, ./minifi --docs <output_dir> will generate additional md files one per extension <output_dir>/modules
Closes #2097
Signed-off-by: Martin Zink <ma...@apache.org>
---
minifi_main/AgentDocs.cpp | 214 +++++++++++++++++++++++++++++++---------------
1 file changed, 146 insertions(+), 68 deletions(-)
diff --git a/minifi_main/AgentDocs.cpp b/minifi_main/AgentDocs.cpp
index ff37268c9..3ce243ca8 100644
--- a/minifi_main/AgentDocs.cpp
+++ b/minifi_main/AgentDocs.cpp
@@ -35,6 +35,7 @@
#include "range/v3/view/join.hpp"
#include "range/v3/view/transform.hpp"
#include "utils/StringUtils.h"
+#include "utils/file/FileUtils.h"
namespace {
@@ -42,33 +43,24 @@ namespace minifi = org::apache::nifi::minifi;
std::string formatName(std::string_view name_view, bool is_required) {
std::string name{name_view};
- if (is_required) {
- return "**" + name + "**";
- } else {
- return name;
- }
+ if (is_required) { return "**" + name + "**"; }
+ return name;
}
std::string formatAllowedValues(const minifi::core::Property& property) {
- if (property.getValidator().getEquivalentNifiStandardValidatorName() == minifi::core::StandardPropertyValidators::BOOLEAN_VALIDATOR.getEquivalentNifiStandardValidatorName()) {
+ if (property.getValidator().getEquivalentNifiStandardValidatorName() ==
+ minifi::core::StandardPropertyValidators::BOOLEAN_VALIDATOR.getEquivalentNifiStandardValidatorName()) {
return "true<br/>false";
- } else {
- const auto allowed_values = property.getAllowedValues();
- return allowed_values
- | ranges::views::join(std::string_view{"<br/>"})
- | ranges::to<std::string>();
}
+ const auto allowed_values = property.getAllowedValues();
+ return allowed_values | ranges::views::join(std::string_view{"<br/>"}) | ranges::to<std::string>();
}
std::string formatDescription(std::string_view description_view, bool is_sensitive = false, bool supports_expression_language = false) {
std::string description{description_view};
minifi::utils::string::replaceAll(description, "\n", "<br/>");
- if (is_sensitive) {
- description += "<br/>**Sensitive Property: true**";
- }
- if (supports_expression_language) {
- description += "<br/>**Supports Expression Language: true**";
- }
+ if (is_sensitive) { description += "<br/>**Sensitive Property: true**"; }
+ if (supports_expression_language) { description += "<br/>**Supports Expression Language: true**"; }
return description;
}
@@ -103,7 +95,7 @@ void writeHeader(std::ostream& docs, const std::vector<std::pair<std::string, mi
docs << APACHE_LICENSE;
docs << "\n\n## Table of Contents\n\n";
- for (const auto& [name, documentation] : class_descriptions) {
+ for (const auto& [name, documentation]: class_descriptions) {
docs << "- [" << name << "](#" << name << ")\n";
}
}
@@ -122,13 +114,11 @@ void writeProperties(std::ostream& docs, const minifi::ClassDescription& documen
docs << "\n\nIn the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. "
<< "The table also indicates any default values, and whether a property supports the NiFi Expression Language.";
minifi::docs::Table properties{{"Name", "Default Value", "Allowable Values", "Description"}};
- for (const auto &property : documentation.class_properties_) {
- properties.addRow({
- formatName(property.getName(), property.getRequired()),
+ for (const auto& property: documentation.class_properties_) {
+ properties.addRow({formatName(property.getName(), property.getRequired()),
property.getDefaultValue().value_or(""),
formatAllowedValues(property),
- formatDescription(property)
- });
+ formatDescription(property)});
}
docs << "\n\n" << properties.toString();
}
@@ -138,12 +128,8 @@ void writeDynamicProperties(std::ostream& docs, const minifi::ClassDescription&
docs << "\n### Dynamic Properties\n\n";
minifi::docs::Table dynamic_properties{{"Name", "Value", "Description"}};
- for (const auto &dynamic_property : documentation.dynamic_properties_) {
- dynamic_properties.addRow({
- formatName(dynamic_property.name, false),
- std::string(dynamic_property.value),
- formatDescription(dynamic_property)
- });
+ for (const auto& dynamic_property: documentation.dynamic_properties_) {
+ dynamic_properties.addRow({formatName(dynamic_property.name, false), std::string(dynamic_property.value), formatDescription(dynamic_property)});
}
docs << dynamic_properties.toString();
}
@@ -151,7 +137,7 @@ void writeDynamicProperties(std::ostream& docs, const minifi::ClassDescription&
void writeRelationships(std::ostream& docs, const minifi::ClassDescription& documentation) {
docs << "\n### Relationships\n\n";
minifi::docs::Table relationships{{"Name", "Description"}};
- for (const auto &rel : documentation.class_relationships_) {
+ for (const auto& rel: documentation.class_relationships_) {
relationships.addRow({rel.getName(), formatDescription(rel.getDescription())});
}
docs << relationships.toString();
@@ -162,9 +148,8 @@ void writeOutputAttributes(std::ostream& docs, const minifi::ClassDescription& d
docs << "\n### Output Attributes";
minifi::docs::Table output_attributes{{"Attribute", "Relationship", "Description"}};
- for (const auto &output_attribute : documentation.output_attributes_) {
- output_attributes.addRow({
- std::string(output_attribute.name),
+ for (const auto& output_attribute: documentation.output_attributes_) {
+ output_attributes.addRow({std::string(output_attribute.name),
formatListOfRelationships(output_attribute.relationships),
formatDescription(output_attribute.description)});
}
@@ -177,58 +162,151 @@ std::string extractClassName(const std::string& full_class_name) {
std::string lowercaseFirst(const std::pair<std::string, minifi::ClassDescription>& key_value) {
return minifi::utils::string::toLower(key_value.first);
-};
+}
-} // namespace
+void writeProcessor(std::ostream& os, const std::string_view name, const minifi::ClassDescription& documentation) {
+ writeName(os, name);
+ writeDescription(os, documentation);
+ writeProperties(os, documentation);
+ writeDynamicProperties(os, documentation);
+ writeRelationships(os, documentation);
+ writeOutputAttributes(os, documentation);
+}
-namespace org::apache::nifi::minifi::docs {
+void writeControllerService(std::ostream& os, const std::string_view name, const minifi::ClassDescription& documentation) {
+ writeName(os, name);
+ writeDescription(os, documentation);
+ writeProperties(os, documentation);
+}
-void AgentDocs::generate(const std::filesystem::path& docs_dir) {
- std::vector<std::pair<std::string, minifi::ClassDescription>> controller_services;
- std::vector<std::pair<std::string, minifi::ClassDescription>> processors;
- std::vector<std::pair<std::string, minifi::ClassDescription>> parameter_providers;
+void writeParameterProvider(std::ostream& os, const std::string_view name, const minifi::ClassDescription& documentation) {
+ writeName(os, name);
+ writeDescription(os, documentation);
+ writeProperties(os, documentation);
+}
- for (const auto& [bundle_id, components]: ClassDescriptionRegistry::getClassDescriptions()) {
- for (const auto &controller_service_description : components.controller_services) {
+class MonolithDocumentation {
+ public:
+ explicit MonolithDocumentation() {
+ for (const auto& [bundle_id, component]: minifi::ClassDescriptionRegistry::getClassDescriptions()) {
+ addComponents(component);
+ }
+ sort();
+ }
+
+ void write(const std::filesystem::path& docs_dir) {
+ std::ofstream controllers_md(docs_dir / "CONTROLLERS.md");
+ std::ofstream processors_md(docs_dir / "PROCESSORS.md");
+ std::ofstream parameter_providers_md(docs_dir / "PARAMETER_PROVIDERS.md");
+ gsl_Assert(controllers_md && processors_md && parameter_providers_md);
+
+ writeControllers(controllers_md);
+ writeProcessors(processors_md);
+ writeParameterProviders(parameter_providers_md);
+ }
+
+ private:
+ void addComponents(const minifi::Components& components) {
+ for (const auto& controller_service_description: components.controller_services) {
controller_services.emplace_back(extractClassName(controller_service_description.full_name_), controller_service_description);
}
- for (const auto &processor_description : components.processors) {
+ for (const auto& processor_description: components.processors) {
processors.emplace_back(extractClassName(processor_description.full_name_), processor_description);
}
- for (const auto& parameter_provider_description : components.parameter_providers) {
+ for (const auto& parameter_provider_description: components.parameter_providers) {
parameter_providers.emplace_back(extractClassName(parameter_provider_description.full_name_), parameter_provider_description);
}
}
- std::ranges::sort(controller_services, std::less(), lowercaseFirst);
- std::ranges::sort(processors, std::less(), lowercaseFirst);
- std::ranges::sort(parameter_providers, std::less(), lowercaseFirst);
- std::ofstream controllers_md(docs_dir / "CONTROLLERS.md");
- writeHeader(controllers_md, controller_services);
- for (const auto& [name, documentation] : controller_services) {
- writeName(controllers_md, name);
- writeDescription(controllers_md, documentation);
- writeProperties(controllers_md, documentation);
+ void sort() {
+ std::ranges::sort(controller_services, std::less(), lowercaseFirst);
+ std::ranges::sort(processors, std::less(), lowercaseFirst);
+ std::ranges::sort(parameter_providers, std::less(), lowercaseFirst);
+ }
+
+ void writeControllers(std::ostream& os) {
+ writeHeader(os, controller_services);
+ for (const auto& [name, documentation]: controller_services) {
+ writeControllerService(os, name, documentation);
+ }
+ }
+
+ void writeProcessors(std::ostream& os) {
+ writeHeader(os, processors);
+ for (const auto& [name, documentation]: processors) {
+ writeProcessor(os, name, documentation);
+ }
+ }
+
+ void writeParameterProviders(std::ostream& os) {
+ writeHeader(os, parameter_providers);
+ for (const auto& [name, documentation]: parameter_providers) {
+ writeParameterProvider(os, name, documentation);
+ }
+ }
+
+ std::vector<std::pair<std::string, minifi::ClassDescription>> controller_services;
+ std::vector<std::pair<std::string, minifi::ClassDescription>> processors;
+ std::vector<std::pair<std::string, minifi::ClassDescription>> parameter_providers;
+};
+
+class ModularDocumentation {
+ public:
+ static void write(const std::filesystem::path& docs_dir) {
+ for (const auto& [bundle_id, component]: minifi::ClassDescriptionRegistry::getClassDescriptions()) {
+ writeModule(docs_dir, bundle_id.name, component);
+ }
}
- std::ofstream processors_md(docs_dir / "PROCESSORS.md");
- writeHeader(processors_md, processors);
- for (const auto& [name, documentation] : processors) {
- writeName(processors_md, name);
- writeDescription(processors_md, documentation);
- writeProperties(processors_md, documentation);
- writeDynamicProperties(processors_md, documentation);
- writeRelationships(processors_md, documentation);
- writeOutputAttributes(processors_md, documentation);
+ private:
+ static void writeComponentParts(std::ostream& os, const std::vector<minifi::ClassDescription>& class_descriptions, const std::string_view h3) {
+ if (!class_descriptions.empty()) {
+ os << fmt::format("### {}\n\n", h3);
+ for (const auto& class_description: class_descriptions) {
+ const auto name = extractClassName(class_description.full_name_);
+ os << "- [" << name << "](#" << name << ")\n";
+ }
+ }
}
- std::ofstream parameter_providers_md(docs_dir / "PARAMETER_PROVIDERS.md");
- writeHeader(parameter_providers_md, parameter_providers);
- for (const auto& [name, documentation] : parameter_providers) {
- writeName(parameter_providers_md, name);
- writeDescription(parameter_providers_md, documentation);
- writeProperties(parameter_providers_md, documentation);
+ static void writeToC(std::ostream& os, const minifi::Components& components) {
+ os << "\n\n## Table of Contents\n\n";
+ writeComponentParts(os, components.processors, "Processors");
+ writeComponentParts(os, components.controller_services, "Controller Services");
+ writeComponentParts(os, components.parameter_providers, "Parameter Providers");
}
+
+ static void writeModule(const std::filesystem::path& docs_dir, const std::string_view module_name, const minifi::Components& components) {
+ const auto dir_creation_result = minifi::utils::file::create_dir(docs_dir / "modules");
+ gsl_Assert(dir_creation_result == 0);
+ std::ofstream os(docs_dir / "modules" / (std::string(module_name) + ".md"));
+ gsl_Assert(os);
+ os << APACHE_LICENSE;
+
+ writeToC(os, components);
+
+ for (const auto& processor: components.processors) {
+ writeProcessor(os, extractClassName(processor.full_name_), processor);
+ }
+
+ for (const auto& controller_service: components.controller_services) {
+ writeControllerService(os, extractClassName(controller_service.full_name_), controller_service);
+ }
+
+ for (const auto& parameter_provider_description: components.parameter_providers) {
+ writeParameterProvider(os, extractClassName(parameter_provider_description.full_name_), parameter_provider_description);
+ }
+ }
+};
+} // namespace
+
+namespace org::apache::nifi::minifi::docs {
+
+void AgentDocs::generate(const std::filesystem::path& docs_dir) {
+ MonolithDocumentation monolith_docs;
+ monolith_docs.write(docs_dir);
+
+ ModularDocumentation::write(docs_dir);
}
} // namespace org::apache::nifi::minifi::docs
(nifi-minifi-cpp) 01/04: MINIFICPP-2708 Controller Service C API
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
martinzink pushed a commit to branch apache-rusty
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 2d943909b60d099fe86de15cf7408bd837bba328
Author: Martin Zink <ma...@apache.org>
AuthorDate: Thu Feb 12 16:51:46 2026 +0100
MINIFICPP-2708 Controller Service C API
Built upon ongoing #2065 #2098 and #2105 PR-s
Closes #2096
Signed-off-by: Martin Zink <ma...@apache.org>
---
Extensions.md | 4 +-
core-framework/include/agent/agent_docs.h | 13 +-
.../core/controller/ControllerServiceBase.h | 6 +-
.../core/controller/ControllerServiceFactoryImpl.h | 4 +-
core-framework/include/http/HTTPClient.h | 1 +
core-framework/include/utils/ThreadPool.h | 2 +-
.../aws/tests/MultipartUploadStateStorageTest.cpp | 2 +-
.../llamacpp/processors/ExtensionInitializer.cpp | 2 +
extensions/opencv/OpenCVLoader.cpp | 4 +-
.../python/pythonlibloader/PythonLibLoader.cpp | 4 +-
extensions/python/pythonloader/PyProcLoader.cpp | 4 +-
extensions/sftp/SFTPLoader.cpp | 4 +-
.../controllers/NetworkPrioritizerService.h | 2 +-
.../include/core/controller/ControllerService.h | 9 +-
libminifi/include/utils/CControllerService.h | 110 ++++++++++++++++
libminifi/include/utils/CProcessor.h | 9 +-
libminifi/src/controllers/SSLContextService.cpp | 2 +-
libminifi/src/minifi-c.cpp | 138 +++++++++++++++++++--
libminifi/src/properties/Properties.cpp | 1 +
libminifi/test/libtest/integration/HTTPHandlers.h | 13 +-
.../test/libtest/unit/ControllerServiceUtils.h | 2 +-
.../test/unit/NetworkPrioritizerServiceTests.cpp | 2 +-
.../minifi-cpp/core}/ControllerServiceMetadata.h | 6 +-
minifi-api/include/minifi-c/minifi-c.h | 35 ++++++
minifi-api/include/minifi-cpp/agent/agent_docs.h | 4 +-
.../minifi-cpp/controllers/RecordSetReader.h | 4 +-
.../minifi-cpp/controllers/RecordSetWriter.h | 4 +-
.../controllers/SSLContextServiceInterface.h | 6 +-
...trollerServiceApi.h => ControllerServiceType.h} | 6 +-
...inition.h => ControllerServiceTypeDefinition.h} | 2 +-
.../core/controller/ControllerServiceDescriptor.h | 1 +
.../core/controller/ControllerServiceFactory.h | 5 +-
.../core/controller/ControllerServiceInterface.h | 15 ---
minifi-api/minifi-c-api.def | 2 +
34 files changed, 352 insertions(+), 76 deletions(-)
diff --git a/Extensions.md b/Extensions.md
index 3a82e212f..0dcf4c604 100644
--- a/Extensions.md
+++ b/Extensions.md
@@ -47,7 +47,9 @@ extern "C" MinifiExtension* MinifiInitExtension(MinifiConfig* /*config*/) {
.deinit = nullptr,
.user_data = nullptr,
.processors_count = 0,
- .processors_ptr = nullptr
+ .processors_ptr = nullptr,
+ .controller_services_count = 0,
+ .controller_services_ptr = nullptr,
};
return MinifiCreateExtension(minifi::utils::toStringView(MINIFI_API_VERSION), &ext_create_info);
}
diff --git a/core-framework/include/agent/agent_docs.h b/core-framework/include/agent/agent_docs.h
index 14544f2d6..4c7e25035 100644
--- a/core-framework/include/agent/agent_docs.h
+++ b/core-framework/include/agent/agent_docs.h
@@ -16,18 +16,17 @@
*/
#pragma once
-#include "minifi-cpp/agent/agent_docs.h"
-
#include <map>
#include <string>
#include <utility>
#include <vector>
-#include "utils/StringUtils.h"
#include "core/ClassName.h"
-#include "minifi-cpp/core/OutputAttribute.h"
-#include "minifi-cpp/core/ControllerServiceApi.h"
+#include "minifi-cpp/agent/agent_docs.h"
+#include "minifi-cpp/core/ControllerServiceType.h"
#include "minifi-cpp/core/DynamicProperty.h"
+#include "minifi-cpp/core/OutputAttribute.h"
+#include "utils/StringUtils.h"
namespace org::apache::nifi::minifi {
@@ -44,8 +43,8 @@ inline auto toVector(std::span<const core::OutputAttributeReference> attributes)
return std::vector<core::OutputAttribute>(attributes.begin(), attributes.end());
}
-inline auto toVector(std::span<const core::ControllerServiceApiDefinition> apis) {
- return std::vector<core::ControllerServiceApi>(apis.begin(), apis.end());
+inline auto toVector(std::span<const core::ControllerServiceTypeDefinition> apis) {
+ return std::vector<core::ControllerServiceType>(apis.begin(), apis.end());
}
inline auto toVector(std::span<const core::DynamicPropertyDefinition> properties) {
diff --git a/core-framework/include/core/controller/ControllerServiceBase.h b/core-framework/include/core/controller/ControllerServiceBase.h
index 2fd22b299..621f48295 100644
--- a/core-framework/include/core/controller/ControllerServiceBase.h
+++ b/core-framework/include/core/controller/ControllerServiceBase.h
@@ -28,8 +28,8 @@
#include "core/Connectable.h"
#include "minifi-cpp/core/controller/ControllerServiceApi.h"
#include "minifi-cpp/core/controller/ControllerServiceInterface.h"
-#include "minifi-cpp/core/ControllerServiceApiDefinition.h"
-#include "minifi-cpp/core/controller/ControllerServiceMetadata.h"
+#include "minifi-cpp/core/ControllerServiceTypeDefinition.h"
+#include "minifi-cpp/core/ControllerServiceMetadata.h"
namespace org::apache::nifi::minifi::core::controller {
@@ -95,7 +95,7 @@ class ControllerServiceBase : public ControllerServiceApi {
}
- static constexpr auto ImplementsApis = std::array<ControllerServiceApiDefinition, 0>{};
+ static constexpr auto ImplementsApis = std::array<ControllerServiceTypeDefinition, 0>{};
protected:
std::string name_;
diff --git a/core-framework/include/core/controller/ControllerServiceFactoryImpl.h b/core-framework/include/core/controller/ControllerServiceFactoryImpl.h
index 4060e9d18..a30bd48c6 100644
--- a/core-framework/include/core/controller/ControllerServiceFactoryImpl.h
+++ b/core-framework/include/core/controller/ControllerServiceFactoryImpl.h
@@ -17,10 +17,12 @@
#pragma once
-#include <string>
#include <memory>
+#include <string>
#include <utility>
+
#include "core/ClassName.h"
+#include "minifi-cpp/core/ControllerServiceMetadata.h"
#include "minifi-cpp/core/controller/ControllerServiceFactory.h"
namespace org::apache::nifi::minifi::core::controller {
diff --git a/core-framework/include/http/HTTPClient.h b/core-framework/include/http/HTTPClient.h
index 9339a5b21..b27c54df5 100644
--- a/core-framework/include/http/HTTPClient.h
+++ b/core-framework/include/http/HTTPClient.h
@@ -45,6 +45,7 @@
#include "minifi-cpp/core/logging/Logger.h"
#include "core/logging/LoggerFactory.h"
#include "minifi-cpp/controllers/SSLContextServiceInterface.h"
+#include "minifi-cpp/core/PropertyDefinition.h"
#include "utils/ByteArrayCallback.h"
namespace org::apache::nifi::minifi::http {
diff --git a/core-framework/include/utils/ThreadPool.h b/core-framework/include/utils/ThreadPool.h
index f7ecf8cd6..4f2b2af79 100644
--- a/core-framework/include/utils/ThreadPool.h
+++ b/core-framework/include/utils/ThreadPool.h
@@ -135,7 +135,7 @@ class WorkerThread {
*/
class ThreadPool {
public:
- ThreadPool(int max_worker_threads = 2, std::string name = "NamelessPool");
+ explicit ThreadPool(int max_worker_threads = 2, std::string name = "NamelessPool");
ThreadPool(const ThreadPool &other) = delete;
ThreadPool& operator=(const ThreadPool &other) = delete;
diff --git a/extensions/aws/tests/MultipartUploadStateStorageTest.cpp b/extensions/aws/tests/MultipartUploadStateStorageTest.cpp
index c791e85c9..fc6f757be 100644
--- a/extensions/aws/tests/MultipartUploadStateStorageTest.cpp
+++ b/extensions/aws/tests/MultipartUploadStateStorageTest.cpp
@@ -32,7 +32,7 @@ class MultipartUploadStateStorageTestFixture {
MultipartUploadStateStorageTestFixture() {
LogTestController::getInstance().setDebug<minifi::aws::s3::MultipartUploadStateStorage>();
const auto storage_uuid = minifi::utils::IdGenerator::getIdGenerator()->generate();
- state_storage_ = std::make_unique<minifi::controllers::VolatileMapStateStorage>(core::controller::ControllerServiceMetadata{
+ state_storage_ = std::make_unique<minifi::controllers::VolatileMapStateStorage>(core::ControllerServiceMetadata{
.uuid = storage_uuid,
.name = "KeyValueStateStorage",
.logger = logging::LoggerFactory<minifi::controllers::VolatileMapStateStorage>::getLogger(storage_uuid)
diff --git a/extensions/llamacpp/processors/ExtensionInitializer.cpp b/extensions/llamacpp/processors/ExtensionInitializer.cpp
index b90b8b133..f55afba69 100644
--- a/extensions/llamacpp/processors/ExtensionInitializer.cpp
+++ b/extensions/llamacpp/processors/ExtensionInitializer.cpp
@@ -34,6 +34,8 @@ extern "C" MinifiExtension* MinifiInitExtension(MinifiConfig* /*config*/) {
.user_data = nullptr,
.processors_count = 1,
.processors_ptr = &description,
+ .controller_services_count = 0,
+ .controller_services_ptr = nullptr,
};
extension = MinifiCreateExtension(minifi::api::utils::toStringView(MINIFI_API_VERSION), &ext_create_info);
});
diff --git a/extensions/opencv/OpenCVLoader.cpp b/extensions/opencv/OpenCVLoader.cpp
index 3f91395c4..f159cae8e 100644
--- a/extensions/opencv/OpenCVLoader.cpp
+++ b/extensions/opencv/OpenCVLoader.cpp
@@ -40,7 +40,9 @@ extern "C" MinifiExtension* MinifiInitExtension(MinifiConfig* /*config*/) {
.deinit = nullptr,
.user_data = nullptr,
.processors_count = 0,
- .processors_ptr = nullptr
+ .processors_ptr = nullptr,
+ .controller_services_count = 0,
+ .controller_services_ptr = nullptr,
};
return MinifiCreateExtension(minifi::utils::toStringView(MINIFI_API_VERSION), &ext_create_info);
}
diff --git a/extensions/python/pythonlibloader/PythonLibLoader.cpp b/extensions/python/pythonlibloader/PythonLibLoader.cpp
index cdf0356b7..c2617f186 100644
--- a/extensions/python/pythonlibloader/PythonLibLoader.cpp
+++ b/extensions/python/pythonlibloader/PythonLibLoader.cpp
@@ -112,7 +112,9 @@ extern "C" MinifiExtension* MinifiInitExtension(MinifiConfig* config) {
.deinit = nullptr,
.user_data = nullptr,
.processors_count = 0,
- .processors_ptr = nullptr
+ .processors_ptr = nullptr,
+ .controller_services_count = 0,
+ .controller_services_ptr = nullptr,
};
return MinifiCreateExtension(minifi::utils::toStringView(MINIFI_API_VERSION), &ext_create_info);
}
diff --git a/extensions/python/pythonloader/PyProcLoader.cpp b/extensions/python/pythonloader/PyProcLoader.cpp
index 55ad4ebef..f37041340 100644
--- a/extensions/python/pythonloader/PyProcLoader.cpp
+++ b/extensions/python/pythonloader/PyProcLoader.cpp
@@ -47,7 +47,9 @@ extern "C" MinifiExtension* MinifiInitExtension(MinifiConfig* config) {
.deinit = nullptr,
.user_data = nullptr,
.processors_count = 0,
- .processors_ptr = nullptr
+ .processors_ptr = nullptr,
+ .controller_services_count = 0,
+ .controller_services_ptr = nullptr,
};
return MinifiCreateExtension(minifi::utils::toStringView(MINIFI_API_VERSION), &ext_create_info);
}
diff --git a/extensions/sftp/SFTPLoader.cpp b/extensions/sftp/SFTPLoader.cpp
index bb23e9998..8b8d9bffa 100644
--- a/extensions/sftp/SFTPLoader.cpp
+++ b/extensions/sftp/SFTPLoader.cpp
@@ -42,7 +42,9 @@ extern "C" MinifiExtension* MinifiInitExtension(MinifiConfig* /*config*/) {
},
.user_data = nullptr,
.processors_count = 0,
- .processors_ptr = nullptr
+ .processors_ptr = nullptr,
+ .controller_services_count = 0,
+ .controller_services_ptr = nullptr,
};
return MinifiCreateExtension(minifi::utils::toStringView(MINIFI_API_VERSION), &ext_create_info);
}
diff --git a/libminifi/include/controllers/NetworkPrioritizerService.h b/libminifi/include/controllers/NetworkPrioritizerService.h
index 6e8e8593b..a3fc85cb6 100644
--- a/libminifi/include/controllers/NetworkPrioritizerService.h
+++ b/libminifi/include/controllers/NetworkPrioritizerService.h
@@ -51,7 +51,7 @@ class NetworkPrioritizerService : public core::controller::ControllerServiceBase
public:
using ControllerServiceBase::ControllerServiceBase;
- explicit NetworkPrioritizerService(core::controller::ControllerServiceMetadata metadata,
+ explicit NetworkPrioritizerService(core::ControllerServiceMetadata metadata,
std::shared_ptr<utils::timeutils::Clock> clock)
: ControllerServiceBase(std::move(metadata)),
clock_(std::move(clock)) {
diff --git a/libminifi/include/core/controller/ControllerService.h b/libminifi/include/core/controller/ControllerService.h
index c80830db7..0dff3a884 100644
--- a/libminifi/include/core/controller/ControllerService.h
+++ b/libminifi/include/core/controller/ControllerService.h
@@ -27,7 +27,7 @@
#include "core/ConfigurableComponentImpl.h"
#include "core/Connectable.h"
#include "minifi-cpp/core/controller/ControllerServiceApi.h"
-#include "minifi-cpp/core/ControllerServiceApiDefinition.h"
+#include "minifi-cpp/core/ControllerServiceTypeDefinition.h"
namespace org::apache::nifi::minifi::core::controller {
@@ -44,6 +44,7 @@ class ControllerService : public ConfigurableComponentImpl, public CoreComponent
public:
explicit ControllerServiceDescriptorImpl(ControllerService& impl): impl_(impl) {}
void setSupportedProperties(std::span<const PropertyReference> properties) override;
+ void setSupportedProperties(std::span<const Property> properties) override;
private:
ControllerService& impl_;
@@ -130,7 +131,7 @@ class ControllerService : public ConfigurableComponentImpl, public CoreComponent
return false;
}
- static constexpr auto ImplementsApis = std::array<ControllerServiceApiDefinition, 0>{};
+ static constexpr auto ImplementsApis = std::array<ControllerServiceTypeDefinition, 0>{};
protected:
std::unique_ptr<ControllerServiceApi> impl_;
@@ -146,6 +147,10 @@ inline void ControllerService::ControllerServiceDescriptorImpl::setSupportedProp
impl_.setSupportedProperties(properties);
}
+inline void ControllerService::ControllerServiceDescriptorImpl::setSupportedProperties(std::span<const Property> properties) {
+ impl_.setSupportedProperties(properties);
+}
+
inline nonstd::expected<std::string, std::error_code> ControllerService::ControllerServiceContextImpl::getProperty(std::string_view name) const {
return impl_.getProperty(name);
}
diff --git a/libminifi/include/utils/CControllerService.h b/libminifi/include/utils/CControllerService.h
new file mode 100644
index 000000000..0d0ea4929
--- /dev/null
+++ b/libminifi/include/utils/CControllerService.h
@@ -0,0 +1,110 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "minifi-c/minifi-c.h"
+#include "minifi-cpp/Exception.h"
+#include "minifi-cpp/core/ControllerServiceMetadata.h"
+#include "minifi-cpp/core/Property.h"
+#include "minifi-cpp/core/controller/ControllerServiceApi.h"
+
+namespace org::apache::nifi::minifi::utils {
+class CControllerService;
+
+
+struct CControllerServiceClassDescription {
+ std::string group_name;
+ std::string name;
+ std::string version;
+
+ std::vector<core::Property> class_properties;
+
+ MinifiControllerServiceCallbacks callbacks;
+};
+
+class CControllerService final : public core::controller::ControllerServiceApi, public core::controller::ControllerServiceInterface {
+ public:
+ CControllerService(CControllerServiceClassDescription class_description, core::ControllerServiceMetadata metadata)
+ : class_description_(std::move(class_description)),
+ metadata_(std::move(metadata)) {
+ MinifiControllerServiceMetadata c_metadata;
+ auto uuid_str = metadata_.uuid.to_string();
+ c_metadata.uuid = MinifiStringView{.data = uuid_str.data(), .length = uuid_str.length()};
+ c_metadata.name = MinifiStringView{.data = metadata_.name.data(), .length = metadata_.name.length()};
+ c_metadata.logger = reinterpret_cast<MinifiLogger*>(&metadata_.logger);
+ impl_ = class_description_.callbacks.create(c_metadata);
+ }
+ CControllerService(CControllerServiceClassDescription class_description, core::ControllerServiceMetadata metadata, gsl::owner<void*> impl)
+ : class_description_(std::move(class_description)),
+ impl_(impl),
+ metadata_(std::move(metadata)) {}
+ ~CControllerService() override {
+ class_description_.callbacks.destroy(impl_);
+ }
+
+ void initialize(core::controller::ControllerServiceDescriptor& descriptor) override {
+ descriptor.setSupportedProperties(std::span(class_description_.class_properties));
+ }
+
+ void onEnable(core::controller::ControllerServiceContext& controller_service_context,
+ const std::shared_ptr<Configure>&,
+ const std::vector<std::shared_ptr<core::controller::ControllerServiceInterface>>&) override {
+ const auto enable_status = class_description_.callbacks.enable(impl_, reinterpret_cast<MinifiControllerServiceContext*>(&controller_service_context));
+ if (enable_status != MINIFI_STATUS_SUCCESS) {
+ throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Could not enable controller service");
+ }
+ }
+
+ void notifyStop() override {
+ class_description_.callbacks.notifyStop(impl_);
+ }
+
+ ControllerServiceInterface* getControllerServiceInterface() override {
+ return this;
+ }
+
+ [[nodiscard]] std::string getName() const {
+ return metadata_.name;
+ }
+
+ [[nodiscard]] Identifier getUUID() const {
+ return metadata_.uuid;
+ }
+
+ [[nodiscard]] void* getImpl() const {
+ return this->impl_;
+ }
+
+ [[nodiscard]] const CControllerServiceClassDescription& getClassDescription() const {
+ return class_description_;
+ }
+
+ private:
+ CControllerServiceClassDescription class_description_;
+ gsl::owner<void*> impl_;
+ minifi::core::ControllerServiceMetadata metadata_;
+};
+
+void useCControllerServiceClassDescription(const MinifiControllerServiceClassDefinition& class_description,
+ const BundleIdentifier& bundle_id,
+ const std::function<void(ClassDescription, CControllerServiceClassDescription)>& fn);
+
+} // namespace org::apache::nifi::minifi::utils
diff --git a/libminifi/include/utils/CProcessor.h b/libminifi/include/utils/CProcessor.h
index bef73d9e9..6b8e6660e 100644
--- a/libminifi/include/utils/CProcessor.h
+++ b/libminifi/include/utils/CProcessor.h
@@ -18,6 +18,7 @@
#pragma once
#include <string>
+#include <utility>
#include "minifi-cpp/core/Annotation.h"
#include "core/ProcessorMetrics.h"
@@ -63,18 +64,18 @@ class CProcessor : public minifi::core::ProcessorApi {
CProcessor(CProcessorClassDescription class_description, minifi::core::ProcessorMetadata metadata)
: class_description_(std::move(class_description)),
metrics_extension_(std::make_shared<CProcessorMetricsWrapper>(*this)) {
- metadata_ = metadata;
+ metadata_ = std::move(metadata);
MinifiProcessorMetadata c_metadata;
- auto uuid_str = metadata.uuid.to_string();
+ const auto uuid_str = metadata.uuid.to_string();
c_metadata.uuid = MinifiStringView{.data = uuid_str.data(), .length = uuid_str.length()};
- c_metadata.name = MinifiStringView{.data = metadata.name.data(), .length = metadata.name.length()};
+ c_metadata.name = MinifiStringView{.data = metadata_.name.data(), .length = metadata_.name.length()};
c_metadata.logger = reinterpret_cast<MinifiLogger*>(&metadata_.logger);
impl_ = class_description_.callbacks.create(c_metadata);
}
CProcessor(CProcessorClassDescription class_description, minifi::core::ProcessorMetadata metadata, gsl::owner<void*> impl)
: class_description_(std::move(class_description)),
impl_(impl),
- metadata_(metadata),
+ metadata_(std::move(metadata)),
metrics_extension_(std::make_shared<CProcessorMetricsWrapper>(*this)) {}
~CProcessor() override {
class_description_.callbacks.destroy(impl_);
diff --git a/libminifi/src/controllers/SSLContextService.cpp b/libminifi/src/controllers/SSLContextService.cpp
index 584b5109d..5d7cfc72e 100644
--- a/libminifi/src/controllers/SSLContextService.cpp
+++ b/libminifi/src/controllers/SSLContextService.cpp
@@ -598,7 +598,7 @@ void SSLContextService::verifyCertificateExpiration() {
std::shared_ptr<SSLContextService> SSLContextService::createAndEnable(std::string_view name, const std::shared_ptr<Configure>& configuration) {
auto uuid = utils::IdGenerator::getIdGenerator()->generate();
auto logger = core::logging::LoggerFactory<SSLContextService>::getLogger(uuid);
- auto service = std::make_shared<core::controller::ControllerService>(name, uuid, std::make_unique<SSLContextService>(core::controller::ControllerServiceMetadata{
+ auto service = std::make_shared<core::controller::ControllerService>(name, uuid, std::make_unique<SSLContextService>(core::ControllerServiceMetadata{
.uuid = uuid,
.name = std::string{name},
.logger = logger,
diff --git a/libminifi/src/minifi-c.cpp b/libminifi/src/minifi-c.cpp
index 6a2d947ab..9c85a26bb 100644
--- a/libminifi/src/minifi-c.cpp
+++ b/libminifi/src/minifi-c.cpp
@@ -37,11 +37,16 @@
#include "core/extension/ExtensionManager.h"
#include "utils/PropertyErrors.h"
#include "utils/CProcessor.h"
+#include "utils/CControllerService.h"
namespace minifi = org::apache::nifi::minifi;
namespace {
+MinifiStringView minifiStringView(const std::string_view s) {
+ return MinifiStringView{.data = s.data(), .length = s.size()};
+}
+
std::string toString(MinifiStringView sv) {
return {sv.data, sv.length};
}
@@ -146,6 +151,37 @@ class CProcessorFactory : public minifi::core::ProcessorFactory {
minifi::utils::CProcessorClassDescription class_description_;
};
+class CControllerServiceFactory : public minifi::core::controller::ControllerServiceFactory {
+ public:
+ CControllerServiceFactory(std::string group_name, std::string class_name, minifi::utils::CControllerServiceClassDescription class_description)
+ : group_name_(std::move(group_name)),
+ class_name_(std::move(class_name)),
+ class_description_(std::move(class_description)) {}
+ std::unique_ptr<minifi::core::controller::ControllerServiceApi> create(minifi::core::ControllerServiceMetadata metadata) override {
+ return std::make_unique<minifi::utils::CControllerService>(class_description_, std::move(metadata));
+ }
+
+ [[nodiscard]] std::string getGroupName() const override {
+ return group_name_;
+ }
+
+ [[nodiscard]] std::string getClassName() const override {
+ return class_name_;
+ }
+
+ CControllerServiceFactory() = delete;
+ CControllerServiceFactory(const CControllerServiceFactory&) = delete;
+ CControllerServiceFactory& operator=(const CControllerServiceFactory&) = delete;
+ CControllerServiceFactory(CControllerServiceFactory&&) = delete;
+ CControllerServiceFactory& operator=(CControllerServiceFactory&&) = delete;
+ ~CControllerServiceFactory() override = default;
+
+ private:
+ std::string group_name_;
+ std::string class_name_;
+ minifi::utils::CControllerServiceClassDescription class_description_;
+};
+
MinifiExtension* MinifiCreateExtensionImpl(MinifiStringView /*api_version*/, const MinifiExtensionCreateInfo* extension_create_info) {
gsl_Assert(extension_create_info);
auto extension_name = toString(extension_create_info->name);
@@ -162,6 +198,14 @@ MinifiExtension* MinifiCreateExtensionImpl(MinifiStringView /*api_version*/, con
bundle_components.processors.emplace_back(description);
});
}
+ for (size_t controller_service_idx = 0; controller_service_idx < extension_create_info->controller_services_count; ++controller_service_idx) {
+ minifi::utils::useCControllerServiceClassDescription(extension_create_info->controller_services_ptr[controller_service_idx], bundle, [&](const auto& description, const auto& c_class_description) {
+ minifi::core::ClassLoader::getDefaultClassLoader().getClassLoader(extension_name).registerClass(
+ c_class_description.name,
+ std::make_unique<CControllerServiceFactory>(extension_name, toString(extension_create_info->controller_services_ptr[controller_service_idx].full_name), c_class_description));
+ bundle_components.controller_services.emplace_back(description);
+ });
+ }
return reinterpret_cast<MinifiExtension*>(new org::apache::nifi::minifi::core::extension::Extension::Info{
.name = toString(extension_create_info->name),
.version = toString(extension_create_info->version),
@@ -245,6 +289,35 @@ void useCProcessorClassDescription(const MinifiProcessorClassDefinition& class_d
fn(description, c_class_description);
}
+void useCControllerServiceClassDescription(const MinifiControllerServiceClassDefinition& class_description,
+ const BundleIdentifier& bundle_id,
+ const std::function<void(ClassDescription, CControllerServiceClassDescription)>& fn) {
+ std::vector<minifi::core::Property> properties;
+ properties.reserve(class_description.class_properties_count);
+ for (size_t i = 0; i < class_description.class_properties_count; ++i) {
+ properties.push_back(createProperty(&class_description.class_properties_ptr[i]));
+ }
+
+ auto name_segments = minifi::utils::string::split(toStringView(class_description.full_name), "::");
+ gsl_Assert(!name_segments.empty());
+
+ minifi::ClassDescription description{
+ .type_ = minifi::ResourceType::ControllerService,
+ .short_name_ = name_segments.back(),
+ .full_name_ = minifi::utils::string::join(".", name_segments),
+ .description_ = toString(class_description.description),
+ .class_properties_ = properties,
+ };
+
+ minifi::utils::CControllerServiceClassDescription c_class_description{
+ .group_name = bundle_id.name,
+ .name = name_segments.back(),
+ .version = bundle_id.version,
+ .class_properties = properties,
+ .callbacks = class_description.callbacks
+ };
+ fn(description, c_class_description);
+}
} // namespace org::apache::nifi::minifi::utils
extern "C" {
@@ -266,7 +339,7 @@ MinifiStatus MinifiProcessContextGetProperty(MinifiProcessContext* context, Mini
auto result = reinterpret_cast<minifi::core::ProcessContext*>(context)->getProperty(toStringView(property_name),
flowfile != MINIFI_NULL ? reinterpret_cast<std::shared_ptr<minifi::core::FlowFile>*>(flowfile)->get() : nullptr);
if (result) {
- result_cb(user_ctx, MinifiStringView{.data = result.value().data(), .length = result.value().length()});
+ result_cb(user_ctx, minifiStringView(result.value()));
return MINIFI_STATUS_SUCCESS;
}
switch (static_cast<minifi::core::PropertyErrorCode>(result.error().value())) {
@@ -281,7 +354,7 @@ MinifiStatus MinifiProcessContextGetProperty(MinifiProcessContext* context, Mini
void MinifiProcessContextGetProcessorName(MinifiProcessContext* context, void(*cb)(void* user_ctx, MinifiStringView result), void* user_ctx) {
gsl_Assert(context != MINIFI_NULL);
auto name = reinterpret_cast<minifi::core::ProcessContext*>(context)->getProcessorInfo().getName();
- cb(user_ctx, MinifiStringView{.data = name.data(), .length = name.length()});
+ cb(user_ctx, minifiStringView(name));
}
MinifiBool MinifiProcessContextHasNonEmptyProperty(MinifiProcessContext* context, MinifiStringView property_name) {
@@ -293,10 +366,7 @@ void MinifiConfigGet(MinifiConfig* configure, MinifiStringView key, void(*cb)(vo
gsl_Assert(configure != MINIFI_NULL);
auto value = reinterpret_cast<minifi::Configure*>(configure)->get(toString(key));
if (value) {
- cb(user_ctx, MinifiStringView{
- .data = value->data(),
- .length = value->length()
- });
+ cb(user_ctx, minifiStringView(*value));
}
}
@@ -420,7 +490,7 @@ void MinifiStatusToString(MinifiStatus status, void(*cb)(void* user_ctx, MinifiS
default: return "Unknown error";
}
}();
- cb(user_ctx, MinifiStringView{.data = message.data(), .length = message.size()});
+ cb(user_ctx, minifiStringView(message));
}
void MinifiFlowFileSetAttribute(MinifiProcessSession* session, MinifiFlowFile* flowfile, MinifiStringView attribute_name, const MinifiStringView* attribute_value) {
@@ -442,7 +512,7 @@ MinifiBool MinifiFlowFileGetAttribute(MinifiProcessSession* session, MinifiFlowF
if (!value.has_value()) {
return false;
}
- cb(user_ctx, MinifiStringView{.data = value->data(), .length = value->size()});
+ cb(user_ctx, minifiStringView(*value));
return true;
}
@@ -451,8 +521,58 @@ void MinifiFlowFileGetAttributes(MinifiProcessSession* session, MinifiFlowFile*
gsl_Assert(session != MINIFI_NULL);
gsl_Assert(flowfile != MINIFI_NULL);
for (auto& [key, value] : (*reinterpret_cast<std::shared_ptr<minifi::core::FlowFile>*>(flowfile))->getAttributes()) {
- cb(user_ctx, MinifiStringView{.data = key.data(), .length = key.size()}, MinifiStringView{.data = value.data(), .length = value.size()});
+ cb(user_ctx, minifiStringView(key), minifiStringView(value));
}
}
+MinifiStatus MinifiControllerServiceContextGetProperty(MinifiControllerServiceContext* context, MinifiStringView property_name,
+ void (*result_cb)(void* user_ctx, MinifiStringView result), void* user_ctx) {
+ gsl_Assert(context != MINIFI_NULL);
+ auto result = reinterpret_cast<minifi::core::controller::ControllerServiceContext*>(context)->getProperty(toStringView(property_name));
+ if (result) {
+ result_cb(user_ctx, minifiStringView(result.value()));
+ return MINIFI_STATUS_SUCCESS;
+ }
+ switch (static_cast<minifi::core::PropertyErrorCode>(result.error().value())) {
+ case minifi::core::PropertyErrorCode::NotSupportedProperty: return MINIFI_STATUS_NOT_SUPPORTED_PROPERTY;
+ case minifi::core::PropertyErrorCode::DynamicPropertiesNotSupported: return MINIFI_STATUS_DYNAMIC_PROPERTIES_NOT_SUPPORTED;
+ case minifi::core::PropertyErrorCode::PropertyNotSet: return MINIFI_STATUS_PROPERTY_NOT_SET;
+ case minifi::core::PropertyErrorCode::ValidationFailed: return MINIFI_STATUS_VALIDATION_FAILED;
+ default: return MINIFI_STATUS_UNKNOWN_ERROR;
+ }
+}
+
+
+MinifiStatus MinifiProcessContextGetControllerService(
+ MinifiProcessContext* process_context,
+ MinifiStringView controller_service_name,
+ void(*cb)(
+ void* user_ctx,
+ void* service,
+ MinifiStringView type,
+ MinifiStringView group,
+ MinifiStringView version),
+ void* user_ctx) {
+
+ gsl_Assert(process_context != MINIFI_NULL);
+ const auto context = reinterpret_cast<minifi::core::ProcessContext*>(process_context);
+ const auto name_str = std::string{toStringView(controller_service_name)};
+ const auto service_shared_ptr = context->getControllerService(name_str, context->getProcessorInfo().getUUID());
+ if (!service_shared_ptr) {
+ return MINIFI_STATUS_UNKNOWN_ERROR;
+ }
+ const minifi::utils::CControllerService* c_controller_service = dynamic_cast<minifi::utils::CControllerService*>(&*service_shared_ptr);
+ if (c_controller_service) {
+ const auto class_description = c_controller_service->getClassDescription();
+ cb(user_ctx,
+ c_controller_service->getImpl(),
+ minifiStringView(class_description.group_name),
+ minifiStringView(class_description.name),
+ minifiStringView(class_description.version));
+ return MINIFI_STATUS_SUCCESS;
+ }
+ return MINIFI_STATUS_UNKNOWN_ERROR; // TODO(mzink) better error handling
+}
+
+
} // extern "C"
diff --git a/libminifi/src/properties/Properties.cpp b/libminifi/src/properties/Properties.cpp
index 1624a8557..2f5723a64 100644
--- a/libminifi/src/properties/Properties.cpp
+++ b/libminifi/src/properties/Properties.cpp
@@ -22,6 +22,7 @@
#include <string>
#include "core/logging/LoggerConfiguration.h"
+#include "minifi-cpp/core/PropertyValidator.h"
#include "properties/Configuration.h"
#include "properties/PropertiesFile.h"
#include "utils/StringUtils.h"
diff --git a/libminifi/test/libtest/integration/HTTPHandlers.h b/libminifi/test/libtest/integration/HTTPHandlers.h
index 25a0dba5d..40304948a 100644
--- a/libminifi/test/libtest/integration/HTTPHandlers.h
+++ b/libminifi/test/libtest/integration/HTTPHandlers.h
@@ -21,21 +21,22 @@
#include <map>
#include <memory>
#include <optional>
+#include <set>
#include <string>
+#include <unordered_map>
#include <utility>
#include <vector>
-#include <set>
-#include <unordered_map>
-#include "civetweb.h"
#include "CivetServer.h"
-#include "concurrentqueue.h"
-#include "rapidjson/document.h"
-#include "utils/HTTPUtils.h"
#include "ServerAwareHandler.h"
#include "c2/C2Payload.h"
+#include "civetweb.h"
+#include "concurrentqueue.h"
+#include "minifi-cpp/properties/Configure.h"
#include "properties/Configure.h"
+#include "rapidjson/document.h"
#include "unit/Catch.h"
+#include "utils/HTTPUtils.h"
namespace org::apache::nifi::minifi::test {
diff --git a/libminifi/test/libtest/unit/ControllerServiceUtils.h b/libminifi/test/libtest/unit/ControllerServiceUtils.h
index ff19982d3..ee8d79c37 100644
--- a/libminifi/test/libtest/unit/ControllerServiceUtils.h
+++ b/libminifi/test/libtest/unit/ControllerServiceUtils.h
@@ -29,7 +29,7 @@ std::unique_ptr<core::controller::ControllerService> make_controller_service(std
if (!uuid) {
uuid = minifi::utils::IdGenerator::getIdGenerator()->generate();
}
- auto processor_impl = std::make_unique<T>(core::controller::ControllerServiceMetadata{
+ auto processor_impl = std::make_unique<T>(core::ControllerServiceMetadata{
.uuid = uuid.value(),
.name = std::string{name},
.logger = minifi::core::logging::LoggerFactory<T>::getLogger(uuid.value())
diff --git a/libminifi/test/unit/NetworkPrioritizerServiceTests.cpp b/libminifi/test/unit/NetworkPrioritizerServiceTests.cpp
index a54ba109a..a05d21ad2 100644
--- a/libminifi/test/unit/NetworkPrioritizerServiceTests.cpp
+++ b/libminifi/test/unit/NetworkPrioritizerServiceTests.cpp
@@ -31,7 +31,7 @@ std::shared_ptr<core::controller::ControllerService> createNetworkPrioritizerSer
const std::shared_ptr<utils::timeutils::Clock>& clock = std::make_shared<minifi::test::utils::ManualClock>()) {
return std::make_shared<core::controller::ControllerService>(
name, utils::Identifier{},
- std::make_unique<minifi::controllers::NetworkPrioritizerService>(core::controller::ControllerServiceMetadata{
+ std::make_unique<minifi::controllers::NetworkPrioritizerService>(core::ControllerServiceMetadata{
.uuid = utils::Identifier{},
.name = name,
.logger = logging::LoggerFactory<minifi::controllers::NetworkPrioritizerService>::getLogger()
diff --git a/minifi-api/include/minifi-cpp/core/controller/ControllerServiceMetadata.h b/minifi-api/common/include/minifi-cpp/core/ControllerServiceMetadata.h
similarity index 84%
rename from minifi-api/include/minifi-cpp/core/controller/ControllerServiceMetadata.h
rename to minifi-api/common/include/minifi-cpp/core/ControllerServiceMetadata.h
index 3c6fb63b7..22200a84d 100644
--- a/minifi-api/include/minifi-cpp/core/controller/ControllerServiceMetadata.h
+++ b/minifi-api/common/include/minifi-cpp/core/ControllerServiceMetadata.h
@@ -1,5 +1,5 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
+* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
@@ -21,7 +21,7 @@
#include "minifi-cpp/utils/Id.h"
#include "minifi-cpp/core/logging/Logger.h"
-namespace org::apache::nifi::minifi::core::controller {
+namespace org::apache::nifi::minifi::core {
struct ControllerServiceMetadata {
utils::Identifier uuid;
@@ -29,4 +29,4 @@ struct ControllerServiceMetadata {
std::shared_ptr<logging::Logger> logger;
};
-} // namespace org::apache::nifi::minifi::core::controller
+} // namespace org::apache::nifi::minifi::core
diff --git a/minifi-api/include/minifi-c/minifi-c.h b/minifi-api/include/minifi-c/minifi-c.h
index 97af61f5b..811bd73ec 100644
--- a/minifi-api/include/minifi-c/minifi-c.h
+++ b/minifi-api/include/minifi-c/minifi-c.h
@@ -96,6 +96,8 @@ typedef struct MinifiExtension MinifiExtension;
typedef struct MinifiPublishedMetrics MinifiPublishedMetrics;
typedef struct MinifiApiVersion MinifiApiVersion;
+typedef struct MinifiControllerServiceContext MinifiControllerServiceContext;
+
typedef enum MinifiStatus {
MINIFI_STATUS_SUCCESS = 0,
MINIFI_STATUS_UNKNOWN_ERROR = 1,
@@ -149,6 +151,12 @@ typedef struct MinifiProcessorMetadata {
MinifiLogger* logger; // borrowed reference, live until the processor is live
} MinifiProcessorMetadata;
+typedef struct MinifiControllerServiceMetadata {
+ MinifiStringView uuid;
+ MinifiStringView name;
+ MinifiLogger* logger; // borrowed reference, live until the controller service is live
+} MinifiControllerServiceMetadata;
+
typedef struct MinifiProcessorCallbacks {
MINIFI_OWNED void*(*create)(MinifiProcessorMetadata);
void(*destroy)(MINIFI_OWNED void*);
@@ -161,6 +169,13 @@ typedef struct MinifiProcessorCallbacks {
MINIFI_OWNED MinifiPublishedMetrics*(*calculateMetrics)(void*);
} MinifiProcessorCallbacks;
+typedef struct MinifiControllerServiceCallbacks {
+ MINIFI_OWNED void*(*create)(MinifiControllerServiceMetadata);
+ void(*destroy)(MINIFI_OWNED void*);
+ MinifiStatus(*enable)(void*, MinifiControllerServiceContext*);
+ void(*notifyStop)(void*);
+} MinifiControllerServiceCallbacks;
+
typedef struct MinifiProcessorClassDefinition {
MinifiStringView full_name; // '::'-delimited fully qualified name e.g. 'org::apache::nifi::minifi::GenerateFlowFile'
MinifiStringView description;
@@ -180,6 +195,16 @@ typedef struct MinifiProcessorClassDefinition {
MinifiProcessorCallbacks callbacks;
} MinifiProcessorClassDefinition;
+typedef struct MinifiControllerServiceClassDefinition {
+ MinifiStringView full_name; // '::'-delimited fully qualified name e.g. 'org::apache::nifi::minifi::extensions::gcp::GCPCredentialsControllerService
+ MinifiStringView description;
+ size_t class_properties_count;
+ const MinifiPropertyDefinition* class_properties_ptr;
+
+ MinifiControllerServiceCallbacks callbacks;
+} MinifiControllerServiceClassDefinition;
+
+
typedef struct MinifiExtensionCreateInfo {
MinifiStringView name;
MinifiStringView version;
@@ -187,6 +212,8 @@ typedef struct MinifiExtensionCreateInfo {
void* user_data;
size_t processors_count;
const MinifiProcessorClassDefinition* processors_ptr;
+ size_t controller_services_count;
+ const MinifiControllerServiceClassDefinition* controller_services_ptr;
} MinifiExtensionCreateInfo;
// api_version is used to provide backwards compatible changes to the MinifiExtensionCreateInfo structure,
@@ -201,6 +228,9 @@ MinifiStatus MinifiProcessContextGetProperty(MinifiProcessContext* context, Mini
void MinifiProcessContextGetProcessorName(MinifiProcessContext* context, void(*cb)(void* user_ctx, MinifiStringView processor_name), void* user_ctx);
MinifiBool MinifiProcessContextHasNonEmptyProperty(MinifiProcessContext* context, MinifiStringView property_name);
+MinifiStatus MinifiProcessContextGetControllerService(MinifiProcessContext*, MinifiStringView controller_service_name,
+ void(*cb)(void* user_ctx, void* service, MinifiStringView type, MinifiStringView group, MinifiStringView version), void* user_ctx);
+
void MinifiLoggerSetMaxLogSize(MinifiLogger*, int32_t);
void MinifiLoggerLogString(MinifiLogger*, MinifiLogLevel, MinifiStringView);
MinifiBool MinifiLoggerShouldLog(MinifiLogger*, MinifiLogLevel);
@@ -229,6 +259,11 @@ MinifiBool MinifiFlowFileGetAttribute(MinifiProcessSession* session, MinifiFlowF
void(*cb)(void* user_ctx, MinifiStringView attribute_value), void* user_ctx);
void MinifiFlowFileGetAttributes(MinifiProcessSession* session, MinifiFlowFile* flowfile, void(*cb)(void* user_ctx, MinifiStringView attribute_name, MinifiStringView attribute_value), void* user_ctx);
+MinifiStatus MinifiControllerServiceContextGetProperty(MinifiControllerServiceContext* context,
+ MinifiStringView property_name,
+ void(*cb)(void* user_ctx, MinifiStringView property_value),
+ void* user_ctx);
+
#ifdef __cplusplus
} // extern "C"
#endif // __cplusplus
diff --git a/minifi-api/include/minifi-cpp/agent/agent_docs.h b/minifi-api/include/minifi-cpp/agent/agent_docs.h
index 912377d39..b6a6b9017 100644
--- a/minifi-api/include/minifi-cpp/agent/agent_docs.h
+++ b/minifi-api/include/minifi-cpp/agent/agent_docs.h
@@ -22,7 +22,7 @@
#include <vector>
#include "minifi-cpp/core/Annotation.h"
-#include "minifi-cpp/core/ControllerServiceApi.h"
+#include "minifi-cpp/core/ControllerServiceType.h"
#include "minifi-cpp/core/DynamicProperty.h"
#include "minifi-cpp/core/OutputAttribute.h"
#include "minifi-cpp/core/Property.h"
@@ -44,7 +44,7 @@ struct ClassDescription {
std::vector<core::DynamicProperty> dynamic_properties_{};
std::vector<core::Relationship> class_relationships_{};
std::vector<core::OutputAttribute> output_attributes_{};
- std::vector<core::ControllerServiceApi> api_implementations{};
+ std::vector<core::ControllerServiceType> api_implementations{};
bool supports_dynamic_properties_ = false;
bool supports_dynamic_relationships_ = false;
std::string inputRequirement_{};
diff --git a/minifi-api/include/minifi-cpp/controllers/RecordSetReader.h b/minifi-api/include/minifi-cpp/controllers/RecordSetReader.h
index f91dee361..2b8b8731d 100644
--- a/minifi-api/include/minifi-cpp/controllers/RecordSetReader.h
+++ b/minifi-api/include/minifi-cpp/controllers/RecordSetReader.h
@@ -17,7 +17,7 @@
#pragma once
#include "minifi-cpp/core/controller/ControllerServiceInterface.h"
-#include "minifi-cpp/core/ControllerServiceApiDefinition.h"
+#include "minifi-cpp/core/ControllerServiceTypeDefinition.h"
#include "minifi-cpp/core/Record.h"
#include "utils/Enum.h"
#include "utils/ProcessorConfigUtils.h"
@@ -28,7 +28,7 @@ namespace org::apache::nifi::minifi::core {
class RecordSetReader : public controller::ControllerServiceInterface {
public:
- static constexpr auto ProvidesApi = core::ControllerServiceApiDefinition{
+ static constexpr auto ProvidesApi = core::ControllerServiceTypeDefinition{
.artifact = "minifi-system",
.group = "org.apache.nifi.minifi",
.type = "org.apache.nifi.minifi.core.RecordSetReader",
diff --git a/minifi-api/include/minifi-cpp/controllers/RecordSetWriter.h b/minifi-api/include/minifi-cpp/controllers/RecordSetWriter.h
index bbe1ebdde..8453f2215 100644
--- a/minifi-api/include/minifi-cpp/controllers/RecordSetWriter.h
+++ b/minifi-api/include/minifi-cpp/controllers/RecordSetWriter.h
@@ -18,7 +18,7 @@
#include "minifi-cpp/core/controller/ControllerServiceInterface.h"
-#include "minifi-cpp/core/ControllerServiceApiDefinition.h"
+#include "minifi-cpp/core/ControllerServiceTypeDefinition.h"
#include "minifi-cpp/core/FlowFile.h"
#include "minifi-cpp/core/ProcessSession.h"
#include "minifi-cpp/core/Record.h"
@@ -28,7 +28,7 @@ namespace org::apache::nifi::minifi::core {
class RecordSetWriter : public controller::ControllerServiceInterface {
public:
- static constexpr auto ProvidesApi = core::ControllerServiceApiDefinition{
+ static constexpr auto ProvidesApi = core::ControllerServiceTypeDefinition{
.artifact = "minifi-system",
.group = "org.apache.nifi.minifi",
.type = "org.apache.nifi.minifi.core.RecordSetWriter",
diff --git a/minifi-api/include/minifi-cpp/controllers/SSLContextServiceInterface.h b/minifi-api/include/minifi-cpp/controllers/SSLContextServiceInterface.h
index 68d79380b..3058ade0b 100644
--- a/minifi-api/include/minifi-cpp/controllers/SSLContextServiceInterface.h
+++ b/minifi-api/include/minifi-cpp/controllers/SSLContextServiceInterface.h
@@ -16,12 +16,12 @@
*/
#pragma once
-#include <iostream>
+#include <filesystem>
#include <memory>
#include <string>
#include <utility>
-#include "minifi-cpp/core/ControllerServiceApiDefinition.h"
+#include "minifi-cpp/core/ControllerServiceTypeDefinition.h"
#include "minifi-cpp/agent/agent_version.h"
#include "minifi-cpp/core/controller/ControllerServiceInterface.h"
@@ -37,7 +37,7 @@ namespace org::apache::nifi::minifi::controllers {
*/
class SSLContextServiceInterface : public core::controller::ControllerServiceInterface {
public:
- static constexpr auto ProvidesApi = core::ControllerServiceApiDefinition{
+ static constexpr auto ProvidesApi = core::ControllerServiceTypeDefinition{
.artifact = "minifi-system",
.group = "org.apache.nifi.minifi",
.type = "org.apache.nifi.minifi.controllers.SSLContextServiceInterface",
diff --git a/minifi-api/include/minifi-cpp/core/ControllerServiceApi.h b/minifi-api/include/minifi-cpp/core/ControllerServiceType.h
similarity index 84%
rename from minifi-api/include/minifi-cpp/core/ControllerServiceApi.h
rename to minifi-api/include/minifi-cpp/core/ControllerServiceType.h
index fe0d450f7..6d5b39de3 100644
--- a/minifi-api/include/minifi-cpp/core/ControllerServiceApi.h
+++ b/minifi-api/include/minifi-cpp/core/ControllerServiceType.h
@@ -17,16 +17,16 @@
#pragma once
#include <string>
-#include "minifi-cpp/core/ControllerServiceApiDefinition.h"
+#include "minifi-cpp/core/ControllerServiceTypeDefinition.h"
namespace org::apache::nifi::minifi::core {
-struct ControllerServiceApi {
+struct ControllerServiceType {
std::string artifact;
std::string group;
std::string type;
- ControllerServiceApi(const ControllerServiceApiDefinition& definition) // NOLINT(runtime/explicit)
+ ControllerServiceType(const ControllerServiceTypeDefinition& definition) // NOLINT(runtime/explicit)
: artifact(definition.artifact),
group(definition.group),
type(definition.type) {}
diff --git a/minifi-api/include/minifi-cpp/core/ControllerServiceApiDefinition.h b/minifi-api/include/minifi-cpp/core/ControllerServiceTypeDefinition.h
similarity index 96%
rename from minifi-api/include/minifi-cpp/core/ControllerServiceApiDefinition.h
rename to minifi-api/include/minifi-cpp/core/ControllerServiceTypeDefinition.h
index 37084baea..f40d871a6 100644
--- a/minifi-api/include/minifi-cpp/core/ControllerServiceApiDefinition.h
+++ b/minifi-api/include/minifi-cpp/core/ControllerServiceTypeDefinition.h
@@ -20,7 +20,7 @@
namespace org::apache::nifi::minifi::core {
-struct ControllerServiceApiDefinition {
+struct ControllerServiceTypeDefinition {
std::string_view artifact;
std::string_view group;
std::string_view type;
diff --git a/minifi-api/include/minifi-cpp/core/controller/ControllerServiceDescriptor.h b/minifi-api/include/minifi-cpp/core/controller/ControllerServiceDescriptor.h
index 525a6d6ce..39fb6da89 100644
--- a/minifi-api/include/minifi-cpp/core/controller/ControllerServiceDescriptor.h
+++ b/minifi-api/include/minifi-cpp/core/controller/ControllerServiceDescriptor.h
@@ -26,6 +26,7 @@ class ControllerServiceDescriptor {
virtual ~ControllerServiceDescriptor() = default;
virtual void setSupportedProperties(std::span<const PropertyReference> properties) = 0;
+ virtual void setSupportedProperties(std::span<const Property> properties) = 0;
};
} // namespace org::apache::nifi::minifi::core::controller
diff --git a/minifi-api/include/minifi-cpp/core/controller/ControllerServiceFactory.h b/minifi-api/include/minifi-cpp/core/controller/ControllerServiceFactory.h
index 725944bfa..aafc571dc 100644
--- a/minifi-api/include/minifi-cpp/core/controller/ControllerServiceFactory.h
+++ b/minifi-api/include/minifi-cpp/core/controller/ControllerServiceFactory.h
@@ -17,10 +17,11 @@
#pragma once
-#include <string>
#include <memory>
+#include <string>
#include <utility>
-#include "minifi-cpp/core/controller/ControllerServiceMetadata.h"
+
+#include "minifi-cpp/core/ControllerServiceMetadata.h"
namespace org::apache::nifi::minifi::core::controller {
diff --git a/minifi-api/include/minifi-cpp/core/controller/ControllerServiceInterface.h b/minifi-api/include/minifi-cpp/core/controller/ControllerServiceInterface.h
index e1d7da19e..799446f69 100644
--- a/minifi-api/include/minifi-cpp/core/controller/ControllerServiceInterface.h
+++ b/minifi-api/include/minifi-cpp/core/controller/ControllerServiceInterface.h
@@ -17,23 +17,8 @@
*/
#pragma once
-#include <memory>
-#include <string>
-#include <utility>
-#include <vector>
-
-#include "minifi-cpp/properties/Configure.h"
-#include "minifi-cpp/core/ConfigurableComponent.h"
-#include "minifi-cpp/core/Connectable.h"
-
namespace org::apache::nifi::minifi::core::controller {
-/**
- * Controller Service base class that contains some pure virtual methods.
- *
- * Design: OnEnable is executed when the controller service is being enabled.
- * Note that keeping state here must be protected in this function.
- */
class ControllerServiceInterface {
public:
virtual ~ControllerServiceInterface() = default;
diff --git a/minifi-api/minifi-c-api.def b/minifi-api/minifi-c-api.def
index d7faf9426..de4b384b0 100644
--- a/minifi-api/minifi-c-api.def
+++ b/minifi-api/minifi-c-api.def
@@ -2,9 +2,11 @@ LIBRARY core-minifi.dll
EXPORTS
MinifiCreateExtension_0_1
MinifiPublishedMetricsCreate
+ MinifiProcessContextGetControllerService
MinifiProcessContextGetProperty
MinifiProcessContextGetProcessorName
MinifiProcessContextHasNonEmptyProperty
+ MinifiControllerServiceContextGetProperty
MinifiLoggerSetMaxLogSize
MinifiLoggerLogString
MinifiLoggerShouldLog
(nifi-minifi-cpp) 03/04: MINIFICPP-2711 Docker tests, missing logs if container fails to start
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
martinzink pushed a commit to branch apache-rusty
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 2e3c8f4fd9ad03727c1a5e7af7fcc9e9c96b0a82
Author: Martin Zink <ma...@apache.org>
AuthorDate: Thu Feb 12 16:52:57 2026 +0100
MINIFICPP-2711 Docker tests, missing logs if container fails to start
Closes #2099
Signed-off-by: Martin Zink <ma...@apache.org>
---
.../src/minifi_test_framework/containers/container.py | 8 +++++++-
.../src/minifi_test_framework/containers/http_proxy_container.py | 6 +++---
.../src/minifi_test_framework/containers/minifi_container.py | 6 +++---
.../src/minifi_test_framework/containers/nifi_container.py | 6 +++---
behave_framework/src/minifi_test_framework/steps/core_steps.py | 4 ++--
extensions/aws/tests/features/steps/kinesis_server_container.py | 6 +++---
extensions/aws/tests/features/steps/s3_server_container.py | 6 +++---
extensions/azure/tests/features/steps/azure_server_container.py | 6 +++---
extensions/azure/tests/features/steps/steps.py | 2 +-
.../couchbase/tests/features/steps/couchbase_server_container.py | 6 +++---
extensions/couchbase/tests/features/steps/steps.py | 2 +-
.../elasticsearch/tests/features/steps/elastic_base_container.py | 6 +++---
.../elasticsearch/tests/features/steps/elasticsearch_container.py | 4 ++--
.../elasticsearch/tests/features/steps/opensearch_container.py | 4 ++--
extensions/elasticsearch/tests/features/steps/steps.py | 4 ++--
extensions/gcp/tests/features/steps/fake_gcs_server_container.py | 6 +++---
extensions/gcp/tests/features/steps/steps.py | 2 +-
.../grafana-loki/tests/features/steps/grafana_loki_container.py | 6 +++---
.../grafana-loki/tests/features/steps/reverse_proxy_container.py | 6 +++---
extensions/kafka/tests/features/steps/kafka_server_container.py | 6 +++---
extensions/kafka/tests/features/steps/steps.py | 2 +-
extensions/opc/tests/features/steps/opc_ua_server_container.py | 6 +++---
extensions/splunk/tests/features/steps/splunk_container.py | 6 +++---
extensions/splunk/tests/features/steps/steps.py | 2 +-
extensions/sql/tests/features/steps/postgress_server_container.py | 7 ++++---
.../tests/features/steps/diag_slave_container.py | 6 +++---
extensions/standard-processors/tests/features/steps/steps.py | 2 +-
.../tests/features/steps/tcp_client_container.py | 6 +++---
28 files changed, 73 insertions(+), 66 deletions(-)
diff --git a/behave_framework/src/minifi_test_framework/containers/container.py b/behave_framework/src/minifi_test_framework/containers/container.py
index 8c59586f5..75660d785 100644
--- a/behave_framework/src/minifi_test_framework/containers/container.py
+++ b/behave_framework/src/minifi_test_framework/containers/container.py
@@ -14,6 +14,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+from __future__ import annotations
+from typing import TYPE_CHECKING
+
import json
import logging
import os
@@ -28,6 +31,9 @@ from minifi_test_framework.containers.directory import Directory
from minifi_test_framework.containers.file import File
from minifi_test_framework.containers.host_file import HostFile
+if TYPE_CHECKING:
+ from minifi_test_framework.core.minifi_test_context import MinifiTestContext
+
class Container:
def __init__(self, image_name: str, container_name: str, network: Network, command: str | None = None, entrypoint: str | None = None):
@@ -74,7 +80,7 @@ class Container:
self._write_content_to_file(file_path, None, content)
self.volumes[temp_path] = {"bind": directory.path, "mode": directory.mode}
- def deploy(self) -> bool:
+ def deploy(self, context: MinifiTestContext | None) -> bool:
self._temp_dir = tempfile.TemporaryDirectory()
self._configure_volumes_of_container_files()
self._configure_volumes_of_container_dirs()
diff --git a/behave_framework/src/minifi_test_framework/containers/http_proxy_container.py b/behave_framework/src/minifi_test_framework/containers/http_proxy_container.py
index d07c674ff..49ba2e766 100644
--- a/behave_framework/src/minifi_test_framework/containers/http_proxy_container.py
+++ b/behave_framework/src/minifi_test_framework/containers/http_proxy_container.py
@@ -46,14 +46,14 @@ class HttpProxy(Container):
super().__init__("minifi-http-proxy:latest", f"http-proxy-{test_context.scenario_id}", test_context.network)
- def deploy(self):
- super().deploy()
+ def deploy(self, context: MinifiTestContext | None) -> bool:
+ super().deploy(context)
finished_str = "Accepting HTTP Socket connections at"
return wait_for_condition(
condition=lambda: finished_str in self.get_logs(),
timeout_seconds=5,
bail_condition=lambda: self.exited,
- context=None
+ context=context
)
def check_http_proxy_access(self, url):
diff --git a/behave_framework/src/minifi_test_framework/containers/minifi_container.py b/behave_framework/src/minifi_test_framework/containers/minifi_container.py
index be72155e9..75415017f 100644
--- a/behave_framework/src/minifi_test_framework/containers/minifi_container.py
+++ b/behave_framework/src/minifi_test_framework/containers/minifi_container.py
@@ -59,7 +59,7 @@ class MinifiContainer(Container):
self._fill_default_properties()
self._fill_default_log_properties()
- def deploy(self) -> bool:
+ def deploy(self, context: MinifiTestContext | None) -> bool:
flow_config = self.flow_definition.to_yaml()
logging.info(f"Deploying MiNiFi container '{self.container_name}' with flow configuration:\n{flow_config}")
if self.is_fhs:
@@ -77,7 +77,7 @@ class MinifiContainer(Container):
resource_dir = Path(__file__).resolve().parent / "resources" / "minifi-controller"
self.host_files.append(HostFile("/tmp/resources/minifi-controller/config.yml", os.path.join(resource_dir, "config.yml")))
- if not super().deploy():
+ if not super().deploy(context):
return False
finished_str = "MiNiFi started"
@@ -85,7 +85,7 @@ class MinifiContainer(Container):
condition=lambda: finished_str in self.get_logs(),
timeout_seconds=15,
bail_condition=lambda: self.exited,
- context=None)
+ context=context)
def set_property(self, key: str, value: str):
self.properties[key] = value
diff --git a/behave_framework/src/minifi_test_framework/containers/nifi_container.py b/behave_framework/src/minifi_test_framework/containers/nifi_container.py
index 396b1e1a7..d40724de1 100644
--- a/behave_framework/src/minifi_test_framework/containers/nifi_container.py
+++ b/behave_framework/src/minifi_test_framework/containers/nifi_container.py
@@ -70,7 +70,7 @@ class NifiContainer(Container):
super().__init__("apache/nifi:" + self.NIFI_VERSION, name, test_context.network, entrypoint=command)
- def deploy(self):
+ def deploy(self, context: MinifiTestContext | None) -> bool:
flow_config = self.flow_definition.to_json()
buffer = io.BytesIO()
@@ -80,10 +80,10 @@ class NifiContainer(Container):
gzipped_bytes = buffer.getvalue()
self.files.append(File("/tmp/nifi_config/flow.json.gz", gzipped_bytes))
- super().deploy()
+ super().deploy(context)
finished_str = "Started Application in"
return wait_for_condition(
condition=lambda: finished_str in self.get_logs(),
timeout_seconds=300,
bail_condition=lambda: self.exited,
- context=None)
+ context=context)
diff --git a/behave_framework/src/minifi_test_framework/steps/core_steps.py b/behave_framework/src/minifi_test_framework/steps/core_steps.py
index 836b58834..95176f591 100644
--- a/behave_framework/src/minifi_test_framework/steps/core_steps.py
+++ b/behave_framework/src/minifi_test_framework/steps/core_steps.py
@@ -36,13 +36,13 @@ from minifi_test_framework.core.minifi_test_context import DEFAULT_MINIFI_CONTAI
@when("all instances start up")
def step_impl(context: MinifiTestContext):
for container in context.containers.values():
- assert container.deploy() or container.log_app_output()
+ assert container.deploy(context)
logging.debug("All instances started up")
@when("the MiNiFi instance starts up")
def step_impl(context: MinifiTestContext):
- assert context.get_or_create_default_minifi_container().deploy()
+ assert context.get_or_create_default_minifi_container().deploy(context)
logging.debug("MiNiFi instance started up")
diff --git a/extensions/aws/tests/features/steps/kinesis_server_container.py b/extensions/aws/tests/features/steps/kinesis_server_container.py
index e92b33c41..01b6dced2 100644
--- a/extensions/aws/tests/features/steps/kinesis_server_container.py
+++ b/extensions/aws/tests/features/steps/kinesis_server_container.py
@@ -34,14 +34,14 @@ class KinesisServerContainer(Container):
self.environment.append("INITIALIZE_STREAMS=test_stream:3")
self.environment.append("LOG_LEVEL=DEBUG")
- def deploy(self):
- super().deploy()
+ def deploy(self, context: MinifiTestContext | None) -> bool:
+ super().deploy(context)
finished_str = "Starting Kinesis Plain Mock Service on port 4568"
return wait_for_condition(
condition=lambda: finished_str in self.get_logs(),
timeout_seconds=300,
bail_condition=lambda: self.exited,
- context=None)
+ context=context)
@retry_check()
def check_kinesis_server_record_data(self, record_data):
diff --git a/extensions/aws/tests/features/steps/s3_server_container.py b/extensions/aws/tests/features/steps/s3_server_container.py
index 01db20030..699617c71 100644
--- a/extensions/aws/tests/features/steps/s3_server_container.py
+++ b/extensions/aws/tests/features/steps/s3_server_container.py
@@ -26,14 +26,14 @@ class S3ServerContainer(Container):
super().__init__("adobe/s3mock:3.12.0", f"s3-server-{test_context.scenario_id}", test_context.network)
self.environment.append("initialBuckets=test_bucket")
- def deploy(self):
- super().deploy()
+ def deploy(self, context: MinifiTestContext | None) -> bool:
+ super().deploy(context)
finished_str = "Started S3MockApplication"
return wait_for_condition(
condition=lambda: finished_str in self.get_logs(),
timeout_seconds=15,
bail_condition=lambda: self.exited,
- context=None)
+ context=context)
def check_s3_server_object_data(self, test_data):
(code, output) = self.exec_run(["find", "/s3mockroot/test_bucket", "-mindepth", "1", "-maxdepth", "1", "-type", "d"])
diff --git a/extensions/azure/tests/features/steps/azure_server_container.py b/extensions/azure/tests/features/steps/azure_server_container.py
index 8cdc3205b..4caeb786b 100644
--- a/extensions/azure/tests/features/steps/azure_server_container.py
+++ b/extensions/azure/tests/features/steps/azure_server_container.py
@@ -33,13 +33,13 @@ class AzureServerContainer(Container):
self.azure_connection_string = "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;" \
f"BlobEndpoint=http://{azure_storage_hostname}:10000/devstoreaccount1;QueueEndpoint=http://{azure_storage_hostname}:10001/devstoreaccount1;"
- def deploy(self):
- super().deploy()
+ def deploy(self, context: MinifiTestContext | None) -> bool:
+ super().deploy(context)
finished_str = "Azurite Queue service is successfully listening at"
return wait_for_condition(condition=lambda: finished_str in self.get_logs(),
timeout_seconds=15,
bail_condition=lambda: self.exited,
- context=None)
+ context=context)
def check_azure_storage_server_data(self, test_data):
(code, output) = self.exec_run(["find", "/data/__blobstorage__", "-type", "f"])
diff --git a/extensions/azure/tests/features/steps/steps.py b/extensions/azure/tests/features/steps/steps.py
index 3331d7e71..58ad46e46 100644
--- a/extensions/azure/tests/features/steps/steps.py
+++ b/extensions/azure/tests/features/steps/steps.py
@@ -43,7 +43,7 @@ def step_impl(context: MinifiTestContext, processor_name: str):
@step("an Azure storage server is set up")
def step_impl(context):
context.containers["azure-storage-server"] = AzureServerContainer(context)
- assert context.containers["azure-storage-server"].deploy()
+ assert context.containers["azure-storage-server"].deploy(context)
@then('the object on the Azure storage server is "{object_data}"')
diff --git a/extensions/couchbase/tests/features/steps/couchbase_server_container.py b/extensions/couchbase/tests/features/steps/couchbase_server_container.py
index 96cddf821..52228951d 100644
--- a/extensions/couchbase/tests/features/steps/couchbase_server_container.py
+++ b/extensions/couchbase/tests/features/steps/couchbase_server_container.py
@@ -37,14 +37,14 @@ class CouchbaseServerContainer(Container):
couchbase_key_content = crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=couchbase_key)
self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/pkey.key", couchbase_key_content, permissions=0o666))
- def deploy(self):
- super().deploy()
+ def deploy(self, context: MinifiTestContext | None) -> bool:
+ super().deploy(context)
finished_str = "logs available in"
assert wait_for_condition(
condition=lambda: finished_str in self.get_logs(),
timeout_seconds=15,
bail_condition=lambda: self.exited,
- context=None)
+ context=context)
return self.run_post_startup_commands()
def run_post_startup_commands(self):
diff --git a/extensions/couchbase/tests/features/steps/steps.py b/extensions/couchbase/tests/features/steps/steps.py
index f20cfda2e..c1d8ba73d 100644
--- a/extensions/couchbase/tests/features/steps/steps.py
+++ b/extensions/couchbase/tests/features/steps/steps.py
@@ -28,7 +28,7 @@ from couchbase_server_container import CouchbaseServerContainer
@step("a Couchbase server is started")
def step_impl(context: MinifiTestContext):
context.containers["couchbase-server"] = CouchbaseServerContainer(context)
- assert context.containers["couchbase-server"].deploy()
+ assert context.containers["couchbase-server"].deploy(context)
@step("a CouchbaseClusterService controller service is set up to communicate with the Couchbase server")
diff --git a/extensions/elasticsearch/tests/features/steps/elastic_base_container.py b/extensions/elasticsearch/tests/features/steps/elastic_base_container.py
index 515d71aeb..5db6e61dc 100644
--- a/extensions/elasticsearch/tests/features/steps/elastic_base_container.py
+++ b/extensions/elasticsearch/tests/features/steps/elastic_base_container.py
@@ -23,13 +23,13 @@ class ElasticBaseContainer(Container):
super().__init__(image, container_name, test_context.network)
self.user = None
- def deploy(self, finished_str: str):
- super().deploy()
+ def deploy(self, context: MinifiTestContext | None, finished_str: str):
+ super().deploy(context)
return wait_for_condition(
condition=lambda: finished_str in self.get_logs(),
timeout_seconds=300,
bail_condition=lambda: self.exited,
- context=None)
+ context=context)
def create_doc_elasticsearch(self, index_name: str, doc_id: str) -> bool:
(code, output) = self.exec_run(["/bin/bash", "-c",
diff --git a/extensions/elasticsearch/tests/features/steps/elasticsearch_container.py b/extensions/elasticsearch/tests/features/steps/elasticsearch_container.py
index e06458751..037711d75 100644
--- a/extensions/elasticsearch/tests/features/steps/elasticsearch_container.py
+++ b/extensions/elasticsearch/tests/features/steps/elasticsearch_container.py
@@ -53,8 +53,8 @@ class ElasticsearchContainer(ElasticBaseContainer):
self.environment.append("ELASTIC_PASSWORD=password")
- def deploy(self):
- return super().deploy('"current.health":"GREEN"')
+ def deploy(self, context: MinifiTestContext | None) -> bool:
+ return super().deploy(context, '"current.health":"GREEN"')
def elastic_generate_apikey(self):
api_url = "https://localhost:9200/_security/api_key"
diff --git a/extensions/elasticsearch/tests/features/steps/opensearch_container.py b/extensions/elasticsearch/tests/features/steps/opensearch_container.py
index 23882f530..cdf5008d5 100644
--- a/extensions/elasticsearch/tests/features/steps/opensearch_container.py
+++ b/extensions/elasticsearch/tests/features/steps/opensearch_container.py
@@ -43,8 +43,8 @@ class OpensearchContainer(ElasticBaseContainer):
features_dir = Path(__file__).resolve().parent.parent
self.host_files.append(HostFile('/usr/share/opensearch/config/opensearch.yml', os.path.join(features_dir, "resources", "opensearch.yml")))
- def deploy(self):
- return super().deploy('Hot-reloading of audit configuration is enabled')
+ def deploy(self, context: MinifiTestContext | None) -> bool:
+ return super().deploy(context, 'Hot-reloading of audit configuration is enabled')
def add_elastic_user_to_opensearch(self):
curl_cmd = [
diff --git a/extensions/elasticsearch/tests/features/steps/steps.py b/extensions/elasticsearch/tests/features/steps/steps.py
index 810c1516d..9a1fe6954 100644
--- a/extensions/elasticsearch/tests/features/steps/steps.py
+++ b/extensions/elasticsearch/tests/features/steps/steps.py
@@ -30,7 +30,7 @@ from opensearch_container import OpensearchContainer
@step('an Elasticsearch server is set up and a single document is present with "preloaded_id" in "my_index" with "value1" in "field1"')
def step_impl(context: MinifiTestContext):
context.containers["elasticsearch"] = ElasticsearchContainer(context)
- assert context.containers["elasticsearch"].deploy()
+ assert context.containers["elasticsearch"].deploy(context)
assert context.containers["elasticsearch"].create_doc_elasticsearch("my_index", "preloaded_id") or context.containers["elasticsearch"].log_app_output()
@@ -65,7 +65,7 @@ def step_impl(context):
@given('an Opensearch server is set up and a single document is present with "preloaded_id" in "my_index" with "value1" in "field1"')
def step_impl(context):
context.containers["opensearch"] = OpensearchContainer(context)
- context.containers["opensearch"].deploy()
+ context.containers["opensearch"].deploy(context)
context.containers["opensearch"].add_elastic_user_to_opensearch()
context.containers["opensearch"].create_doc_elasticsearch("my_index", "preloaded_id")
diff --git a/extensions/gcp/tests/features/steps/fake_gcs_server_container.py b/extensions/gcp/tests/features/steps/fake_gcs_server_container.py
index 132931878..a9733ba4e 100644
--- a/extensions/gcp/tests/features/steps/fake_gcs_server_container.py
+++ b/extensions/gcp/tests/features/steps/fake_gcs_server_container.py
@@ -26,14 +26,14 @@ class FakeGcsServerContainer(Container):
command=f'-scheme http -host fake-gcs-server-{test_context.scenario_id}')
self.dirs.append(Directory(path="/data/test-bucket", files={"test-file": "preloaded data\n"}))
- def deploy(self):
- super().deploy()
+ def deploy(self, context: MinifiTestContext | None) -> bool:
+ super().deploy(context)
finished_str = "server started at http"
return wait_for_condition(
condition=lambda: finished_str in self.get_logs(),
timeout_seconds=30,
bail_condition=lambda: self.exited,
- context=None)
+ context=context)
@retry_check()
def check_google_cloud_storage(self, content):
diff --git a/extensions/gcp/tests/features/steps/steps.py b/extensions/gcp/tests/features/steps/steps.py
index 639434d9d..175ff63c4 100644
--- a/extensions/gcp/tests/features/steps/steps.py
+++ b/extensions/gcp/tests/features/steps/steps.py
@@ -28,7 +28,7 @@ from fake_gcs_server_container import FakeGcsServerContainer
@step('a Google Cloud storage server is set up and a single object with contents "preloaded data" is present')
def step_impl(context: MinifiTestContext):
context.containers["fake-gcs-server"] = FakeGcsServerContainer(context)
- assert context.containers["fake-gcs-server"].deploy()
+ assert context.containers["fake-gcs-server"].deploy(context)
@then('an object with the content \"{content}\" is present in the Google Cloud storage')
diff --git a/extensions/grafana-loki/tests/features/steps/grafana_loki_container.py b/extensions/grafana-loki/tests/features/steps/grafana_loki_container.py
index 46fc84b5a..1284b95b0 100644
--- a/extensions/grafana-loki/tests/features/steps/grafana_loki_container.py
+++ b/extensions/grafana-loki/tests/features/steps/grafana_loki_container.py
@@ -126,14 +126,14 @@ analytics:
self.files.append(File("/etc/loki/local-config.yaml", grafana_loki_yml_content.encode(), permissions=0o644))
- def deploy(self):
- super().deploy()
+ def deploy(self, context: MinifiTestContext | None) -> bool:
+ super().deploy(context)
finished_str = "Loki started"
return wait_for_condition(
condition=lambda: finished_str in self.get_logs(),
timeout_seconds=120,
bail_condition=lambda: self.exited,
- context=None)
+ context=context)
@retry_check()
def are_lines_present(self, lines: str, timeout: int, ssl: bool, tenant_id: str = "") -> bool:
diff --git a/extensions/grafana-loki/tests/features/steps/reverse_proxy_container.py b/extensions/grafana-loki/tests/features/steps/reverse_proxy_container.py
index c415f6dba..409a9b9e6 100644
--- a/extensions/grafana-loki/tests/features/steps/reverse_proxy_container.py
+++ b/extensions/grafana-loki/tests/features/steps/reverse_proxy_container.py
@@ -28,11 +28,11 @@ class ReverseProxyContainer(Container):
"FORWARD_PORT=3100",
]
- def deploy(self):
- super().deploy()
+ def deploy(self, context: MinifiTestContext | None) -> bool:
+ super().deploy(context)
finished_str = "start worker process"
return wait_for_condition(
condition=lambda: finished_str in self.get_logs(),
timeout_seconds=60,
bail_condition=lambda: self.exited,
- context=None)
+ context=context)
diff --git a/extensions/kafka/tests/features/steps/kafka_server_container.py b/extensions/kafka/tests/features/steps/kafka_server_container.py
index 49c15a429..30073186f 100644
--- a/extensions/kafka/tests/features/steps/kafka_server_container.py
+++ b/extensions/kafka/tests/features/steps/kafka_server_container.py
@@ -88,14 +88,14 @@ Client {
"""
self.files.append(File("/opt/kafka/config/kafka_jaas.conf", jaas_config_file_content, permissions=0o644))
- def deploy(self):
- super().deploy()
+ def deploy(self, context: MinifiTestContext | None) -> bool:
+ super().deploy(context)
finished_str = "Kafka Server started"
return wait_for_condition(
condition=lambda: finished_str in self.get_logs(),
timeout_seconds=60,
bail_condition=lambda: self.exited,
- context=None)
+ context=context)
def create_topic(self, topic_name: str):
(code, output) = self.exec_run(["/bin/sh", "-c", f"/opt/kafka/bin/kafka-topics.sh --create --topic '{topic_name}' --bootstrap-server '{self.container_name}':9092"])
diff --git a/extensions/kafka/tests/features/steps/steps.py b/extensions/kafka/tests/features/steps/steps.py
index 474cc7bf1..0b3e2ed03 100644
--- a/extensions/kafka/tests/features/steps/steps.py
+++ b/extensions/kafka/tests/features/steps/steps.py
@@ -66,7 +66,7 @@ def step_impl(context):
def step_impl(context: MinifiTestContext):
kafka_server_container = context.containers["kafka-server"]
assert isinstance(kafka_server_container, KafkaServer)
- assert kafka_server_container.deploy()
+ assert kafka_server_container.deploy(context)
@step('the topic "{topic_name}" is initialized on the kafka broker')
diff --git a/extensions/opc/tests/features/steps/opc_ua_server_container.py b/extensions/opc/tests/features/steps/opc_ua_server_container.py
index 4973b9046..868e54202 100644
--- a/extensions/opc/tests/features/steps/opc_ua_server_container.py
+++ b/extensions/opc/tests/features/steps/opc_ua_server_container.py
@@ -23,11 +23,11 @@ class OPCUAServerContainer(Container):
def __init__(self, test_context: MinifiTestContext, command: Optional[List[str]] = None):
super().__init__("lordgamez/open62541:1.4.10", f"opcua-server-{test_context.scenario_id}", test_context.network, command=command)
- def deploy(self):
- super().deploy()
+ def deploy(self, context: MinifiTestContext | None) -> bool:
+ super().deploy(context)
finished_str = "New DiscoveryUrl added: opc.tcp://"
return wait_for_condition(
condition=lambda: finished_str in self.get_logs(),
timeout_seconds=15,
bail_condition=lambda: self.exited,
- context=None)
+ context=context)
diff --git a/extensions/splunk/tests/features/steps/splunk_container.py b/extensions/splunk/tests/features/steps/splunk_container.py
index 865407a64..4106bb63e 100644
--- a/extensions/splunk/tests/features/steps/splunk_container.py
+++ b/extensions/splunk/tests/features/steps/splunk_container.py
@@ -49,14 +49,14 @@ splunk:
self.files.append(File("/opt/splunk/etc/auth/splunk_cert.pem", splunk_cert_content.decode() + splunk_key_content.decode() + root_ca_content.decode(), permissions=0o644))
self.files.append(File("/opt/splunk/etc/auth/root_ca.pem", root_ca_content.decode(), permissions=0o644))
- def deploy(self):
- super().deploy()
+ def deploy(self, context: MinifiTestContext | None) -> bool:
+ super().deploy(context)
finished_str = "Ansible playbook complete, will begin streaming splunkd_stderr.log"
return wait_for_condition(
condition=lambda: finished_str in self.get_logs(),
timeout_seconds=300,
bail_condition=lambda: False,
- context=None)
+ context=context)
@retry_check()
def check_splunk_event(self, query: str) -> bool:
diff --git a/extensions/splunk/tests/features/steps/steps.py b/extensions/splunk/tests/features/steps/steps.py
index 16a8d66e9..5902fd2c9 100644
--- a/extensions/splunk/tests/features/steps/steps.py
+++ b/extensions/splunk/tests/features/steps/steps.py
@@ -26,7 +26,7 @@ from splunk_container import SplunkContainer
@step("a Splunk HEC is set up and running")
def step_impl(context: MinifiTestContext):
context.containers["splunk"] = SplunkContainer(context)
- assert context.containers["splunk"].deploy()
+ assert context.containers["splunk"].deploy(context)
assert context.containers["splunk"].enable_splunk_hec_indexer('splunk_hec_token') or context.containers["splunk"].log_app_output()
diff --git a/extensions/sql/tests/features/steps/postgress_server_container.py b/extensions/sql/tests/features/steps/postgress_server_container.py
index 96e0eb7eb..467eecc4b 100644
--- a/extensions/sql/tests/features/steps/postgress_server_container.py
+++ b/extensions/sql/tests/features/steps/postgress_server_container.py
@@ -20,6 +20,7 @@ from textwrap import dedent
from minifi_test_framework.containers.container import Container
from minifi_test_framework.containers.docker_image_builder import DockerImageBuilder
from minifi_test_framework.core.helpers import wait_for_condition
+from minifi_test_framework.core.minifi_test_context import MinifiTestContext
class PostgresContainer(Container):
@@ -48,14 +49,14 @@ class PostgresContainer(Container):
super(PostgresContainer, self).__init__("minifi-postgres-server:latest", f"postgres-server-{context.scenario_id}", context.network)
self.environment = ["POSTGRES_PASSWORD=password"]
- def deploy(self) -> bool:
- super(PostgresContainer, self).deploy()
+ def deploy(self, context: MinifiTestContext | None) -> bool:
+ super(PostgresContainer, self).deploy(context)
finished_str = "database system is ready to accept connections"
return wait_for_condition(
condition=lambda: finished_str in self.get_logs(),
timeout_seconds=5,
bail_condition=lambda: self.exited,
- context=None)
+ context=context)
def check_query_results(self, query: str, number_of_rows: int) -> bool:
(code, output) = self.exec_run(["psql", "-U", "postgres", "-c", query])
diff --git a/extensions/standard-processors/tests/features/steps/diag_slave_container.py b/extensions/standard-processors/tests/features/steps/diag_slave_container.py
index a168c7325..ba89e75c3 100644
--- a/extensions/standard-processors/tests/features/steps/diag_slave_container.py
+++ b/extensions/standard-processors/tests/features/steps/diag_slave_container.py
@@ -40,14 +40,14 @@ ENV PROTOCOL=tcp
super().__init__("minifi-diag-slave-tcp:latest", f"diag-slave-tcp-{test_context.scenario_id}", test_context.network)
- def deploy(self):
- super().deploy()
+ def deploy(self, context: MinifiTestContext | None) -> bool:
+ super().deploy(context)
finished_str = "Server started up successfully."
return wait_for_condition(
condition=lambda: finished_str in self.get_logs(),
timeout_seconds=5,
bail_condition=lambda: self.exited,
- context=None
+ context=context
)
def set_value_on_plc_with_modbus(self, modbus_cmd):
diff --git a/extensions/standard-processors/tests/features/steps/steps.py b/extensions/standard-processors/tests/features/steps/steps.py
index 85158cb9e..baa220e59 100644
--- a/extensions/standard-processors/tests/features/steps/steps.py
+++ b/extensions/standard-processors/tests/features/steps/steps.py
@@ -40,7 +40,7 @@ def step_impl(context: MinifiTestContext):
@step('there is an accessible PLC with modbus enabled')
def step_impl(context: MinifiTestContext):
modbus_container = context.containers["diag-slave-tcp"] = DiagSlave(context)
- assert modbus_container.deploy()
+ assert modbus_container.deploy(context)
@step('PLC register has been set with {modbus_cmd} command')
diff --git a/extensions/standard-processors/tests/features/steps/tcp_client_container.py b/extensions/standard-processors/tests/features/steps/tcp_client_container.py
index d3f02d16a..aca77dfae 100644
--- a/extensions/standard-processors/tests/features/steps/tcp_client_container.py
+++ b/extensions/standard-processors/tests/features/steps/tcp_client_container.py
@@ -29,12 +29,12 @@ class TcpClientContainer(Container):
)
super().__init__("alpine:3.17.3", f"tcp-client-{test_context.scenario_id}", test_context.network, cmd)
- def deploy(self):
- super().deploy()
+ def deploy(self, context: MinifiTestContext | None) -> bool:
+ super().deploy(context)
finished_str = "TCP client container started"
return wait_for_condition(
condition=lambda: finished_str in self.get_logs(),
timeout_seconds=5,
bail_condition=lambda: self.exited,
- context=None
+ context=context
)