You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ma...@apache.org on 2026/02/19 15:51:47 UTC
(nifi-minifi-cpp) 03/04: MINIFICPP-2718 Windows based docker tests
This is an automated email from the ASF dual-hosted git repository.
martinzink pushed a commit to branch apache-rusty
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit a6128abb1ea7cfb773e603587d0ad0ee656e889d
Author: Martin Zink <ma...@apache.org>
AuthorDate: Thu Feb 12 16:53:10 2026 +0100
MINIFICPP-2718 Windows based docker tests
Closes #2106
Signed-off-by: Martin Zink <ma...@apache.org>
---
.gitignore | 2 +
behave_framework/pyproject.toml | 5 +-
.../{container.py => container_linux.py} | 4 +-
.../containers/container_protocol.py | 37 ++
.../containers/container_windows.py | 371 +++++++++++++++++++++
.../containers/http_proxy_container.py | 4 +-
.../containers/minifi_container.py | 17 +-
.../containers/minifi_controller.py | 79 +++++
.../containers/minifi_fhs_container.py | 119 +++++++
.../containers/minifi_linux_container.py | 119 +++++++
.../containers/minifi_protocol.py | 9 +
.../containers/minifi_win_container.py | 59 ++++
.../containers/nifi_container.py | 4 +-
.../src/minifi_test_framework/core/hooks.py | 12 +-
.../core/minifi_test_context.py | 33 +-
.../src/minifi_test_framework/core/ssl_utils.py | 222 ++++++------
docker/installed/win.Dockerfile | 24 ++
.../features/steps/kinesis_server_container.py | 4 +-
.../tests/features/steps/s3_server_container.py | 4 +-
.../tests/features/steps/azure_server_container.py | 4 +-
.../features/steps/couchbase_server_container.py | 17 +-
.../tests/features/steps/elastic_base_container.py | 4 +-
.../features/steps/elasticsearch_container.py | 13 +-
.../tests/features/steps/opensearch_container.py | 9 +-
.../features/steps/fake_gcs_server_container.py | 4 +-
.../tests/features/steps/grafana_loki_container.py | 13 +-
.../features/steps/reverse_proxy_container.py | 4 +-
.../tests/features/steps/kafka_server_container.py | 13 +-
.../features/steps/opc_ua_server_container.py | 4 +-
.../tests/features/steps/splunk_container.py | 13 +-
.../features/steps/postgress_server_container.py | 4 +-
.../tests/features/core_functionality.feature | 8 +-
.../tests/features/replace_text.feature | 2 +-
.../tests/features/steps/diag_slave_container.py | 4 +-
.../features/steps/minifi_controller_steps.py | 24 +-
.../tests/features/steps/syslog_container.py | 4 +-
.../tests/features/steps/tcp_client_container.py | 4 +-
run_flake8.sh | 2 +-
38 files changed, 1038 insertions(+), 240 deletions(-)
diff --git a/.gitignore b/.gitignore
index d30e23ff6..faaf21d60 100644
--- a/.gitignore
+++ b/.gitignore
@@ -44,6 +44,8 @@ cmake_install.cmake
install_manifest.txt
CTestTestfile.cmake
cmake-build-debug
+venv
+behave_venv
# Generated files
*flowfile_checkpoint
diff --git a/behave_framework/pyproject.toml b/behave_framework/pyproject.toml
index 263750d9f..5bf1f29f2 100644
--- a/behave_framework/pyproject.toml
+++ b/behave_framework/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "minifi-test-framework"
-version = "0.1.0"
+version = "0.2.0"
requires-python = ">= 3.10"
description = "A testing framework for MiNiFi extensions."
dependencies = [
@@ -8,8 +8,7 @@ dependencies = [
"docker==7.1.0",
"PyYAML==6.0.3",
"humanfriendly==10.0",
- "m2crypto==0.46.2",
- "pyopenssl==25.0.0",
+ "cryptography==46.0.5",
"pyjks==20.0.0"
]
diff --git a/behave_framework/src/minifi_test_framework/containers/container.py b/behave_framework/src/minifi_test_framework/containers/container_linux.py
similarity index 99%
rename from behave_framework/src/minifi_test_framework/containers/container.py
rename to behave_framework/src/minifi_test_framework/containers/container_linux.py
index 75660d785..28df65790 100644
--- a/behave_framework/src/minifi_test_framework/containers/container.py
+++ b/behave_framework/src/minifi_test_framework/containers/container_linux.py
@@ -27,6 +27,7 @@ import tarfile
import docker
from docker.models.networks import Network
+from minifi_test_framework.containers.container_protocol import ContainerProtocol
from minifi_test_framework.containers.directory import Directory
from minifi_test_framework.containers.file import File
from minifi_test_framework.containers.host_file import HostFile
@@ -35,8 +36,9 @@ if TYPE_CHECKING:
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
-class Container:
+class LinuxContainer(ContainerProtocol):
def __init__(self, image_name: str, container_name: str, network: Network, command: str | None = None, entrypoint: str | None = None):
+ super().__init__()
self.image_name: str = image_name
self.container_name: str = container_name
self.network: Network = network
diff --git a/behave_framework/src/minifi_test_framework/containers/container_protocol.py b/behave_framework/src/minifi_test_framework/containers/container_protocol.py
new file mode 100644
index 000000000..8713eff9c
--- /dev/null
+++ b/behave_framework/src/minifi_test_framework/containers/container_protocol.py
@@ -0,0 +1,37 @@
+from typing import Protocol
+
+
+class ContainerProtocol(Protocol):
+ def deploy(self, context) -> bool:
+ ...
+
+ def clean_up(self):
+ ...
+
+ def exec_run(self, command):
+ ...
+
+ def directory_contains_file_with_content(self, directory_path: str, expected_content: str) -> bool:
+ ...
+
+ def directory_contains_file_with_regex(self, directory_path: str, regex_str: str) -> bool:
+ ...
+
+ def path_with_content_exists(self, path: str, content: str) -> bool:
+ ...
+
+ def get_logs(self) -> str:
+ ...
+
+ @property
+ def exited(self) -> bool:
+ ...
+
+ def get_number_of_files(self, directory_path: str) -> int:
+ ...
+
+ def verify_file_contents(self, directory_path: str, expected_contents: list[str]) -> bool:
+ ...
+
+ def log_app_output(self) -> bool:
+ ...
diff --git a/behave_framework/src/minifi_test_framework/containers/container_windows.py b/behave_framework/src/minifi_test_framework/containers/container_windows.py
new file mode 100644
index 000000000..2418d3ccf
--- /dev/null
+++ b/behave_framework/src/minifi_test_framework/containers/container_windows.py
@@ -0,0 +1,371 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from __future__ import annotations
+import logging
+import os
+import tempfile
+import base64
+import tarfile
+import io
+from typing import Union, Optional, Tuple, List, Dict, TYPE_CHECKING
+
+import docker
+from docker.models.networks import Network
+from docker.models.containers import Container
+
+from minifi_test_framework.containers.container_protocol import ContainerProtocol
+from minifi_test_framework.containers.directory import Directory
+from minifi_test_framework.containers.file import File
+from minifi_test_framework.containers.host_file import HostFile
+
+if TYPE_CHECKING:
+ from minifi_test_framework.core.minifi_test_context import MinifiTestContext
+
+
+class WindowsContainer(ContainerProtocol):
+ def __init__(self, image_name: str, container_name: str, network: Network, command: str | None = None, entrypoint: str | None = None):
+ super().__init__()
+ self.image_name: str = image_name
+ self.container_name: str = container_name
+ self.network: Network = network
+
+ self.client: docker.DockerClient = docker.from_env()
+ self.container: Optional[Container] = None
+ self.files: List[File] = []
+ self.dirs: List[Directory] = []
+ self.host_files: List[HostFile] = []
+ self.volumes: Dict = {}
+ self.command: str | None = command
+ self.entrypoint: str | None = entrypoint
+ self._temp_dir: Optional[tempfile.TemporaryDirectory] = None
+ self.ports: Optional[Dict[str, int]] = None
+ self.environment: List[str] = []
+
+ def _normalize_path(self, path: str) -> str:
+ """
+ Converts paths to Windows format (C:\\...)
+ Handles leading slashes and ensures backslashes.
+ """
+ clean_path = path.strip().replace("/", "\\")
+ if clean_path.startswith("\\"):
+ clean_path = clean_path[1:]
+
+ # If it doesn't already have a drive letter, assume C:
+ if ":" not in clean_path:
+ return f"C:\\{clean_path}"
+ return clean_path
+
+ def deploy(self, context: MinifiTestContext | None) -> bool:
+ # Cleanup previous temp dir if it exists to prevent leaks
+ if self._temp_dir:
+ self._temp_dir.cleanup()
+ self._temp_dir = tempfile.TemporaryDirectory()
+
+ # 1. Prepare Volumes (Directory objects)
+ for directory in self.dirs:
+ # Mirror directory structure locally in temp
+ # usage of os.sep ensures we use host-native separators for the temp setup
+ rel_path = directory.path.strip("/\\")
+ temp_subdir = os.path.join(self._temp_dir.name, rel_path)
+ os.makedirs(temp_subdir, exist_ok=True)
+
+ for file_name, content in directory.files.items():
+ file_path = os.path.join(temp_subdir, file_name)
+ # Write with UTF-8 to ensure content is preserved on host
+ with open(file_path, "w", encoding="utf-8") as temp_file:
+ logging.info(f"writing content into {temp_file.name}")
+ temp_file.write(content)
+
+ # Mount using Windows path format
+ container_bind_path = self._normalize_path(directory.path)
+ self.volumes[temp_subdir] = {
+ "bind": container_bind_path,
+ "mode": directory.mode
+ }
+
+ for host_file in self.host_files:
+ container_bind_path = self._normalize_path(host_file.container_path)
+ self.volumes[container_bind_path] = {"bind": host_file.host_path, "mode": host_file.mode}
+
+ # 3. Cleanup existing container
+ try:
+ existing_container = self.client.containers.get(self.container_name)
+ logging.warning(f"Found existing container '{self.container_name}'. Removing it first.")
+ existing_container.remove(force=True)
+ except docker.errors.NotFound:
+ pass
+
+ # 4. Create and Start
+ try:
+ print(f"Creating and starting container '{self.container_name}'...")
+ self.container = self.client.containers.create(
+ image=self.image_name,
+ name=self.container_name,
+ ports=self.ports,
+ environment=self.environment,
+ volumes=self.volumes,
+ network=self.network.name,
+ command=self.command,
+ entrypoint=self.entrypoint,
+ detach=True,
+ tty=False
+ )
+
+ self.container.start()
+
+ for file in self.files:
+ self._copy_content_to_container(file.content, file.path)
+
+ except Exception as e:
+ logging.error(f"Error starting container: {e}")
+ self.clean_up()
+ raise
+ return True
+
+ def _copy_content_to_container(self, content: str, target_path: str):
+ if not self.container:
+ return
+
+ win_path = self._normalize_path(target_path)
+ dir_name = os.path.dirname(win_path)
+ file_name = os.path.basename(win_path)
+
+ self._run_powershell(f"New-Item -ItemType Directory -Force -Path '{dir_name}'")
+
+ tar_stream = io.BytesIO()
+ with tarfile.open(fileobj=tar_stream, mode='w') as tar:
+ encoded_data = content.encode('utf-8')
+ tarinfo = tarfile.TarInfo(name=file_name)
+ tarinfo.size = len(encoded_data)
+ tar.addfile(tarinfo, io.BytesIO(encoded_data))
+
+ tar_stream.seek(0)
+
+ self.container.put_archive(path=dir_name, data=tar_stream)
+
+ def clean_up(self):
+ if self._temp_dir:
+ try:
+ self._temp_dir.cleanup()
+ self._temp_dir = None
+ except Exception as e:
+ logging.warning(f"Failed to cleanup temp dir: {e}")
+
+ if self.container:
+ try:
+ self.container.remove(force=True)
+ except docker.errors.NotFound:
+ pass
+ except Exception as e:
+ logging.warning(f"Failed to remove container: {e}")
+ finally:
+ self.container = None
+
+ def exec_run(self, command: Union[str, list]) -> Tuple[int | None, str]:
+ logging.debug(f"Running command: {command}")
+ if self.container:
+ # Passing a list (if provided) bypasses shell parsing
+ (code, output) = self.container.exec_run(command, detach=False)
+ # errors='replace' handles non-utf8 characters often found in Windows logs
+ decoded_output = output.decode("utf-8", errors='replace')
+ logging.debug(f"Result {code}, output: {decoded_output}")
+ return code, decoded_output
+ return None, "Container not running."
+
+ def _run_powershell(self, ps_script: str) -> Tuple[int | None, str]:
+ if not self.container:
+ return None, "Container not running"
+
+ encoded_command = base64.b64encode(ps_script.encode('utf_16_le')).decode('utf-8')
+
+ cmd_parts = ["powershell", "-NonInteractive", "-NoProfile", "-EncodedCommand", encoded_command]
+
+ return self.exec_run(cmd_parts)
+
+ def not_empty_dir_exists(self, directory_path: str) -> bool:
+ if not self.container:
+ return False
+
+ win_path = self._normalize_path(directory_path)
+ ps_script = (
+ f"if (Test-Path -Path '{win_path}' -PathType Container) {{ "
+ f" if (Get-ChildItem -Path '{win_path}') {{ exit 0 }} else {{ exit 1 }} "
+ f"}} else {{ exit 2 }}"
+ )
+
+ exit_code, _ = self._run_powershell(ps_script)
+ return exit_code == 0
+
+ def directory_contains_file_with_content(self, directory_path: str, expected_content: str) -> bool:
+ if not self.container:
+ return False
+
+ win_path = self._normalize_path(directory_path)
+ escaped_content = expected_content.replace("'", "''")
+
+ ps_script = (
+ f"if (-not (Test-Path '{win_path}')) {{ exit 2 }}; "
+ f"$matches = Get-ChildItem -Path '{win_path}' -File -Depth 0 | "
+ f"Select-String -Pattern '{escaped_content}' -SimpleMatch -List; "
+ f"if ($matches) {{ exit 0 }} else {{ exit 1 }}"
+ )
+
+ exit_code, _ = self._run_powershell(ps_script)
+ return exit_code == 0
+
+ def directory_contains_file_with_regex(self, directory_path: str, regex_str: str) -> bool:
+ if not self.container:
+ return False
+
+ win_path = self._normalize_path(directory_path)
+ escaped_regex = regex_str.replace("'", "''")
+
+ ps_script = (
+ f"if (-not (Test-Path '{win_path}')) {{ exit 2 }}; "
+ f"$matches = Get-ChildItem -Path '{win_path}' -File -Depth 0 | "
+ f"Select-String -Pattern '{escaped_regex}' -List; "
+ f"if ($matches) {{ exit 0 }} else {{ exit 1 }}"
+ )
+
+ exit_code, _ = self._run_powershell(ps_script)
+ return exit_code == 0
+
+ def path_with_content_exists(self, path: str, content: str) -> bool:
+ if not self.container:
+ return False
+
+ win_path = self._normalize_path(path)
+ # Handle CRLF and escape quotes
+ escaped_content = content.replace("'", "''").replace("\n", "\r\n")
+
+ ps_script = (
+ f"try {{ "
+ f" $found = Select-String -Path '{win_path}' -Pattern '^{escaped_content}$'; "
+ f" if ($found.Count -eq 1) {{ exit 0 }} else {{ exit 1 }} "
+ f"}} catch {{ exit 2 }}"
+ )
+
+ exit_code, output = self._run_powershell(ps_script)
+ if exit_code != 0:
+ logging.debug(f"path_with_content_exists failed for {win_path}. Output: {output}")
+
+ return exit_code == 0
+
+ def directory_has_single_file_with_content(self, directory_path: str, expected_content: str) -> bool:
+ if not self.container:
+ return False
+
+ win_path = self._normalize_path(directory_path)
+ escaped_content = expected_content.strip().replace("'", "''").replace("\n", "\r\n")
+
+ ps_script = (
+ f"$files = Get-ChildItem -Path '{win_path}' -File -Depth 0; "
+ f"if ($files.Count -ne 1) {{ exit 1 }}; "
+ f"$actual_content = Get-Content -Path $files[0].FullName -Raw; "
+ f"if ($actual_content.Trim() -eq '{escaped_content}') {{ exit 0 }} else {{ exit 2 }};"
+ )
+
+ exit_code, output = self._run_powershell(ps_script)
+ if exit_code != 0:
+ logging.debug(f"Check for single file failed in {win_path}. Output: {output}")
+
+ return exit_code == 0
+
+ def get_logs(self) -> str:
+ logging.debug("Getting logs from container '%s'", self.container_name)
+ if not self.container:
+ return ""
+ logs_as_bytes = self.container.logs()
+ # Windows logs may contain mixed encodings; replace errors to avoid crash
+ return logs_as_bytes.decode('utf-8', errors='replace')
+
+ def log_app_output(self) -> bool:
+ logs = self.get_logs()
+ logging.info("Logs of container '%s':", self.container_name)
+ for line in logs.splitlines():
+ logging.info(line)
+ return False
+
+ @property
+ def exited(self) -> bool:
+ if not self.container:
+ return False
+ try:
+ self.container.reload()
+ return self.container.status == 'exited'
+ except docker.errors.NotFound:
+ self.container = None
+ return False
+ except Exception:
+ return False
+
+ def get_number_of_files(self, directory_path: str) -> int:
+ if not self.container:
+ return -1
+
+ win_path = self._normalize_path(directory_path)
+ ps_script = f"(Get-ChildItem -Path '{win_path}' -File -Depth 0).Count"
+
+ exit_code, output = self._run_powershell(ps_script)
+
+ if exit_code != 0:
+ logging.error(f"Error counting files in '{win_path}': {output}")
+ return -1
+
+ try:
+ return int(output.strip())
+ except (ValueError, IndexError):
+ return -1
+
+ def verify_file_contents(self, directory_path: str, expected_contents: List[str]) -> bool:
+ if not self.container:
+ return False
+
+ win_path = self._normalize_path(directory_path)
+
+ # 1. Get list of files
+ ps_list = f"Get-ChildItem -Path '{win_path}' -File -Depth 0 | Select-Object -ExpandProperty FullName"
+ exit_code, output = self._run_powershell(ps_list)
+
+ if exit_code != 0:
+ logging.error(f"Error listing files in '{win_path}': {output}")
+ return False
+
+ actual_filepaths = [path.strip() for path in output.splitlines() if path.strip()]
+
+ if len(actual_filepaths) != len(expected_contents):
+ return False
+
+ actual_file_contents = []
+ for path in actual_filepaths:
+ # Read, normalize CRLF to LF, and trim
+ ps_read = (
+ f"$c = Get-Content -Path '{path}' -Raw; "
+ f"if ($c) {{ ($c -replace '\\r\\n', '\\n' -replace '\\r', '\\n').Trim() }}"
+ )
+
+ exit_code, content = self._run_powershell(ps_read)
+ if exit_code != 0:
+ return False
+ actual_file_contents.append(content)
+
+ # Normalize expected contents
+ normalized_expected = [
+ s.strip().replace("\r\n", "\n").replace("\r", "\n") for s in expected_contents
+ ]
+
+ return sorted(actual_file_contents) == sorted(normalized_expected)
diff --git a/behave_framework/src/minifi_test_framework/containers/http_proxy_container.py b/behave_framework/src/minifi_test_framework/containers/http_proxy_container.py
index 49ba2e766..c4b4d8823 100644
--- a/behave_framework/src/minifi_test_framework/containers/http_proxy_container.py
+++ b/behave_framework/src/minifi_test_framework/containers/http_proxy_container.py
@@ -17,13 +17,13 @@
from textwrap import dedent
-from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.container_linux import LinuxContainer
from minifi_test_framework.containers.docker_image_builder import DockerImageBuilder
from minifi_test_framework.core.helpers import wait_for_condition
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
-class HttpProxy(Container):
+class HttpProxy(LinuxContainer):
def __init__(self, test_context: MinifiTestContext):
dockerfile = dedent("""\
FROM {base_image}
diff --git a/behave_framework/src/minifi_test_framework/containers/minifi_container.py b/behave_framework/src/minifi_test_framework/containers/minifi_container.py
index 75415017f..32728321f 100644
--- a/behave_framework/src/minifi_test_framework/containers/minifi_container.py
+++ b/behave_framework/src/minifi_test_framework/containers/minifi_container.py
@@ -18,13 +18,12 @@
import logging
import os
from pathlib import Path
-from OpenSSL import crypto
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
from minifi_test_framework.containers.file import File
from minifi_test_framework.containers.host_file import HostFile
from minifi_test_framework.minifi.minifi_flow_definition import MinifiFlowDefinition
-from minifi_test_framework.core.ssl_utils import make_cert_without_extended_usage, make_client_cert, make_server_cert
+from minifi_test_framework.core.ssl_utils import make_cert_without_extended_usage, make_client_cert, make_server_cert, dump_cert, dump_key
from minifi_test_framework.core.helpers import wait_for_condition, retry_check
from .container import Container
@@ -37,18 +36,18 @@ class MinifiContainer(Container):
self.log_properties: dict[str, str] = {}
minifi_client_cert, minifi_client_key = make_cert_without_extended_usage(common_name=self.container_name, ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key)
- self.files.append(File("/usr/local/share/certs/ca-root-nss.crt", crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert)))
- self.files.append(File("/tmp/resources/root_ca.crt", crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert)))
- self.files.append(File("/tmp/resources/minifi_client.crt", crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=minifi_client_cert)))
- self.files.append(File("/tmp/resources/minifi_client.key", crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=minifi_client_key)))
+ self.files.append(File("/usr/local/share/certs/ca-root-nss.crt", dump_cert(test_context.root_ca_cert)))
+ self.files.append(File("/tmp/resources/root_ca.crt", dump_cert(test_context.root_ca_cert)))
+ self.files.append(File("/tmp/resources/minifi_client.crt", dump_cert(minifi_client_cert)))
+ self.files.append(File("/tmp/resources/minifi_client.key", dump_key(minifi_client_key)))
clientuser_cert, clientuser_key = make_client_cert("clientuser", ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key)
- self.files.append(File("/tmp/resources/clientuser.crt", crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=clientuser_cert)))
- self.files.append(File("/tmp/resources/clientuser.key", crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=clientuser_key)))
+ self.files.append(File("/tmp/resources/clientuser.crt", dump_cert(clientuser_cert)))
+ self.files.append(File("/tmp/resources/clientuser.key", dump_key(clientuser_key)))
minifi_server_cert, minifi_server_key = make_server_cert(common_name=f"server-{test_context.scenario_id}", ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key)
self.files.append(File("/tmp/resources/minifi_server.crt",
- crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=minifi_server_cert) + crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=minifi_server_key)))
+ dump_cert(minifi_server_cert) + dump_key(minifi_server_key)))
self.is_fhs = 'MINIFI_INSTALLATION_TYPE=FHS' in str(self.client.images.get(test_context.minifi_container_image).history())
if self.is_fhs:
diff --git a/behave_framework/src/minifi_test_framework/containers/minifi_controller.py b/behave_framework/src/minifi_test_framework/containers/minifi_controller.py
new file mode 100644
index 000000000..0dc5b8330
--- /dev/null
+++ b/behave_framework/src/minifi_test_framework/containers/minifi_controller.py
@@ -0,0 +1,79 @@
+import logging
+from minifi_test_framework.core.helpers import retry_check
+
+
+class MinifiController:
+ def __init__(self, minifi_container, config_path):
+ self.minifi_container = minifi_container
+ self.config_path = config_path
+
+ def set_controller_socket_properties(self):
+ self.minifi_container.properties["controller.socket.enable"] = "true"
+ self.minifi_container.properties["controller.socket.host"] = "localhost"
+ self.minifi_container.properties["controller.socket.port"] = "9998"
+ self.minifi_container.properties["controller.socket.local.any.interface"] = "false"
+
+ def update_flow_config_through_controller(self):
+ self.minifi_container.exec_run([self.minifi_container.minifi_controller_path, "--updateflow", "/tmp/resources/minifi-controller/config.yml"])
+
+ def updated_config_is_persisted(self) -> bool:
+ exit_code, output = self.minifi_container.exec_run(["cat", self.config_path])
+ if exit_code != 0:
+ logging.error("Failed to read MiNiFi config file to check if updated config is persisted")
+ return False
+ return "2f2a3b47-f5ba-49f6-82b5-bc1c86b96f38" in output
+
+ def stop_component_through_controller(self, component: str):
+ self.minifi_container.exec_run([self.minifi_container.minifi_controller_path, "--stop", component])
+
+ def start_component_through_controller(self, component: str):
+ self.minifi_container.exec_run([self.minifi_container.minifi_controller_path, "--start", component])
+
+ @retry_check(10, 1)
+ def is_component_running(self, component: str) -> bool:
+ (code, output) = self.minifi_container.exec_run([self.minifi_container.minifi_controller_path, "--list", "components"])
+ return code == 0 and component + ", running: true" in output
+
+ def get_connections(self):
+ (_, output) = self.minifi_container.exec_run([self.minifi_container.minifi_controller_path, "--list", "connections"])
+ connections = []
+ for line in output.split('\n'):
+ if not line.startswith('[') and not line.startswith('Connection Names'):
+ connections.append(line)
+ return connections
+
+ @retry_check(10, 1)
+ def connection_found_through_controller(self, connection: str) -> bool:
+ return connection in self.get_connections()
+
+ def get_full_connection_count(self) -> int:
+ (_, output) = self.minifi_container.exec_run([self.minifi_container.minifi_controller_path, "--getfull"])
+ for line in output.split('\n'):
+ if "are full" in line:
+ return int(line.split(' ')[0])
+ return -1
+
+ def get_connection_size(self, connection: str):
+ (_, output) = self.minifi_container.exec_run([self.minifi_container.minifi_controller_path, "--getsize", connection])
+ for line in output.split('\n'):
+ if "Size/Max of " + connection in line:
+ size_and_max = line.split(connection)[1].split('/')
+ return (int(size_and_max[0].strip()), int(size_and_max[1].strip()))
+ return (-1, -1)
+
+ def get_manifest(self) -> str:
+ (_, output) = self.minifi_container.exec_run([self.minifi_container.minifi_controller_path, "--manifest"])
+ manifest = ""
+ for line in output.split('\n'):
+ if not line.startswith('['):
+ manifest += line
+ return manifest
+
+ def create_debug_bundle(self) -> bool:
+ (code, _) = self.minifi_container.exec_run([self.minifi_container.minifi_controller_path, "--debug", "/tmp"])
+ if code != 0:
+ logging.error("Minifi controller debug command failed with code: %d", code)
+ return False
+
+ (code, _) = self.minifi_container.exec_run(["test", "-f", "/tmp/debug.tar.gz"])
+ return code == 0
diff --git a/behave_framework/src/minifi_test_framework/containers/minifi_fhs_container.py b/behave_framework/src/minifi_test_framework/containers/minifi_fhs_container.py
new file mode 100644
index 000000000..49c5236f0
--- /dev/null
+++ b/behave_framework/src/minifi_test_framework/containers/minifi_fhs_container.py
@@ -0,0 +1,119 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import logging
+import os
+from pathlib import Path
+
+from .container_linux import LinuxContainer
+
+from minifi_test_framework.core.minifi_test_context import MinifiTestContext
+from minifi_test_framework.containers.file import File
+from minifi_test_framework.containers.host_file import HostFile
+from minifi_test_framework.minifi.minifi_flow_definition import MinifiFlowDefinition
+from minifi_test_framework.core.ssl_utils import make_cert_without_extended_usage, make_client_cert, make_server_cert, dump_cert, dump_key
+from minifi_test_framework.core.helpers import wait_for_condition
+
+from .minifi_protocol import MinifiProtocol
+from .minifi_controller import MinifiController
+
+
+class MinifiFhsContainer(LinuxContainer, MinifiProtocol):
+ def __init__(self, container_name: str, test_context: MinifiTestContext):
+ super().__init__(test_context.minifi_container_image, f"{container_name}-{test_context.scenario_id}", test_context.network)
+ self.flow_definition = MinifiFlowDefinition()
+ self.properties: dict[str, str] = {}
+ self.log_properties: dict[str, str] = {}
+ self.controller = MinifiController(self, "/etc/nifi-minifi-cpp/config.yml")
+
+ minifi_client_cert, minifi_client_key = make_cert_without_extended_usage(common_name=self.container_name, ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key)
+ self.files.append(File("/usr/local/share/certs/ca-root-nss.crt", dump_cert(test_context.root_ca_cert)))
+ self.files.append(File("/tmp/resources/root_ca.crt", dump_cert(test_context.root_ca_cert)))
+ self.files.append(File("/tmp/resources/minifi_client.crt", dump_cert(minifi_client_cert)))
+ self.files.append(File("/tmp/resources/minifi_client.key", dump_key(minifi_client_key)))
+
+ clientuser_cert, clientuser_key = make_client_cert("clientuser", ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key)
+ self.files.append(File("/tmp/resources/clientuser.crt", dump_cert(clientuser_cert)))
+ self.files.append(File("/tmp/resources/clientuser.key", dump_key(clientuser_key)))
+
+ minifi_server_cert, minifi_server_key = make_server_cert(common_name=f"server-{test_context.scenario_id}", ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key)
+ self.files.append(File("/tmp/resources/minifi_server.crt",
+ dump_cert(minifi_server_cert) + dump_key(minifi_server_key)))
+
+ self.minifi_controller_path = '/usr/bin/minifi-controller'
+
+ self._fill_default_properties()
+ self._fill_default_log_properties()
+
+ def deploy(self, context: MinifiTestContext | None) -> bool:
+ flow_config = self.flow_definition.to_yaml()
+ logging.info(f"Deploying MiNiFi container '{self.container_name}' with flow configuration:\n{flow_config}")
+ self.files.append(File("/etc/nifi-minifi-cpp/config.yml", flow_config))
+ self.files.append(File("/etc/nifi-minifi-cpp/minifi.properties", self._get_properties_file_content()))
+ self.files.append(File("/etc/nifi-minifi-cpp/minifi-log.properties", self._get_log_properties_file_content()))
+ resource_dir = Path(__file__).resolve().parent / "resources" / "minifi-controller"
+ self.host_files.append(HostFile("/tmp/resources/minifi-controller/config.yml", os.path.join(resource_dir, "config.yml")))
+
+ if not super().deploy(context):
+ return False
+
+ finished_str = "MiNiFi started"
+ return wait_for_condition(
+ condition=lambda: finished_str in self.get_logs(),
+ timeout_seconds=15,
+ bail_condition=lambda: self.exited,
+ context=context)
+
+ def set_property(self, key: str, value: str):
+ self.properties[key] = value
+
+ def set_log_property(self, key: str, value: str):
+ self.log_properties[key] = value
+
+ def enable_openssl_fips_mode(self):
+ self.properties["nifi.openssl.fips.support.enable"] = "true"
+
+ def _fill_default_properties(self):
+ self.properties["nifi.flow.configuration.file"] = "/etc/nifi-minifi-cpp/config.yml"
+ self.properties["nifi.extension.path"] = "/usr/lib64/nifi-minifi-cpp/extensions/*"
+ self.properties["nifi.administrative.yield.duration"] = "1 sec"
+ self.properties["nifi.bored.yield.duration"] = "100 millis"
+ self.properties["nifi.openssl.fips.support.enable"] = "false"
+ self.properties["nifi.provenance.repository.class.name"] = "NoOpRepository"
+
+ def _fill_default_log_properties(self):
+ self.log_properties["spdlog.pattern"] = "[%Y-%m-%d %H:%M:%S.%e] [%n] [%l] %v"
+
+ self.log_properties["appender.stderr"] = "stderr"
+ self.log_properties["logger.root"] = "DEBUG, stderr"
+ self.log_properties["logger.org::apache::nifi::minifi"] = "DEBUG, stderr"
+
+ def _get_properties_file_content(self):
+ lines = (f"{key}={value}" for key, value in self.properties.items())
+ return "\n".join(lines)
+
+ def _get_log_properties_file_content(self):
+ lines = (f"{key}={value}" for key, value in self.log_properties.items())
+ return "\n".join(lines)
+
+ def get_memory_usage(self) -> int | None:
+ exit_code, output = self.exec_run(["awk", "/VmRSS/ { printf \"%d\\n\", $2 }", "/proc/1/status"])
+ if exit_code != 0:
+ return None
+ memory_usage_in_bytes = int(output.strip()) * 1024
+ logging.info(f"MiNiFi memory usage: {memory_usage_in_bytes} bytes")
+ return memory_usage_in_bytes
diff --git a/behave_framework/src/minifi_test_framework/containers/minifi_linux_container.py b/behave_framework/src/minifi_test_framework/containers/minifi_linux_container.py
new file mode 100644
index 000000000..f0279ebe2
--- /dev/null
+++ b/behave_framework/src/minifi_test_framework/containers/minifi_linux_container.py
@@ -0,0 +1,119 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import logging
+import os
+from pathlib import Path
+
+from .container_linux import LinuxContainer
+
+from minifi_test_framework.core.minifi_test_context import MinifiTestContext
+from minifi_test_framework.containers.file import File
+from minifi_test_framework.containers.host_file import HostFile
+from minifi_test_framework.minifi.minifi_flow_definition import MinifiFlowDefinition
+from minifi_test_framework.core.ssl_utils import make_cert_without_extended_usage, make_client_cert, make_server_cert, dump_cert, dump_key
+from minifi_test_framework.core.helpers import wait_for_condition
+
+from .minifi_protocol import MinifiProtocol
+from .minifi_controller import MinifiController
+
+
+class MinifiLinuxContainer(LinuxContainer, MinifiProtocol):
+ def __init__(self, container_name: str, test_context: MinifiTestContext):
+ super().__init__(test_context.minifi_container_image, f"{container_name}-{test_context.scenario_id}", test_context.network)
+ self.flow_definition = MinifiFlowDefinition()
+ self.properties: dict[str, str] = {}
+ self.log_properties: dict[str, str] = {}
+ self.controller = MinifiController(self, "/opt/minifi/minifi-current/conf/config.yml")
+
+ minifi_client_cert, minifi_client_key = make_cert_without_extended_usage(common_name=self.container_name, ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key)
+ self.files.append(File("/usr/local/share/certs/ca-root-nss.crt", dump_cert(test_context.root_ca_cert)))
+ self.files.append(File("/tmp/resources/root_ca.crt", dump_cert(test_context.root_ca_cert)))
+ self.files.append(File("/tmp/resources/minifi_client.crt", dump_cert(minifi_client_cert)))
+ self.files.append(File("/tmp/resources/minifi_client.key", dump_key(minifi_client_key)))
+
+ clientuser_cert, clientuser_key = make_client_cert("clientuser", ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key)
+ self.files.append(File("/tmp/resources/clientuser.crt", dump_cert(clientuser_cert)))
+ self.files.append(File("/tmp/resources/clientuser.key", dump_key(clientuser_key)))
+
+ minifi_server_cert, minifi_server_key = make_server_cert(common_name=f"server-{test_context.scenario_id}", ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key)
+ self.files.append(File("/tmp/resources/minifi_server.crt",
+ dump_cert(cert=minifi_server_cert) + dump_key(minifi_server_key)))
+
+ self.minifi_controller_path = '/opt/minifi/minifi-current/bin/minifi-controller'
+
+ self._fill_default_properties()
+ self._fill_default_log_properties()
+
+ def deploy(self, context: MinifiTestContext | None) -> bool:
+ flow_config = self.flow_definition.to_yaml()
+ logging.info(f"Deploying MiNiFi container '{self.container_name}' with flow configuration:\n{flow_config}")
+ self.files.append(File("/opt/minifi/minifi-current/conf/config.yml", flow_config))
+ self.files.append(File("/opt/minifi/minifi-current/conf/minifi.properties", self._get_properties_file_content()))
+ self.files.append(File("/opt/minifi/minifi-current/conf/minifi-log.properties", self._get_log_properties_file_content()))
+ resource_dir = Path(__file__).resolve().parent / "resources" / "minifi-controller"
+ self.host_files.append(HostFile("/tmp/resources/minifi-controller/config.yml", os.path.join(resource_dir, "config.yml")))
+
+ if not super().deploy(context):
+ return False
+
+ finished_str = "MiNiFi started"
+ return wait_for_condition(
+ condition=lambda: finished_str in self.get_logs(),
+ timeout_seconds=15,
+ bail_condition=lambda: self.exited,
+ context=context)
+
+ def set_property(self, key: str, value: str):
+ self.properties[key] = value
+
+ def set_log_property(self, key: str, value: str):
+ self.log_properties[key] = value
+
+ def enable_openssl_fips_mode(self):
+ self.properties["nifi.openssl.fips.support.enable"] = "true"
+
+ def _fill_default_properties(self):
+ self.properties["nifi.flow.configuration.file"] = "./conf/config.yml"
+ self.properties["nifi.extension.path"] = "../extensions/*"
+ self.properties["nifi.administrative.yield.duration"] = "1 sec"
+ self.properties["nifi.bored.yield.duration"] = "100 millis"
+ self.properties["nifi.openssl.fips.support.enable"] = "false"
+ self.properties["nifi.provenance.repository.class.name"] = "NoOpRepository"
+
+ def _fill_default_log_properties(self):
+ self.log_properties["spdlog.pattern"] = "[%Y-%m-%d %H:%M:%S.%e] [%n] [%l] %v"
+
+ self.log_properties["appender.stderr"] = "stderr"
+ self.log_properties["logger.root"] = "DEBUG, stderr"
+ self.log_properties["logger.org::apache::nifi::minifi"] = "DEBUG, stderr"
+
+ def _get_properties_file_content(self):
+ lines = (f"{key}={value}" for key, value in self.properties.items())
+ return "\n".join(lines)
+
+ def _get_log_properties_file_content(self):
+ lines = (f"{key}={value}" for key, value in self.log_properties.items())
+ return "\n".join(lines)
+
+ def get_memory_usage(self) -> int | None:
+ exit_code, output = self.exec_run(["awk", "/VmRSS/ { printf \"%d\\n\", $2 }", "/proc/1/status"])
+ if exit_code != 0:
+ return None
+ memory_usage_in_bytes = int(output.strip()) * 1024
+ logging.info(f"MiNiFi memory usage: {memory_usage_in_bytes} bytes")
+ return memory_usage_in_bytes
diff --git a/behave_framework/src/minifi_test_framework/containers/minifi_protocol.py b/behave_framework/src/minifi_test_framework/containers/minifi_protocol.py
new file mode 100644
index 000000000..1c6a110d8
--- /dev/null
+++ b/behave_framework/src/minifi_test_framework/containers/minifi_protocol.py
@@ -0,0 +1,9 @@
+from typing import Protocol
+
+
+class MinifiProtocol(Protocol):
+ def set_property(self, key: str, value: str):
+ ...
+
+ def set_log_property(self, key: str, value: str):
+ ...
diff --git a/behave_framework/src/minifi_test_framework/containers/minifi_win_container.py b/behave_framework/src/minifi_test_framework/containers/minifi_win_container.py
new file mode 100644
index 000000000..2e8952cf3
--- /dev/null
+++ b/behave_framework/src/minifi_test_framework/containers/minifi_win_container.py
@@ -0,0 +1,59 @@
+from typing import Dict
+
+from minifi_test_framework.core.minifi_test_context import MinifiTestContext
+from minifi_test_framework.minifi.minifi_flow_definition import MinifiFlowDefinition
+from minifi_test_framework.containers.directory import Directory
+from .container_windows import WindowsContainer
+from .minifi_protocol import MinifiProtocol
+import logging
+
+
+class MinifiWindowsContainer(WindowsContainer, MinifiProtocol):
+ def __init__(self, container_name: str, test_context: MinifiTestContext):
+ super().__init__(test_context.minifi_container_image, f"{container_name}-{test_context.scenario_id}", test_context.network)
+ self.flow_config_str: str = ""
+ self.flow_definition = MinifiFlowDefinition()
+ self.properties: Dict[str, str] = {}
+ self.log_properties: Dict[str, str] = {}
+
+ self._fill_default_properties()
+ self._fill_default_log_properties()
+
+ def deploy(self, context: MinifiTestContext | None) -> bool:
+ logging.info(self.flow_definition.to_yaml())
+ conf_dir = Directory("\\Program Files\\ApacheNiFiMiNiFi\\nifi-minifi-cpp\\conf")
+ conf_dir.add_file("config.yml", self.flow_definition.to_yaml())
+ conf_dir.add_file("minifi.properties", self._get_properties_file_content())
+ conf_dir.add_file("minifi-log.properties", self._get_log_properties_file_content())
+
+ self.dirs.append(conf_dir)
+ return super().deploy(context)
+
+ def set_property(self, key: str, value: str):
+ self.properties[key] = value
+
+ def set_log_property(self, key: str, value: str):
+ self.log_properties[key] = value
+
+ def _fill_default_properties(self):
+ self.properties["nifi.flow.configuration.file"] = "./conf/config.yml"
+ self.properties["nifi.extension.path"] = "../extensions/*"
+ self.properties["nifi.administrative.yield.duration"] = "1 sec"
+ self.properties["nifi.bored.yield.duration"] = "100 millis"
+ self.properties["nifi.openssl.fips.support.enable"] = "false"
+ self.properties["nifi.provenance.repository.class.name"] = "NoOpRepository"
+
+ def _fill_default_log_properties(self):
+ self.log_properties["spdlog.pattern"] = "[%Y-%m-%d %H:%M:%S.%e] [%n] [%l] %v"
+
+ self.log_properties["appender.stderr"] = "stderr"
+ self.log_properties["logger.root"] = "DEBUG, stderr"
+ self.log_properties["logger.org::apache::nifi::minifi"] = "DEBUG, stderr"
+
+ def _get_properties_file_content(self):
+ lines = (f"{key}={value}" for key, value in self.properties.items())
+ return "\n".join(lines)
+
+ def _get_log_properties_file_content(self):
+ lines = (f"{key}={value}" for key, value in self.log_properties.items())
+ return "\n".join(lines)
diff --git a/behave_framework/src/minifi_test_framework/containers/nifi_container.py b/behave_framework/src/minifi_test_framework/containers/nifi_container.py
index d40724de1..140b5d508 100644
--- a/behave_framework/src/minifi_test_framework/containers/nifi_container.py
+++ b/behave_framework/src/minifi_test_framework/containers/nifi_container.py
@@ -17,13 +17,13 @@ import io
import gzip
from typing import List, Optional
from minifi_test_framework.containers.file import File
-from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.container_linux import LinuxContainer
from minifi_test_framework.core.helpers import wait_for_condition
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
from minifi_test_framework.minifi.nifi_flow_definition import NifiFlowDefinition
-class NifiContainer(Container):
+class NifiContainer(LinuxContainer):
NIFI_VERSION = '2.7.2'
def __init__(self, test_context: MinifiTestContext, command: Optional[List[str]] = None, use_ssl: bool = False):
diff --git a/behave_framework/src/minifi_test_framework/core/hooks.py b/behave_framework/src/minifi_test_framework/core/hooks.py
index 298f27d85..576ac40f8 100644
--- a/behave_framework/src/minifi_test_framework/core/hooks.py
+++ b/behave_framework/src/minifi_test_framework/core/hooks.py
@@ -46,6 +46,10 @@ def inject_scenario_id(context: MinifiTestContext, step):
def common_before_scenario(context: Context, scenario: Scenario):
+ if "SUPPORTS_WINDOWS" not in scenario.effective_tags and os.name == 'nt':
+ scenario.skip("No windows support")
+ return
+
if not hasattr(context, "minifi_container_image"):
context.minifi_container_image = get_minifi_container_image()
@@ -80,6 +84,8 @@ def common_before_scenario(context: Context, scenario: Scenario):
def common_after_scenario(context: MinifiTestContext, scenario: Scenario):
- for container in context.containers.values():
- container.clean_up()
- context.network.remove()
+ if hasattr(context, 'containers'):
+ for container in context.containers.values():
+ container.clean_up()
+ if hasattr(context, 'network'):
+ context.network.remove()
diff --git a/behave_framework/src/minifi_test_framework/core/minifi_test_context.py b/behave_framework/src/minifi_test_framework/core/minifi_test_context.py
index 959f2e1fa..5dd4db9cc 100644
--- a/behave_framework/src/minifi_test_framework/core/minifi_test_context.py
+++ b/behave_framework/src/minifi_test_framework/core/minifi_test_context.py
@@ -16,32 +16,45 @@
#
from __future__ import annotations
-from typing import TYPE_CHECKING
+
+import os
+from typing import Any
+import docker
from behave.runner import Context
from docker.models.networks import Network
-from minifi_test_framework.containers.container import Container
-from OpenSSL import crypto
+from minifi_test_framework.containers.container_protocol import ContainerProtocol
+from minifi_test_framework.containers.minifi_protocol import MinifiProtocol
-if TYPE_CHECKING:
- from minifi_test_framework.containers.minifi_container import MinifiContainer
DEFAULT_MINIFI_CONTAINER_NAME = "minifi-primary"
+class MinifiContainer(ContainerProtocol, MinifiProtocol):
+ pass
+
+
class MinifiTestContext(Context):
- containers: dict[str, Container]
+ containers: dict[str, ContainerProtocol]
scenario_id: str
network: Network
minifi_container_image: str
resource_dir: str | None
- root_ca_key: crypto.PKey
- root_ca_cert: crypto.X509
+ root_ca_key: Any
+ root_ca_cert: Any
def get_or_create_minifi_container(self, container_name: str) -> MinifiContainer:
if container_name not in self.containers:
- from minifi_test_framework.containers.minifi_container import MinifiContainer
- self.containers[container_name] = MinifiContainer(container_name, self)
+ if os.name == 'nt':
+ from minifi_test_framework.containers.minifi_win_container import MinifiWindowsContainer
+ minifi_container = MinifiWindowsContainer(container_name, self)
+ elif 'MINIFI_INSTALLATION_TYPE=FHS' in str(docker.from_env().images.get(self.minifi_container_image).history()):
+ from minifi_test_framework.containers.minifi_fhs_container import MinifiFhsContainer
+ minifi_container = MinifiFhsContainer(container_name, self)
+ else:
+ from minifi_test_framework.containers.minifi_linux_container import MinifiLinuxContainer
+ minifi_container = MinifiLinuxContainer(container_name, self)
+ self.containers[container_name] = minifi_container
return self.containers[container_name]
def get_or_create_default_minifi_container(self) -> MinifiContainer:
diff --git a/behave_framework/src/minifi_test_framework/core/ssl_utils.py b/behave_framework/src/minifi_test_framework/core/ssl_utils.py
index d39cf9646..b11da3443 100644
--- a/behave_framework/src/minifi_test_framework/core/ssl_utils.py
+++ b/behave_framework/src/minifi_test_framework/core/ssl_utils.py
@@ -1,161 +1,125 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-import time
-import logging
-import random
-
-from M2Crypto import X509, EVP, RSA, ASN1
-from OpenSSL import crypto
+import datetime
+from cryptography import x509
+from cryptography.x509.oid import NameOID
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.asymmetric import rsa
+from cryptography.hazmat.primitives import serialization
def gen_cert():
- """
- Generate TLS certificate request for testing
- """
-
- req, key = gen_req()
- pub_key = req.get_pubkey()
- subject = req.get_subject()
- cert = X509.X509()
- # noinspection PyTypeChecker
- cert.set_serial_number(1)
- cert.set_version(2)
- cert.set_subject(subject)
- t = int(time.time())
- now = ASN1.ASN1_UTCTIME()
- now.set_time(t)
- now_plus_year = ASN1.ASN1_UTCTIME()
- now_plus_year.set_time(t + 60 * 60 * 24 * 365)
- cert.set_not_before(now)
- cert.set_not_after(now_plus_year)
- issuer = X509.X509_Name()
- issuer.C = 'US'
- issuer.CN = 'minifi-listen'
- cert.set_issuer(issuer)
- cert.set_pubkey(pub_key)
- cert.sign(key, 'sha256')
+ key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
- return cert, key
-
-
-def rsa_gen_key_callback():
- pass
-
-
-def gen_req():
- """
- Generate TLS certificate request for testing
- """
+ subject = issuer = x509.Name([
+ x509.NameAttribute(NameOID.COUNTRY_NAME, u"US"),
+ x509.NameAttribute(NameOID.COMMON_NAME, u"minifi-listen"),
+ ])
- logging.info('Generating test certificate request')
- key = EVP.PKey()
- req = X509.Request()
- rsa = RSA.gen_key(1024, 65537, rsa_gen_key_callback)
- key.assign_rsa(rsa)
- req.set_pubkey(key)
- name = req.get_subject()
- name.C = 'US'
- name.CN = 'minifi-listen'
- req.sign(key, 'sha256')
+ cert = x509.CertificateBuilder().subject_name(
+ subject
+ ).issuer_name(
+ issuer
+ ).public_key(
+ key.public_key()
+ ).serial_number(
+ x509.random_serial_number()
+ ).not_valid_before(
+ datetime.datetime.utcnow()
+ ).not_valid_after(
+ datetime.datetime.utcnow() + datetime.timedelta(days=365)
+ ).sign(key, hashes.SHA256())
- return req, key
+ return cert, key
def make_self_signed_cert(common_name):
- ca_key = crypto.PKey()
- ca_key.generate_key(crypto.TYPE_RSA, 2048)
+ key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
- ca_cert = crypto.X509()
- ca_cert.set_version(2)
- ca_cert.set_serial_number(random.randint(50000000, 100000000))
-
- ca_subj = ca_cert.get_subject()
- ca_subj.commonName = common_name
-
- ca_cert.add_extensions([
- crypto.X509Extension(b"subjectKeyIdentifier", False, b"hash", subject=ca_cert),
- ])
-
- ca_cert.add_extensions([
- crypto.X509Extension(b"authorityKeyIdentifier", False, b"keyid:always", issuer=ca_cert),
+ subject = issuer = x509.Name([
+ x509.NameAttribute(NameOID.COMMON_NAME, common_name),
])
- ca_cert.add_extensions([
- crypto.X509Extension(b"basicConstraints", False, b"CA:TRUE"),
- crypto.X509Extension(b"keyUsage", False, b"keyCertSign, cRLSign"),
- ])
-
- ca_cert.set_issuer(ca_subj)
- ca_cert.set_pubkey(ca_key)
-
- ca_cert.gmtime_adj_notBefore(0)
- ca_cert.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60)
+ cert = x509.CertificateBuilder().subject_name(
+ subject
+ ).issuer_name(
+ issuer
+ ).public_key(
+ key.public_key()
+ ).serial_number(
+ x509.random_serial_number()
+ ).not_valid_before(
+ datetime.datetime.utcnow()
+ ).not_valid_after(
+ datetime.datetime.utcnow() + datetime.timedelta(days=3650)
+ ).add_extension(
+ x509.SubjectKeyIdentifier.from_public_key(key.public_key()),
+ critical=False,
+ ).add_extension(
+ x509.BasicConstraints(ca=True, path_length=None),
+ critical=True,
+ ).sign(key, hashes.SHA256())
- ca_cert.sign(ca_key, 'sha256')
-
- return ca_cert, ca_key
+ return cert, key
def _make_cert(common_name, ca_cert, ca_key, extended_key_usage=None):
- key = crypto.PKey()
- key.generate_key(crypto.TYPE_RSA, 2048)
-
- cert = crypto.X509()
- cert.set_version(2)
- cert.set_serial_number(random.randint(50000000, 100000000))
-
- client_subj = cert.get_subject()
- client_subj.commonName = common_name
+ key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
- cert.add_extensions([
- crypto.X509Extension(b"basicConstraints", False, b"CA:FALSE"),
- crypto.X509Extension(b"subjectKeyIdentifier", False, b"hash", subject=cert),
+ subject = x509.Name([
+ x509.NameAttribute(NameOID.COMMON_NAME, common_name),
])
- extensions = [crypto.X509Extension(b"authorityKeyIdentifier", False, b"keyid:always", issuer=ca_cert),
- crypto.X509Extension(b"keyUsage", False, b"digitalSignature")]
+ builder = x509.CertificateBuilder().subject_name(
+ subject
+ ).issuer_name(
+ ca_cert.subject
+ ).public_key(
+ key.public_key()
+ ).serial_number(
+ x509.random_serial_number()
+ ).not_valid_before(
+ datetime.datetime.utcnow()
+ ).not_valid_after(
+ datetime.datetime.utcnow() + datetime.timedelta(days=3650)
+ ).add_extension(
+ x509.BasicConstraints(ca=False, path_length=None),
+ critical=True,
+ ).add_extension(
+ x509.SubjectKeyIdentifier.from_public_key(key.public_key()),
+ critical=False,
+ ).add_extension(
+ x509.SubjectAlternativeName([x509.DNSName(common_name)]),
+ critical=False,
+ )
if extended_key_usage:
- extensions.append(crypto.X509Extension(b"extendedKeyUsage", False, extended_key_usage))
-
- cert.add_extensions([
- crypto.X509Extension(b"subjectAltName", False, b"DNS.1:" + common_name.encode())
- ])
-
- cert.add_extensions(extensions)
-
- cert.set_issuer(ca_cert.get_subject())
- cert.set_pubkey(key)
-
- cert.gmtime_adj_notBefore(0)
- cert.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60)
-
- cert.sign(ca_key, 'sha256')
+ builder = builder.add_extension(
+ x509.ExtendedKeyUsage(extended_key_usage),
+ critical=False
+ )
+ cert = builder.sign(ca_key, hashes.SHA256())
return cert, key
def make_client_cert(common_name, ca_cert, ca_key):
- return _make_cert(common_name=common_name, ca_cert=ca_cert, ca_key=ca_key, extended_key_usage=b"clientAuth")
+ return _make_cert(common_name, ca_cert, ca_key, [x509.OID_CLIENT_AUTH])
def make_server_cert(common_name, ca_cert, ca_key):
- return _make_cert(common_name=common_name, ca_cert=ca_cert, ca_key=ca_key, extended_key_usage=b"serverAuth")
+ return _make_cert(common_name, ca_cert, ca_key, [x509.OID_SERVER_AUTH])
def make_cert_without_extended_usage(common_name, ca_cert, ca_key):
- return _make_cert(common_name=common_name, ca_cert=ca_cert, ca_key=ca_key, extended_key_usage=None)
+ return _make_cert(common_name, ca_cert, ca_key, None)
+
+
+def dump_cert(cert, encoding_type=serialization.Encoding.PEM):
+ return cert.public_bytes(encoding_type)
+
+
+def dump_key(key, encoding_type=serialization.Encoding.PEM):
+ return key.private_bytes(
+ encoding=encoding_type,
+ format=serialization.PrivateFormat.TraditionalOpenSSL,
+ encryption_algorithm=serialization.NoEncryption()
+ )
diff --git a/docker/installed/win.Dockerfile b/docker/installed/win.Dockerfile
new file mode 100644
index 000000000..b413b68b9
--- /dev/null
+++ b/docker/installed/win.Dockerfile
@@ -0,0 +1,24 @@
+#escape=`
+
+FROM mcr.microsoft.com/windows/servercore:ltsc2022
+
+LABEL maintainer="Apache NiFi <de...@nifi.apache.org>"
+
+ARG MSI_SOURCE="nifi-minifi-cpp.msi"
+
+ENV MINIFI_HOME="C:\Program Files\ApacheNiFiMiNiFi\nifi-minifi-cpp"
+
+SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
+
+RUN Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))
+
+RUN choco install git -y
+RUN choco install vcredist140 -y
+
+COPY ${MSI_SOURCE} C:\temp\minifi.msi
+
+SHELL ["cmd", "/S", "/C"]
+
+RUN C:\Windows\System32\msiexec.exe /i C:\temp\minifi.msi /qn /norestart /L*V C:\minifi_install.log
+
+CMD ["C:\\Program Files\\ApacheNiFiMiNiFi\\nifi-minifi-cpp\\bin\\minifi.exe"]
\ No newline at end of file
diff --git a/extensions/aws/tests/features/steps/kinesis_server_container.py b/extensions/aws/tests/features/steps/kinesis_server_container.py
index 01b6dced2..d97bb5c83 100644
--- a/extensions/aws/tests/features/steps/kinesis_server_container.py
+++ b/extensions/aws/tests/features/steps/kinesis_server_container.py
@@ -16,13 +16,13 @@
import logging
from pathlib import Path
-from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.container_linux import LinuxContainer
from minifi_test_framework.core.helpers import wait_for_condition, retry_check
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
from minifi_test_framework.containers.docker_image_builder import DockerImageBuilder
-class KinesisServerContainer(Container):
+class KinesisServerContainer(LinuxContainer):
def __init__(self, test_context: MinifiTestContext):
builder = DockerImageBuilder(
image_tag="minifi-kinesis-mock:latest",
diff --git a/extensions/aws/tests/features/steps/s3_server_container.py b/extensions/aws/tests/features/steps/s3_server_container.py
index 699617c71..a2c915743 100644
--- a/extensions/aws/tests/features/steps/s3_server_container.py
+++ b/extensions/aws/tests/features/steps/s3_server_container.py
@@ -16,12 +16,12 @@
import json
import logging
-from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.container_linux import LinuxContainer
from minifi_test_framework.core.helpers import wait_for_condition
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
-class S3ServerContainer(Container):
+class S3ServerContainer(LinuxContainer):
def __init__(self, test_context: MinifiTestContext):
super().__init__("adobe/s3mock:3.12.0", f"s3-server-{test_context.scenario_id}", test_context.network)
self.environment.append("initialBuckets=test_bucket")
diff --git a/extensions/azure/tests/features/steps/azure_server_container.py b/extensions/azure/tests/features/steps/azure_server_container.py
index 4caeb786b..cb0d2e5eb 100644
--- a/extensions/azure/tests/features/steps/azure_server_container.py
+++ b/extensions/azure/tests/features/steps/azure_server_container.py
@@ -18,13 +18,13 @@
import logging
from docker.errors import ContainerError
-from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.container_linux import LinuxContainer
from minifi_test_framework.core.helpers import run_cmd_in_docker_image
from minifi_test_framework.core.helpers import wait_for_condition
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
-class AzureServerContainer(Container):
+class AzureServerContainer(LinuxContainer):
def __init__(self, test_context: MinifiTestContext):
super().__init__("mcr.microsoft.com/azure-storage/azurite:3.35.0",
f"azure-storage-server-{test_context.scenario_id}",
diff --git a/extensions/couchbase/tests/features/steps/couchbase_server_container.py b/extensions/couchbase/tests/features/steps/couchbase_server_container.py
index 52228951d..c0471be50 100644
--- a/extensions/couchbase/tests/features/steps/couchbase_server_container.py
+++ b/extensions/couchbase/tests/features/steps/couchbase_server_container.py
@@ -15,26 +15,25 @@
import logging
-from OpenSSL import crypto
from minifi_test_framework.core.helpers import wait_for_condition, retry_check
-from minifi_test_framework.core.ssl_utils import make_server_cert
-from minifi_test_framework.containers.container import Container
+from minifi_test_framework.core.ssl_utils import make_server_cert, dump_cert, dump_key
+from minifi_test_framework.containers.container_linux import LinuxContainer
from minifi_test_framework.containers.file import File
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
from docker.errors import ContainerError
-class CouchbaseServerContainer(Container):
+class CouchbaseServerContainer(LinuxContainer):
def __init__(self, test_context: MinifiTestContext):
super().__init__("couchbase:enterprise-7.2.5", f"couchbase-server-{test_context.scenario_id}", test_context.network)
couchbase_cert, couchbase_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key)
- root_ca_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert)
+ root_ca_content = dump_cert(test_context.root_ca_cert)
self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/CA/root_ca.crt", root_ca_content, permissions=0o666))
- couchbase_cert_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=couchbase_cert)
+ couchbase_cert_content = dump_cert(couchbase_cert)
self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/chain.pem", couchbase_cert_content, permissions=0o666))
- couchbase_key_content = crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=couchbase_key)
+ couchbase_key_content = dump_key(couchbase_key)
self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/pkey.key", couchbase_key_content, permissions=0o666))
def deploy(self, context: MinifiTestContext | None) -> bool:
@@ -66,7 +65,7 @@ class CouchbaseServerContainer(Container):
return True
- @retry_check(max_tries=12, retry_interval=5)
+ @retry_check(12, 5)
def _run_couchbase_cli_command(self, command):
(code, output) = self.exec_run(command)
if code != 0:
@@ -90,7 +89,7 @@ class CouchbaseServerContainer(Container):
logging.error(f"Unexpected error while running python command '{command}' in couchbase helper docker: '{e}'")
return False
- @retry_check(max_tries=15, retry_interval=2)
+ @retry_check(15, 2)
def _load_couchbase_certs(self):
python_command = f"""
import requests
diff --git a/extensions/elasticsearch/tests/features/steps/elastic_base_container.py b/extensions/elasticsearch/tests/features/steps/elastic_base_container.py
index 5db6e61dc..2287278c4 100644
--- a/extensions/elasticsearch/tests/features/steps/elastic_base_container.py
+++ b/extensions/elasticsearch/tests/features/steps/elastic_base_container.py
@@ -14,11 +14,11 @@
# limitations under the License.
from minifi_test_framework.core.helpers import wait_for_condition, retry_check
-from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.container_linux import LinuxContainer
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
-class ElasticBaseContainer(Container):
+class ElasticBaseContainer(LinuxContainer):
def __init__(self, test_context: MinifiTestContext, image: str, container_name: str):
super().__init__(image, container_name, test_context.network)
self.user = None
diff --git a/extensions/elasticsearch/tests/features/steps/elasticsearch_container.py b/extensions/elasticsearch/tests/features/steps/elasticsearch_container.py
index 037711d75..aec928026 100644
--- a/extensions/elasticsearch/tests/features/steps/elasticsearch_container.py
+++ b/extensions/elasticsearch/tests/features/steps/elasticsearch_container.py
@@ -19,8 +19,7 @@ import os
from elastic_base_container import ElasticBaseContainer
from pathlib import Path
-from OpenSSL import crypto
-from minifi_test_framework.core.ssl_utils import make_server_cert, make_cert_without_extended_usage
+from minifi_test_framework.core.ssl_utils import make_server_cert, make_cert_without_extended_usage, dump_cert, dump_key
from minifi_test_framework.containers.file import File
from minifi_test_framework.containers.host_file import HostFile
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
@@ -33,19 +32,19 @@ class ElasticsearchContainer(ElasticBaseContainer):
http_cert, http_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key)
transport_cert, transport_key = make_cert_without_extended_usage(self.container_name, test_context.root_ca_cert, test_context.root_ca_key)
- root_ca_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert)
+ root_ca_content = dump_cert(test_context.root_ca_cert)
self.files.append(File("/usr/share/elasticsearch/config/certs/root_ca.crt", root_ca_content, permissions=0o644))
- http_cert_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=http_cert)
+ http_cert_content = dump_cert(http_cert)
self.files.append(File("/usr/share/elasticsearch/config/certs/elastic_http.crt", http_cert_content, permissions=0o644))
- http_key_content = crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=http_key)
+ http_key_content = dump_key(http_key)
self.files.append(File("/usr/share/elasticsearch/config/certs/elastic_http.key", http_key_content, permissions=0o644))
- transport_cert_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=transport_cert)
+ transport_cert_content = dump_cert(transport_cert)
self.files.append(File("/usr/share/elasticsearch/config/certs/elastic_transport.crt", transport_cert_content, permissions=0o644))
- transport_key_content = crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=transport_key)
+ transport_key_content = dump_key(transport_key)
self.files.append(File("/usr/share/elasticsearch/config/certs/elastic_transport.key", transport_key_content, permissions=0o644))
features_dir = Path(__file__).resolve().parent.parent
diff --git a/extensions/elasticsearch/tests/features/steps/opensearch_container.py b/extensions/elasticsearch/tests/features/steps/opensearch_container.py
index cdf5008d5..234a26d12 100644
--- a/extensions/elasticsearch/tests/features/steps/opensearch_container.py
+++ b/extensions/elasticsearch/tests/features/steps/opensearch_container.py
@@ -18,8 +18,7 @@ import logging
from elastic_base_container import ElasticBaseContainer
from pathlib import Path
-from OpenSSL import crypto
-from minifi_test_framework.core.ssl_utils import make_server_cert
+from minifi_test_framework.core.ssl_utils import make_server_cert, dump_cert, dump_key
from minifi_test_framework.containers.file import File
from minifi_test_framework.containers.host_file import HostFile
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
@@ -31,13 +30,13 @@ class OpensearchContainer(ElasticBaseContainer):
admin_pem, admin_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key)
- root_ca_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert)
+ root_ca_content = dump_cert(test_context.root_ca_cert)
self.files.append(File("/usr/share/opensearch/config/root-ca.pem", root_ca_content, permissions=0o644))
- admin_pem_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=admin_pem)
+ admin_pem_content = dump_cert(admin_pem)
self.files.append(File("/usr/share/opensearch/config/admin.pem", admin_pem_content, permissions=0o644))
- admin_key_content = crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=admin_key)
+ admin_key_content = dump_key(admin_key)
self.files.append(File("/usr/share/opensearch/config/admin-key.pem", admin_key_content, permissions=0o644))
features_dir = Path(__file__).resolve().parent.parent
diff --git a/extensions/gcp/tests/features/steps/fake_gcs_server_container.py b/extensions/gcp/tests/features/steps/fake_gcs_server_container.py
index a9733ba4e..5689065bb 100644
--- a/extensions/gcp/tests/features/steps/fake_gcs_server_container.py
+++ b/extensions/gcp/tests/features/steps/fake_gcs_server_container.py
@@ -15,12 +15,12 @@
import logging
from minifi_test_framework.core.helpers import wait_for_condition, retry_check
-from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.container_linux import LinuxContainer
from minifi_test_framework.containers.directory import Directory
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
-class FakeGcsServerContainer(Container):
+class FakeGcsServerContainer(LinuxContainer):
def __init__(self, test_context: MinifiTestContext):
super().__init__("fsouza/fake-gcs-server:1.45.1", f"fake-gcs-server-{test_context.scenario_id}", test_context.network,
command=f'-scheme http -host fake-gcs-server-{test_context.scenario_id}')
diff --git a/extensions/grafana-loki/tests/features/steps/grafana_loki_container.py b/extensions/grafana-loki/tests/features/steps/grafana_loki_container.py
index 1284b95b0..10c77defd 100644
--- a/extensions/grafana-loki/tests/features/steps/grafana_loki_container.py
+++ b/extensions/grafana-loki/tests/features/steps/grafana_loki_container.py
@@ -14,13 +14,12 @@
# limitations under the License.
import logging
-from OpenSSL import crypto
from minifi_test_framework.core.helpers import wait_for_condition, retry_check
-from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.container_linux import LinuxContainer
from minifi_test_framework.containers.file import File
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
-from minifi_test_framework.core.ssl_utils import make_server_cert
+from minifi_test_framework.core.ssl_utils import make_server_cert, dump_cert, dump_key
from docker.errors import ContainerError
@@ -30,20 +29,20 @@ class GrafanaLokiOptions:
self.enable_multi_tenancy = enable_multi_tenancy
-class GrafanaLokiContainer(Container):
+class GrafanaLokiContainer(LinuxContainer):
def __init__(self, test_context: MinifiTestContext, options: GrafanaLokiOptions):
super().__init__("grafana/loki:3.2.1", f"grafana-loki-server-{test_context.scenario_id}", test_context.network)
extra_ssl_settings = ""
if options.enable_ssl:
grafana_loki_cert, grafana_loki_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key)
- root_ca_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert)
+ root_ca_content = dump_cert(test_context.root_ca_cert)
self.files.append(File("/etc/loki/root_ca.crt", root_ca_content, permissions=0o644))
- grafana_loki_cert_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=grafana_loki_cert)
+ grafana_loki_cert_content = dump_cert(grafana_loki_cert)
self.files.append(File("/etc/loki/cert.pem", grafana_loki_cert_content, permissions=0o644))
- grafana_loki_key_content = crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=grafana_loki_key)
+ grafana_loki_key_content = dump_key(grafana_loki_key)
self.files.append(File("/etc/loki/key.pem", grafana_loki_key_content, permissions=0o644))
extra_ssl_settings = """
diff --git a/extensions/grafana-loki/tests/features/steps/reverse_proxy_container.py b/extensions/grafana-loki/tests/features/steps/reverse_proxy_container.py
index 409a9b9e6..43efd49c5 100644
--- a/extensions/grafana-loki/tests/features/steps/reverse_proxy_container.py
+++ b/extensions/grafana-loki/tests/features/steps/reverse_proxy_container.py
@@ -14,11 +14,11 @@
# limitations under the License.
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
-from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.container_linux import LinuxContainer
from minifi_test_framework.core.helpers import wait_for_condition
-class ReverseProxyContainer(Container):
+class ReverseProxyContainer(LinuxContainer):
def __init__(self, test_context: MinifiTestContext):
super().__init__("minifi-reverse-proxy:latest", f"reverse-proxy-{test_context.scenario_id}", test_context.network)
self.environment = [
diff --git a/extensions/kafka/tests/features/steps/kafka_server_container.py b/extensions/kafka/tests/features/steps/kafka_server_container.py
index 30073186f..09e35e131 100644
--- a/extensions/kafka/tests/features/steps/kafka_server_container.py
+++ b/extensions/kafka/tests/features/steps/kafka_server_container.py
@@ -15,17 +15,18 @@
import logging
import re
+
+from cryptography.hazmat.primitives import serialization
import jks
-from OpenSSL import crypto
from minifi_test_framework.core.helpers import wait_for_condition
-from minifi_test_framework.core.ssl_utils import make_server_cert
-from minifi_test_framework.containers.container import Container
+from minifi_test_framework.core.ssl_utils import make_server_cert, dump_cert, dump_key
+from minifi_test_framework.containers.container_linux import LinuxContainer
from minifi_test_framework.containers.file import File
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
-class KafkaServer(Container):
+class KafkaServer(LinuxContainer):
def __init__(self, test_context: MinifiTestContext):
super().__init__("apache/kafka:4.1.0", f"kafka-server-{test_context.scenario_id}", test_context.network)
self.environment.append("KAFKA_NODE_ID=1")
@@ -55,7 +56,7 @@ class KafkaServer(Container):
kafka_cert, kafka_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key)
- pke = jks.PrivateKeyEntry.new('kafka-broker-cert', [crypto.dump_certificate(crypto.FILETYPE_ASN1, kafka_cert)], crypto.dump_privatekey(crypto.FILETYPE_ASN1, kafka_key), 'rsa_raw')
+ pke = jks.PrivateKeyEntry.new('kafka-broker-cert', [dump_cert(kafka_cert, encoding_type=serialization.Encoding.DER)], dump_key(kafka_key, encoding_type=serialization.Encoding.DER), 'rsa_raw')
server_keystore = jks.KeyStore.new('jks', [pke])
server_keystore_content = server_keystore.saves('abcdefgh')
self.files.append(File("/etc/kafka/secrets/kafka.keystore.jks", server_keystore_content, permissions=0o644))
@@ -63,7 +64,7 @@ class KafkaServer(Container):
trusted_cert = jks.TrustedCertEntry.new(
'root-ca', # Alias for the certificate
- crypto.dump_certificate(crypto.FILETYPE_ASN1, test_context.root_ca_cert)
+ dump_cert(test_context.root_ca_cert, encoding_type=serialization.Encoding.DER)
)
# Create a JKS keystore that will serve as a truststore with the trusted cert entry.
diff --git a/extensions/opc/tests/features/steps/opc_ua_server_container.py b/extensions/opc/tests/features/steps/opc_ua_server_container.py
index 868e54202..c9ba312ec 100644
--- a/extensions/opc/tests/features/steps/opc_ua_server_container.py
+++ b/extensions/opc/tests/features/steps/opc_ua_server_container.py
@@ -14,12 +14,12 @@
# limitations under the License.
from typing import List, Optional
-from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.container_linux import LinuxContainer
from minifi_test_framework.core.helpers import wait_for_condition
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
-class OPCUAServerContainer(Container):
+class OPCUAServerContainer(LinuxContainer):
def __init__(self, test_context: MinifiTestContext, command: Optional[List[str]] = None):
super().__init__("lordgamez/open62541:1.4.10", f"opcua-server-{test_context.scenario_id}", test_context.network, command=command)
diff --git a/extensions/splunk/tests/features/steps/splunk_container.py b/extensions/splunk/tests/features/steps/splunk_container.py
index 4106bb63e..9a1488e63 100644
--- a/extensions/splunk/tests/features/steps/splunk_container.py
+++ b/extensions/splunk/tests/features/steps/splunk_container.py
@@ -15,15 +15,14 @@
import json
-from OpenSSL import crypto
-from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.container_linux import LinuxContainer
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
from minifi_test_framework.core.helpers import wait_for_condition, retry_check
from minifi_test_framework.containers.file import File
-from minifi_test_framework.core.ssl_utils import make_server_cert
+from minifi_test_framework.core.ssl_utils import make_server_cert, dump_cert, dump_key
-class SplunkContainer(Container):
+class SplunkContainer(LinuxContainer):
def __init__(self, test_context: MinifiTestContext):
super().__init__("splunk/splunk:9.2.1-patch2", f"splunk-{test_context.scenario_id}", test_context.network)
self.user = None
@@ -43,9 +42,9 @@ splunk:
self.files.append(File("/tmp/defaults/default.yml", splunk_config_content, mode="rw", permissions=0o644))
splunk_cert, splunk_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key)
- splunk_cert_content = crypto.dump_certificate(crypto.FILETYPE_PEM, splunk_cert)
- splunk_key_content = crypto.dump_privatekey(crypto.FILETYPE_PEM, splunk_key)
- root_ca_content = crypto.dump_certificate(crypto.FILETYPE_PEM, test_context.root_ca_cert)
+ splunk_cert_content = dump_cert(splunk_cert)
+ splunk_key_content = dump_key(splunk_key)
+ root_ca_content = dump_cert(test_context.root_ca_cert)
self.files.append(File("/opt/splunk/etc/auth/splunk_cert.pem", splunk_cert_content.decode() + splunk_key_content.decode() + root_ca_content.decode(), permissions=0o644))
self.files.append(File("/opt/splunk/etc/auth/root_ca.pem", root_ca_content.decode(), permissions=0o644))
diff --git a/extensions/sql/tests/features/steps/postgress_server_container.py b/extensions/sql/tests/features/steps/postgress_server_container.py
index 467eecc4b..a61209528 100644
--- a/extensions/sql/tests/features/steps/postgress_server_container.py
+++ b/extensions/sql/tests/features/steps/postgress_server_container.py
@@ -17,13 +17,13 @@
import logging
from textwrap import dedent
-from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.container_linux import LinuxContainer
from minifi_test_framework.containers.docker_image_builder import DockerImageBuilder
from minifi_test_framework.core.helpers import wait_for_condition
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
-class PostgresContainer(Container):
+class PostgresContainer(LinuxContainer):
def __init__(self, context):
dockerfile = dedent("""\
FROM {base_image}
diff --git a/extensions/standard-processors/tests/features/core_functionality.feature b/extensions/standard-processors/tests/features/core_functionality.feature
index 4b5f6b38b..0acdbb576 100644
--- a/extensions/standard-processors/tests/features/core_functionality.feature
+++ b/extensions/standard-processors/tests/features/core_functionality.feature
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-@CORE
+@CORE @SUPPORTS_WINDOWS
Feature: Core flow functionalities
Test core flow configuration functionalities
@@ -33,8 +33,8 @@ Feature: Core flow functionalities
When all instances start up
- Then at least one file with the content "first_custom_text" is placed in the "/tmp/output" directory in less than 20 seconds
- And at least one file with the content "second_custom_text" is placed in the "/tmp/output" directory in less than 20 seconds
+ Then at least one file with the content "first_custom_text" is placed in the "/tmp/output" directory in less than 200 seconds
+ And at least one file with the content "second_custom_text" is placed in the "/tmp/output" directory in less than 200 seconds
Scenario: A funnel can be used as a terminator
Given a GenerateFlowFile processor with the "Data Format" property set to "Text"
@@ -77,7 +77,7 @@ Feature: Core flow functionalities
And a non-sensitive parameter in the flow config called 'FILE_INPUT_PATH' with the value '/tmp/input' in the parameter context 'my-context'
And a non-sensitive parameter in the flow config called 'FILE_OUTPUT_UPPER_PATH_ATTR' with the value 'upper_out_path_attr' in the parameter context 'my-context'
And a GetFile processor with the "Input Directory" property set to "#{FILE_INPUT_PATH}"
- And a file with filename "test_file_name" and content "test content" is present in "/tmp/input"
+ And a directory at "/tmp/input" has a file ("test_file_name") with the content "test content"
And a UpdateAttribute processor with the "expr-lang-filename" property set to "#{FILENAME}"
And the "is-upper-correct" property of the UpdateAttribute processor is set to "${#{FILENAME_IN_EXPRESSION}:toUpper():equals('TEST_FILE_NAME')}"
And the "upper_out_path_attr" property of the UpdateAttribute processor is set to "/TMP/OUTPUT"
diff --git a/extensions/standard-processors/tests/features/replace_text.feature b/extensions/standard-processors/tests/features/replace_text.feature
index f8a3fdb05..b24a0e873 100644
--- a/extensions/standard-processors/tests/features/replace_text.feature
+++ b/extensions/standard-processors/tests/features/replace_text.feature
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-@CORE
+@CORE @SUPPORTS_WINDOWS
Feature: Changing flowfile contents using the ReplaceText processor
Scenario Outline: Replace text using Entire text mode
diff --git a/extensions/standard-processors/tests/features/steps/diag_slave_container.py b/extensions/standard-processors/tests/features/steps/diag_slave_container.py
index ba89e75c3..8defe490f 100644
--- a/extensions/standard-processors/tests/features/steps/diag_slave_container.py
+++ b/extensions/standard-processors/tests/features/steps/diag_slave_container.py
@@ -18,13 +18,13 @@
import logging
from textwrap import dedent
-from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.container_linux import LinuxContainer
from minifi_test_framework.containers.docker_image_builder import DockerImageBuilder
from minifi_test_framework.core.helpers import wait_for_condition
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
-class DiagSlave(Container):
+class DiagSlave(LinuxContainer):
def __init__(self, test_context: MinifiTestContext):
dockerfile = dedent("""\
FROM panterdsd/diagslave:latest
diff --git a/extensions/standard-processors/tests/features/steps/minifi_controller_steps.py b/extensions/standard-processors/tests/features/steps/minifi_controller_steps.py
index c65d67e0f..c53df9b02 100644
--- a/extensions/standard-processors/tests/features/steps/minifi_controller_steps.py
+++ b/extensions/standard-processors/tests/features/steps/minifi_controller_steps.py
@@ -22,52 +22,52 @@ from minifi_test_framework.core.minifi_test_context import MinifiTestContext
@given('controller socket properties are set up')
def step_impl(context: MinifiTestContext):
- context.get_or_create_default_minifi_container().set_controller_socket_properties()
+ context.get_or_create_default_minifi_container().controller.set_controller_socket_properties()
@when('MiNiFi config is updated through MiNiFi controller')
def step_impl(context: MinifiTestContext):
- context.get_or_create_default_minifi_container().update_flow_config_through_controller()
+ context.get_or_create_default_minifi_container().controller.update_flow_config_through_controller()
@then('the updated config is persisted')
def step_impl(context: MinifiTestContext):
- assert context.get_or_create_default_minifi_container().updated_config_is_persisted()
+ assert context.get_or_create_default_minifi_container().controller.updated_config_is_persisted()
@when('the {component} component is stopped through MiNiFi controller')
def step_impl(context: MinifiTestContext, component: str):
- context.get_or_create_default_minifi_container().stop_component_through_controller(component)
+ context.get_or_create_default_minifi_container().controller.stop_component_through_controller(component)
@when('the {component} component is started through MiNiFi controller')
def step_impl(context: MinifiTestContext, component: str):
- context.get_or_create_default_minifi_container().start_component_through_controller(component)
+ context.get_or_create_default_minifi_container().controller.start_component_through_controller(component)
@then('the {component} component is not running')
def step_impl(context: MinifiTestContext, component: str):
- assert not context.get_or_create_default_minifi_container().is_component_running(component)
+ assert not context.get_or_create_default_minifi_container().controller.is_component_running(component)
@then('the {component} component is running')
def step_impl(context: MinifiTestContext, component: str):
- assert context.get_or_create_default_minifi_container().is_component_running(component)
+ assert context.get_or_create_default_minifi_container().controller.is_component_running(component)
@then('connection \"{connection}\" can be seen through MiNiFi controller')
def step_impl(context: MinifiTestContext, connection: str):
- assert context.get_or_create_default_minifi_container().connection_found_through_controller(connection)
+ assert context.get_or_create_default_minifi_container().controller.connection_found_through_controller(connection)
@then('{connection_count:d} connections can be seen full through MiNiFi controller')
def step_impl(context: MinifiTestContext, connection_count: int):
- assert context.get_or_create_default_minifi_container().get_full_connection_count() == connection_count
+ assert context.get_or_create_default_minifi_container().controller.get_full_connection_count() == connection_count
@retry_check(10, 1)
def check_connection_size_through_controller(context: MinifiTestContext, connection: str, size: int, max_size: int) -> bool:
- return context.get_or_create_default_minifi_container().get_connection_size(connection) == (size, max_size)
+ return context.get_or_create_default_minifi_container().controller.get_connection_size(connection) == (size, max_size)
@then('connection \"{connection}\" has {size:d} size and {max_size:d} max size through MiNiFi controller')
@@ -77,7 +77,7 @@ def step_impl(context: MinifiTestContext, connection: str, size: int, max_size:
@retry_check(10, 1)
def manifest_can_be_retrieved_through_minifi_controller(context: MinifiTestContext) -> bool:
- manifest = context.get_or_create_default_minifi_container().get_manifest()
+ manifest = context.get_or_create_default_minifi_container().controller.get_manifest()
return '"agentManifest": {' in manifest and '"componentManifest": {' in manifest and '"agentType": "cpp"' in manifest
@@ -88,4 +88,4 @@ def step_impl(context: MinifiTestContext):
@then('debug bundle can be retrieved through MiNiFi controller')
def step_impl(context: MinifiTestContext):
- assert context.get_or_create_default_minifi_container().create_debug_bundle()
+ assert context.get_or_create_default_minifi_container().controller.create_debug_bundle()
diff --git a/extensions/standard-processors/tests/features/steps/syslog_container.py b/extensions/standard-processors/tests/features/steps/syslog_container.py
index f2da956f5..c831af6f1 100644
--- a/extensions/standard-processors/tests/features/steps/syslog_container.py
+++ b/extensions/standard-processors/tests/features/steps/syslog_container.py
@@ -15,10 +15,10 @@
# limitations under the License.
#
-from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.container_linux import LinuxContainer
-class SyslogContainer(Container):
+class SyslogContainer(LinuxContainer):
def __init__(self, protocol, context):
super(SyslogContainer, self).__init__("ubuntu:24.04", f"syslog-{protocol}-{context.scenario_id}", context.network)
self.command = f'/bin/bash -c "echo Syslog {protocol} client started; while true; do logger --{protocol} -n minifi-primary-{context.scenario_id} -P 514 sample_log; sleep 1; done"'
diff --git a/extensions/standard-processors/tests/features/steps/tcp_client_container.py b/extensions/standard-processors/tests/features/steps/tcp_client_container.py
index aca77dfae..0d90f3622 100644
--- a/extensions/standard-processors/tests/features/steps/tcp_client_container.py
+++ b/extensions/standard-processors/tests/features/steps/tcp_client_container.py
@@ -13,12 +13,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.container_linux import LinuxContainer
from minifi_test_framework.core.helpers import wait_for_condition
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
-class TcpClientContainer(Container):
+class TcpClientContainer(LinuxContainer):
def __init__(self, test_context: MinifiTestContext):
cmd = (
"/bin/sh -c 'apk add netcat-openbsd && "
diff --git a/run_flake8.sh b/run_flake8.sh
index 2288893bc..1effd79ae 100755
--- a/run_flake8.sh
+++ b/run_flake8.sh
@@ -19,4 +19,4 @@
set -euo pipefail
directory=${1:-.}
-flake8 --exclude .venv,venv,thirdparty,build,cmake-build-*,github_env --builtins log,REL_SUCCESS,REL_FAILURE,REL_ORIGINAL,raw_input --ignore E501,W503,F811 "${directory}"
+flake8 --exclude .venv,venv,venv2,behave-venv,thirdparty,build,cmake-build-*,github_env --builtins log,REL_SUCCESS,REL_FAILURE,REL_ORIGINAL,raw_input --ignore E501,W503,F811 "${directory}"