You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@nifi.apache.org by "lordgamez (via GitHub)" <gi...@apache.org> on 2026/01/06 12:32:03 UTC
Re: [PR] MINIFICPP-2674 Move Couchbase tests to modular docker tests [nifi-minifi-cpp]
lordgamez commented on code in PR #2066:
URL: https://github.com/apache/nifi-minifi-cpp/pull/2066#discussion_r2664778224
##########
behave_framework/src/minifi_test_framework/core/helpers.py:
##########
@@ -82,3 +83,16 @@ def run_cmd_in_docker_image(image_name: str, cmd: str | list, network: str) -> s
def run_shell_cmd_in_docker_image(image_name: str, cmd: str, network: str) -> str:
return run_cmd_in_docker_image(image_name, ["/bin/sh", "-c", cmd], network)
+
+
+def retry_check(max_tries=5, retry_interval=1):
+ def retry_check_func(func):
+ @functools.wraps(func)
+ def retry_wrapper(*args, **kwargs):
+ for _ in range(max_tries):
+ if func(*args, **kwargs):
+ return True
+ time.sleep(retry_interval)
+ return False
+ return retry_wrapper
+ return retry_check_func
Review Comment:
Updated in https://github.com/apache/nifi-minifi-cpp/pull/2066/commits/b769649e4888110f277d7564ee164fec17548b92
##########
extensions/couchbase/tests/features/couchbase.feature:
##########
@@ -220,28 +248,33 @@ Feature: Executing Couchbase operations from MiNiFi-C++
Scenario: A MiNiFi instance can get data from test bucket with GetCouchbaseKey processor using mTLS authentication
Given a MiNiFi CPP server with yaml config
Review Comment:
I don't see why it is significant here, so I removed it in https://github.com/apache/nifi-minifi-cpp/pull/2066/commits/b769649e4888110f277d7564ee164fec17548b92. Currently all modular docker tests use YAML configuration and the JSON support will only be introduced in a later PR. After that we can check where it is worth running tests with both JSON and YAML configurations.
##########
extensions/couchbase/tests/features/steps/couchbase_server_container.py:
##########
@@ -0,0 +1,157 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from OpenSSL import crypto
+from minifi_test_framework.core.helpers import wait_for_condition, retry_check
+from minifi_test_framework.core.ssl_utils import make_server_cert
+from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.file import File
+from minifi_test_framework.core.minifi_test_context import MinifiTestContext
+
+
+class CouchbaseServerContainer(Container):
+ def __init__(self, test_context: MinifiTestContext):
+ super().__init__("couchbase:enterprise-7.2.5", f"couchbase-server-{test_context.scenario_id}", test_context.network)
+
+ couchbase_cert, couchbase_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key)
+
+ root_ca_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert)
+ self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/CA/root_ca.crt", root_ca_content, permissions=0o666))
+ couchbase_cert_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=couchbase_cert)
+ self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/chain.pem", couchbase_cert_content, permissions=0o666))
+ couchbase_key_content = crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=couchbase_key)
+ self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/pkey.key", couchbase_key_content, permissions=0o666))
+
+ def deploy(self):
+ super().deploy()
+ finished_str = "logs available in"
+ assert wait_for_condition(
+ condition=lambda: finished_str in self.get_logs(),
+ timeout_seconds=15,
+ bail_condition=lambda: self.exited,
+ context=None)
+ return self.run_post_startup_commands()
+
+ def run_post_startup_commands(self):
+ commands = [
+ ["couchbase-cli", "cluster-init", "-c", "localhost", "--cluster-username", "Administrator", "--cluster-password", "password123", "--services", "data,index,query",
+ "--cluster-ramsize", "2048", "--cluster-index-ramsize", "256"],
+ ["couchbase-cli", "bucket-create", "-c", "localhost", "--username", "Administrator", "--password", "password123", "--bucket", "test_bucket", "--bucket-type", "couchbase",
+ "--bucket-ramsize", "1024", "--max-ttl", "36000"],
+ ["couchbase-cli", "user-manage", "-c", "localhost", "-u", "Administrator", "-p", "password123", "--set", "--rbac-username", "clientuser", "--rbac-password", "password123",
+ "--roles", "data_reader[test_bucket],data_writer[test_bucket]", "--auth-domain", "local"],
+ ["bash", "-c", 'tee /tmp/auth.json <<< \'{"state": "enable", "prefixes": [ {"path": "subject.cn", "prefix": "", "delimiter": "."}]}\''],
+ ['couchbase-cli', 'ssl-manage', '-c', 'localhost', '-u', 'Administrator', '-p', 'password123', '--set-client-auth', '/tmp/auth.json']
+ ]
+ if not self._run_couchbase_cli_commands(commands):
+ return False
+
+ if not self._load_couchbase_certs():
+ return False
+
+ return True
+
+ @retry_check(max_tries=12, retry_interval=5)
+ def _run_couchbase_cli_command(self, command):
+ (code, _) = self.exec_run(command)
+ if code != 0:
+ logging.error(f"Failed to run command '{command}', returned error code: {code}")
Review Comment:
Added in https://github.com/apache/nifi-minifi-cpp/pull/2066/commits/b769649e4888110f277d7564ee164fec17548b92
##########
extensions/couchbase/tests/features/steps/couchbase_server_container.py:
##########
@@ -0,0 +1,157 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from OpenSSL import crypto
+from minifi_test_framework.core.helpers import wait_for_condition, retry_check
+from minifi_test_framework.core.ssl_utils import make_server_cert
+from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.file import File
+from minifi_test_framework.core.minifi_test_context import MinifiTestContext
+
+
+class CouchbaseServerContainer(Container):
+ def __init__(self, test_context: MinifiTestContext):
+ super().__init__("couchbase:enterprise-7.2.5", f"couchbase-server-{test_context.scenario_id}", test_context.network)
+
+ couchbase_cert, couchbase_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key)
+
+ root_ca_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert)
+ self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/CA/root_ca.crt", root_ca_content, permissions=0o666))
+ couchbase_cert_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=couchbase_cert)
+ self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/chain.pem", couchbase_cert_content, permissions=0o666))
+ couchbase_key_content = crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=couchbase_key)
+ self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/pkey.key", couchbase_key_content, permissions=0o666))
+
+ def deploy(self):
+ super().deploy()
+ finished_str = "logs available in"
+ assert wait_for_condition(
+ condition=lambda: finished_str in self.get_logs(),
+ timeout_seconds=15,
+ bail_condition=lambda: self.exited,
+ context=None)
+ return self.run_post_startup_commands()
+
+ def run_post_startup_commands(self):
+ commands = [
+ ["couchbase-cli", "cluster-init", "-c", "localhost", "--cluster-username", "Administrator", "--cluster-password", "password123", "--services", "data,index,query",
+ "--cluster-ramsize", "2048", "--cluster-index-ramsize", "256"],
+ ["couchbase-cli", "bucket-create", "-c", "localhost", "--username", "Administrator", "--password", "password123", "--bucket", "test_bucket", "--bucket-type", "couchbase",
+ "--bucket-ramsize", "1024", "--max-ttl", "36000"],
+ ["couchbase-cli", "user-manage", "-c", "localhost", "-u", "Administrator", "-p", "password123", "--set", "--rbac-username", "clientuser", "--rbac-password", "password123",
+ "--roles", "data_reader[test_bucket],data_writer[test_bucket]", "--auth-domain", "local"],
+ ["bash", "-c", 'tee /tmp/auth.json <<< \'{"state": "enable", "prefixes": [ {"path": "subject.cn", "prefix": "", "delimiter": "."}]}\''],
+ ['couchbase-cli', 'ssl-manage', '-c', 'localhost', '-u', 'Administrator', '-p', 'password123', '--set-client-auth', '/tmp/auth.json']
+ ]
+ if not self._run_couchbase_cli_commands(commands):
+ return False
+
+ if not self._load_couchbase_certs():
+ return False
+
+ return True
+
+ @retry_check(max_tries=12, retry_interval=5)
+ def _run_couchbase_cli_command(self, command):
+ (code, _) = self.exec_run(command)
+ if code != 0:
+ logging.error(f"Failed to run command '{command}', returned error code: {code}")
+ return False
+ return True
+
+ def _run_couchbase_cli_commands(self, commands):
+ return all(self._run_couchbase_cli_command(command) for command in commands)
+
+ def _run_python_in_couchbase_helper_docker(self, command: str):
+ try:
+ self.client.containers.run("minifi-couchbase-helper:latest", ["python", "-c", command], remove=True, stdout=True, stderr=True, network=self.network.name)
+ return True
+ except Exception as e:
+ logging.error(f"Python command '{command}' failed in couchbase helper docker: {e}")
Review Comment:
Yes it is possible in some cases, added in case of ContainerError in https://github.com/apache/nifi-minifi-cpp/pull/2066/commits/b769649e4888110f277d7564ee164fec17548b92
##########
extensions/couchbase/tests/features/steps/couchbase_server_container.py:
##########
@@ -0,0 +1,157 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from OpenSSL import crypto
+from minifi_test_framework.core.helpers import wait_for_condition, retry_check
+from minifi_test_framework.core.ssl_utils import make_server_cert
+from minifi_test_framework.containers.container import Container
+from minifi_test_framework.containers.file import File
+from minifi_test_framework.core.minifi_test_context import MinifiTestContext
+
+
+class CouchbaseServerContainer(Container):
+ def __init__(self, test_context: MinifiTestContext):
+ super().__init__("couchbase:enterprise-7.2.5", f"couchbase-server-{test_context.scenario_id}", test_context.network)
+
+ couchbase_cert, couchbase_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key)
+
+ root_ca_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert)
+ self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/CA/root_ca.crt", root_ca_content, permissions=0o666))
+ couchbase_cert_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=couchbase_cert)
+ self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/chain.pem", couchbase_cert_content, permissions=0o666))
+ couchbase_key_content = crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=couchbase_key)
+ self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/pkey.key", couchbase_key_content, permissions=0o666))
+
+ def deploy(self):
+ super().deploy()
+ finished_str = "logs available in"
+ assert wait_for_condition(
+ condition=lambda: finished_str in self.get_logs(),
+ timeout_seconds=15,
+ bail_condition=lambda: self.exited,
+ context=None)
+ return self.run_post_startup_commands()
+
+ def run_post_startup_commands(self):
+ commands = [
+ ["couchbase-cli", "cluster-init", "-c", "localhost", "--cluster-username", "Administrator", "--cluster-password", "password123", "--services", "data,index,query",
+ "--cluster-ramsize", "2048", "--cluster-index-ramsize", "256"],
+ ["couchbase-cli", "bucket-create", "-c", "localhost", "--username", "Administrator", "--password", "password123", "--bucket", "test_bucket", "--bucket-type", "couchbase",
+ "--bucket-ramsize", "1024", "--max-ttl", "36000"],
+ ["couchbase-cli", "user-manage", "-c", "localhost", "-u", "Administrator", "-p", "password123", "--set", "--rbac-username", "clientuser", "--rbac-password", "password123",
+ "--roles", "data_reader[test_bucket],data_writer[test_bucket]", "--auth-domain", "local"],
+ ["bash", "-c", 'tee /tmp/auth.json <<< \'{"state": "enable", "prefixes": [ {"path": "subject.cn", "prefix": "", "delimiter": "."}]}\''],
+ ['couchbase-cli', 'ssl-manage', '-c', 'localhost', '-u', 'Administrator', '-p', 'password123', '--set-client-auth', '/tmp/auth.json']
+ ]
+ if not self._run_couchbase_cli_commands(commands):
+ return False
+
+ if not self._load_couchbase_certs():
+ return False
+
+ return True
+
+ @retry_check(max_tries=12, retry_interval=5)
+ def _run_couchbase_cli_command(self, command):
+ (code, _) = self.exec_run(command)
+ if code != 0:
+ logging.error(f"Failed to run command '{command}', returned error code: {code}")
+ return False
+ return True
+
+ def _run_couchbase_cli_commands(self, commands):
+ return all(self._run_couchbase_cli_command(command) for command in commands)
+
+ def _run_python_in_couchbase_helper_docker(self, command: str):
+ try:
+ self.client.containers.run("minifi-couchbase-helper:latest", ["python", "-c", command], remove=True, stdout=True, stderr=True, network=self.network.name)
+ return True
+ except Exception as e:
+ logging.error(f"Python command '{command}' failed in couchbase helper docker: {e}")
+ return False
+
+ @retry_check(max_tries=15, retry_interval=2)
+ def _load_couchbase_certs(self):
+ python_command = f"""
+import requests
+import sys
+from requests.auth import HTTPBasicAuth
+response = requests.post(f"http://{self.container_name}:8091/node/controller/loadTrustedCAs", auth=HTTPBasicAuth("Administrator", "password123"))
+if response.status_code != 200:
+ sys.exit(1)
+
+response = requests.post(f"http://{self.container_name}:8091/node/controller/reloadCertificate", auth=HTTPBasicAuth("Administrator", "password123"))
+if response.status_code != 200:
+ sys.exit(1)
+sys.exit(0)
+ """
+ return self._run_python_in_couchbase_helper_docker(python_command)
+
+ def is_data_present_in_couchbase(self, doc_id: str, bucket_name: str, expected_data: str, expected_data_type: str):
+ python_command = f"""
+from couchbase.cluster import Cluster
+from couchbase.options import ClusterOptions
+from couchbase.auth import PasswordAuthenticator
+from couchbase.transcoder import RawBinaryTranscoder, RawStringTranscoder
+import json
+import sys
+
+try:
+ cluster = Cluster("couchbase://{self.container_name}", ClusterOptions(
+ PasswordAuthenticator('Administrator', 'password123')))
+
+ bucket = cluster.bucket("{bucket_name}")
+ collection = bucket.default_collection()
+
+ if "{expected_data_type}".lower() == "binary":
+ binary_flag = 0x03 << 24
+ result = collection.get("{doc_id}", transcoder=RawBinaryTranscoder())
+ flags = result.flags
+ if not flags & binary_flag:
+ sys.exit(1)
+
+ content = result.content_as[bytes]
+ if content.decode('utf-8') == '{expected_data}':
+ sys.exit(0)
+
+ if "{expected_data_type}".lower() == "json":
+ json_flag = 0x02 << 24
+ result = collection.get("{doc_id}")
+ flags = result.flags
+ if not flags & json_flag:
+ sys.exit(1)
+
+ content = result.content_as[dict]
+ if content == json.loads('{expected_data}'):
+ sys.exit(0)
+ if "{expected_data_type}".lower() == "string":
+ string_flag = 0x04 << 24
+ result = collection.get("{doc_id}", transcoder=RawStringTranscoder())
+ flags = result.flags
+ if not flags & string_flag:
+ print("FAILURE")
Review Comment:
Good point, added in https://github.com/apache/nifi-minifi-cpp/pull/2066/commits/b769649e4888110f277d7564ee164fec17548b92 which should be logged in case of ContainerError
##########
extensions/couchbase/tests/features/steps/steps.py:
##########
@@ -0,0 +1,67 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from behave import step, then
+
+from minifi_test_framework.steps import checking_steps # noqa: F401
+from minifi_test_framework.steps import configuration_steps # noqa: F401
+from minifi_test_framework.steps import core_steps # noqa: F401
+from minifi_test_framework.steps import flow_building_steps # noqa: F401
+from minifi_test_framework.steps.flow_building_steps import add_ssl_context_service_for_minifi
+from minifi_test_framework.core.minifi_test_context import MinifiTestContext
+from minifi_test_framework.core.helpers import log_due_to_failure
+from minifi_test_framework.minifi.controller_service import ControllerService
+from couchbase_server_container import CouchbaseServerContainer
+
+
+@step("a Couchbase server is started")
+def step_impl(context: MinifiTestContext):
+ context.containers["couchbase-server"] = CouchbaseServerContainer(context)
+ assert context.containers["couchbase-server"].deploy()
+
+
+@step("a CouchbaseClusterService controller service is set up to communicate with the Couchbase server")
+def step_impl(context: MinifiTestContext):
+ controller_service = ControllerService(class_name="CouchbaseClusterService", service_name="CouchbaseClusterService")
+ controller_service.add_property("Connection String", f"couchbase://couchbase-server-{context.scenario_id}")
+ controller_service.add_property("User Name", "Administrator")
+ controller_service.add_property("User Password", "password123")
+ context.get_or_create_default_minifi_container().flow_definition.controller_services.append(controller_service)
+
+
+@step("a CouchbaseClusterService is set up using SSL connection")
+def step_impl(context):
+ controller_service = ControllerService(class_name="SSLContextService", service_name="SSLContextService")
+ controller_service.add_property("CA Certificate", "/tmp/resources/root_ca.crt")
+ context.get_or_create_default_minifi_container().flow_definition.controller_services.append(controller_service)
+ controller_service = ControllerService(class_name="CouchbaseClusterService", service_name="CouchbaseClusterService")
+ controller_service.add_property("Connection String", f"couchbases://couchbase-server-{context.scenario_id}")
+ controller_service.add_property("User Name", "Administrator")
+ controller_service.add_property("User Password", "password123")
+ controller_service.add_property("Linked Services", "SSLContextService")
+ context.get_or_create_default_minifi_container().flow_definition.controller_services.append(controller_service)
Review Comment:
Good catch, renamed in https://github.com/apache/nifi-minifi-cpp/pull/2066/commits/b769649e4888110f277d7564ee164fec17548b92
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@nifi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org