You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ab...@apache.org on 2021/03/08 13:13:04 UTC
[nifi-minifi-cpp] branch main updated (287f184 -> 5b6d78d)
This is an automated email from the ASF dual-hosted git repository.
aboda pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git.
from 287f184 MINIFICPP-1496 Remove ASAN from the list of extensions in bootstrap.sh
new 6386078 MINIFICPP-1513 - Increase timeouts for behave tests involving http proxies, clean up logging
new 5b6d78d MINIFICPP-1517 - Fix integration test log cluttering
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CMakeLists.txt | 13 +++++++++++-
cmake/DockerConfig.cmake | 1 +
docker/DockerBuild.sh | 2 ++
docker/DockerVerify.sh | 2 +-
docker/Dockerfile | 5 +++--
.../integration/MiNiFi_integration_test_driver.py | 1 -
docker/test/integration/environment.py | 1 -
docker/test/integration/features/http.feature | 2 +-
docker/test/integration/features/s3.feature | 6 +++---
.../integration/minifi/core/DockerTestCluster.py | 23 +++++++++++++++-------
.../minifi/core/DockerTestDirectoryBindings.py | 1 -
11 files changed, 39 insertions(+), 18 deletions(-)
[nifi-minifi-cpp] 02/02: MINIFICPP-1517 - Fix integration test log
cluttering
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
aboda pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 5b6d78d65dd1a97ee4e8e9049fd49b2ec3640e24
Author: Adam Hunyadi <hu...@gmail.com>
AuthorDate: Mon Mar 1 11:04:00 2021 +0100
MINIFICPP-1517 - Fix integration test log cluttering
Signed-off-by: Arpad Boda <ab...@apache.org>
This closes #1019
---
CMakeLists.txt | 13 ++++++++++++-
cmake/DockerConfig.cmake | 1 +
docker/DockerBuild.sh | 2 ++
docker/Dockerfile | 5 +++--
4 files changed, 18 insertions(+), 3 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index e43b850..68e1fc8 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -819,7 +819,18 @@ registerTest("encrypt-config/tests")
include(BuildDocs)
-include(DockerConfig)
+if (ENABLE_PYTHON OR NOT DISABLE_SCRIPTING)
+ message(STATUS "Python and scripting extensions will disabled for the docker build as they produce many error messages on MiNiFi startup.")
+ set(ENABLE_PYTHON_CACHE ${ENABLE_PYTHON})
+ set(DISABLE_SCRIPTING_CACHE ${DISABLE_SCRIPTING})
+ set(ENABLE_PYTHON OFF)
+ set(DISABLE_SCRIPTING ON) # Implies DISABLE_PYTHON_SCRIPTING as well
+ include(DockerConfig)
+ set(ENABLE_PYTHON ${ENABLE_PYTHON_CACHE})
+ set(DISABLE_SCRIPTING ${DISABLE_SCRIPTING_CACHE})
+else()
+ include(DockerConfig)
+endif()
if(NOT WIN32)
# Create a custom build target that will run the linter.
diff --git a/cmake/DockerConfig.cmake b/cmake/DockerConfig.cmake
index c935f27..5ae9b3b 100644
--- a/cmake/DockerConfig.cmake
+++ b/cmake/DockerConfig.cmake
@@ -53,6 +53,7 @@ add_custom_target(
-c DISABLE_LZMA=${DISABLE_LZMA}
-c DISABLE_BZIP2=${DISABLE_BZIP2}
-c DISABLE_SCRIPTING=${DISABLE_SCRIPTING}
+ -c DISABLE_PYTHON_SCRIPTING=${DISABLE_PYTHON_SCRIPTING}
-c DISABLE_CONTROLLER=${DISABLE_CONTROLLER}
-c DOCKER_BASE_IMAGE=${DOCKER_BASE_IMAGE}
-c BUILD_NUMBER=${BUILD_NUMBER}
diff --git a/docker/DockerBuild.sh b/docker/DockerBuild.sh
index 51814cd..40e4701 100755
--- a/docker/DockerBuild.sh
+++ b/docker/DockerBuild.sh
@@ -58,6 +58,7 @@ DISABLE_LIBARCHIVE=${DISABLE_LIBARCHIVE:-}
DISABLE_LZMA=${DISABLE_LZMA:-}
DISABLE_BZIP2=${DISABLE_BZIP2:-}
DISABLE_SCRIPTING=${DISABLE_SCRIPTING:-}
+DISABLE_PYTHON_SCRIPTING=${DISABLE_PYTHON_SCRIPTING:-}
DISABLE_CONTROLLER=${DISABLE_CONTROLLER:-}
DOCKER_BASE_IMAGE=${DOCKER_BASE_IMAGE:-}
@@ -194,6 +195,7 @@ BUILD_ARGS="--build-arg UID=${UID_ARG} \
--build-arg DISABLE_LZMA=${DISABLE_LZMA} \
--build-arg DISABLE_BZIP2=${DISABLE_BZIP2} \
--build-arg DISABLE_SCRIPTING=${DISABLE_SCRIPTING} \
+ --build-arg DISABLE_PYTHON_SCRIPTING=${DISABLE_PYTHON_SCRIPTING} \
--build-arg DISABLE_CONTROLLER=${DISABLE_CONTROLLER} "
if [ -n "${DOCKER_BASE_IMAGE}" ]; then
diff --git a/docker/Dockerfile b/docker/Dockerfile
index ea6049f..20e8ce8 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -78,7 +78,7 @@ FROM build_deps AS build_minimal
RUN cd ${MINIFI_BASE_DIR} \
&& mkdir build \
&& cd build \
- && cmake -DDISABLE_SCRIPTING=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_AWS=ON -DSKIP_TESTS=true -DCMAKE_BUILD_TYPE=MinSizeRel .. \
+ && cmake -DENABLE_PYTHON=OFF -DDISABLE_SCRIPTING=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_AWS=ON -DSKIP_TESTS=true -DCMAKE_BUILD_TYPE=MinSizeRel .. \
&& make -j$(nproc) package \
&& tar -xzvf ${MINIFI_BASE_DIR}/build/nifi-minifi-cpp-${MINIFI_VERSION}-bin.tar.gz -C ${MINIFI_BASE_DIR}
@@ -115,6 +115,7 @@ ARG DISABLE_LIBARCHIVE
ARG DISABLE_LZMA
ARG DISABLE_BZIP2
ARG DISABLE_SCRIPTING
+ARG DISABLE_PYTHON_SCRIPTING
ARG DISABLE_CONTROLLER
RUN cd ${MINIFI_BASE_DIR} \
&& mkdir build \
@@ -128,7 +129,7 @@ RUN cd ${MINIFI_BASE_DIR} \
-DDISABLE_CURL=${DISABLE_CURL} -DDISABLE_JEMALLOC=${DISABLE_JEMALLOC} -DDISABLE_CIVET=${DISABLE_CIVET} \
-DDISABLE_EXPRESSION_LANGUAGE=${DISABLE_EXPRESSION_LANGUAGE} -DDISABLE_ROCKSDB=${DISABLE_ROCKSDB} \
-DDISABLE_LIBARCHIVE=${DISABLE_LIBARCHIVE} -DDISABLE_LZMA=${DISABLE_LZMA} -DDISABLE_BZIP2=${DISABLE_BZIP2} \
- -DDISABLE_SCRIPTING=${DISABLE_SCRIPTING} -DDISABLE_CONTROLLER=${DISABLE_CONTROLLER} -DCMAKE_BUILD_TYPE=Release .. \
+ -DDISABLE_SCRIPTING=${DISABLE_SCRIPTING} -DDISABLE_PYTHON_SCRIPTING=${DDISABLE_PYTHON_SCRIPTING} -DDISABLE_CONTROLLER=${DISABLE_CONTROLLER} -DCMAKE_BUILD_TYPE=Release .. \
&& make -j$(nproc) package \
&& tar -xzvf ${MINIFI_BASE_DIR}/build/nifi-minifi-cpp-${MINIFI_VERSION}-bin.tar.gz -C ${MINIFI_BASE_DIR}
[nifi-minifi-cpp] 01/02: MINIFICPP-1513 - Increase timeouts for
behave tests involving http proxies, clean up logging
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
aboda pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 63860782fdbb27def1d2d7fda8bd85ac91bbb74b
Author: Adam Hunyadi <hu...@gmail.com>
AuthorDate: Fri Feb 26 15:34:49 2021 +0100
MINIFICPP-1513 - Increase timeouts for behave tests involving http proxies, clean up logging
Signed-off-by: Arpad Boda <ab...@apache.org>
This closes #1018
---
docker/DockerVerify.sh | 2 +-
.../integration/MiNiFi_integration_test_driver.py | 1 -
docker/test/integration/environment.py | 1 -
docker/test/integration/features/http.feature | 2 +-
docker/test/integration/features/s3.feature | 6 +++---
.../integration/minifi/core/DockerTestCluster.py | 23 +++++++++++++++-------
.../minifi/core/DockerTestDirectoryBindings.py | 1 -
7 files changed, 21 insertions(+), 15 deletions(-)
diff --git a/docker/DockerVerify.sh b/docker/DockerVerify.sh
index dc897c2..afb4077 100755
--- a/docker/DockerVerify.sh
+++ b/docker/DockerVerify.sh
@@ -72,7 +72,7 @@ export PATH
PYTHONPATH="${PYTHONPATH}:${docker_dir}/test/integration"
export PYTHONPATH
-BEHAVE_OPTS="-f pretty --logging-level INFO --no-capture"
+BEHAVE_OPTS="-f pretty --logging-level INFO --logging-clear-handlers"
cd "${docker_dir}/test/integration"
exec
diff --git a/docker/test/integration/MiNiFi_integration_test_driver.py b/docker/test/integration/MiNiFi_integration_test_driver.py
index a1c63a0..ca5f04a 100644
--- a/docker/test/integration/MiNiFi_integration_test_driver.py
+++ b/docker/test/integration/MiNiFi_integration_test_driver.py
@@ -22,7 +22,6 @@ from minifi.validators.SingleFileOutputValidator import SingleFileOutputValidato
class MiNiFi_integration_test():
def __init__(self, context):
- logging.info("MiNiFi_integration_test init")
self.test_id = str(uuid.uuid4())
self.clusters = {}
diff --git a/docker/test/integration/environment.py b/docker/test/integration/environment.py
index 68ddd42..4a3c12f 100644
--- a/docker/test/integration/environment.py
+++ b/docker/test/integration/environment.py
@@ -11,7 +11,6 @@ def raise_exception(exception):
@fixture
def test_driver_fixture(context):
- logging.info("Integration test setup")
context.test = MiNiFi_integration_test(context)
yield context.test
logging.info("Integration test teardown...")
diff --git a/docker/test/integration/features/http.feature b/docker/test/integration/features/http.feature
index 0652fd9..1f5e89d 100644
--- a/docker/test/integration/features/http.feature
+++ b/docker/test/integration/features/http.feature
@@ -40,7 +40,7 @@ Feature: Sending data using InvokeHTTP to a receiver using ListenHTTP
And the "success" relationship of the ListenHTTP processor is connected to the PutFile
When all instances start up
- Then a flowfile with the content "test" is placed in the monitored directory in less than 60 seconds
+ Then a flowfile with the content "test" is placed in the monitored directory in less than 120 seconds
And no errors were generated on the "http-proxy" regarding "http://minifi-listen:8080/contentListener"
Scenario: A MiNiFi instance and transfers hashed data to another MiNiFi instance
diff --git a/docker/test/integration/features/s3.feature b/docker/test/integration/features/s3.feature
index 897bf18..03f0e11 100644
--- a/docker/test/integration/features/s3.feature
+++ b/docker/test/integration/features/s3.feature
@@ -40,7 +40,7 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
And the http proxy server "http-proxy" is set up
When all instances start up
- Then a flowfile with the content "test" is placed in the monitored directory in less than 90 seconds
+ Then a flowfile with the content "test" is placed in the monitored directory in less than 150 seconds
And the object on the "s3" s3 server is "LH_O#L|FD<FASD{FO#@$#$%^ \"#\"$L%:\"@#$L\":test_data#$#%#$%?{\"F{"
And the object content type on the "s3" s3 server is "application/octet-stream" and the object metadata matches use metadata
And no errors were generated on the "http-proxy" regarding "http://s3-server:9090/test_bucket/test_object_key"
@@ -102,7 +102,7 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
When all instances start up
- Then a flowfile with the content "test" is placed in the monitored directory in less than 120 seconds
+ Then a flowfile with the content "test" is placed in the monitored directory in less than 150 seconds
And the object bucket on the "s3" s3 server is empty
And no errors were generated on the "http-proxy" regarding "http://s3-server:9090/test_bucket/test_object_key"
@@ -151,5 +151,5 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
When all instances start up
- Then a flowfile with the content "test" is placed in the monitored directory in less than 120 seconds
+ Then a flowfile with the content "test" is placed in the monitored directory in less than 150 seconds
And no errors were generated on the "http-proxy" regarding "http://s3-server:9090/test_bucket/test_object_key"
diff --git a/docker/test/integration/minifi/core/DockerTestCluster.py b/docker/test/integration/minifi/core/DockerTestCluster.py
index 772558a..2b0f13e 100644
--- a/docker/test/integration/minifi/core/DockerTestCluster.py
+++ b/docker/test/integration/minifi/core/DockerTestCluster.py
@@ -41,6 +41,15 @@ class DockerTestCluster(SingleNodeDockerCluster):
return True
return False
+ @staticmethod
+ def get_stdout_encoding():
+ # Use UTF-8 both when sys.stdout present but set to None (explicitly piped output
+ # and also some CI such as GitHub Actions).
+ encoding = getattr(sys.stdout, "encoding", None)
+ if encoding is None:
+ encoding = "utf8"
+ return encoding
+
def get_app_log(self):
for container in self.containers.values():
container = self.client.containers.get(container.id)
@@ -91,7 +100,7 @@ class DockerTestCluster(SingleNodeDockerCluster):
raise Exception("Container failed to start up.")
def check_http_proxy_access(self, url):
- output = subprocess.check_output(["docker", "exec", "http-proxy", "cat", "/var/log/squid/access.log"]).decode(sys.stdout.encoding)
+ output = subprocess.check_output(["docker", "exec", "http-proxy", "cat", "/var/log/squid/access.log"]).decode(self.get_stdout_encoding())
return url in output and \
((output.count("TCP_DENIED/407") != 0 and \
output.count("TCP_MISS/200") == output.count("TCP_DENIED/407")) or \
@@ -99,21 +108,21 @@ class DockerTestCluster(SingleNodeDockerCluster):
@retry_check()
def check_s3_server_object_data(self, test_data):
- s3_mock_dir = subprocess.check_output(["docker", "exec", "s3-server", "find", "/tmp/", "-type", "d", "-name", "s3mock*"]).decode(sys.stdout.encoding).strip()
- file_data = subprocess.check_output(["docker", "exec", "s3-server", "cat", s3_mock_dir + "/test_bucket/test_object_key/fileData"]).decode(sys.stdout.encoding)
+ s3_mock_dir = subprocess.check_output(["docker", "exec", "s3-server", "find", "/tmp/", "-type", "d", "-name", "s3mock*"]).decode(self.get_stdout_encoding()).strip()
+ file_data = subprocess.check_output(["docker", "exec", "s3-server", "cat", s3_mock_dir + "/test_bucket/test_object_key/fileData"]).decode(self.get_stdout_encoding())
return file_data == test_data
@retry_check()
def check_s3_server_object_metadata(self, content_type="application/octet-stream", metadata=dict()):
- s3_mock_dir = subprocess.check_output(["docker", "exec", "s3-server", "find", "/tmp/", "-type", "d", "-name", "s3mock*"]).decode(sys.stdout.encoding).strip()
- metadata_json = subprocess.check_output(["docker", "exec", "s3-server", "cat", s3_mock_dir + "/test_bucket/test_object_key/metadata"]).decode(sys.stdout.encoding)
+ s3_mock_dir = subprocess.check_output(["docker", "exec", "s3-server", "find", "/tmp/", "-type", "d", "-name", "s3mock*"]).decode(self.get_stdout_encoding()).strip()
+ metadata_json = subprocess.check_output(["docker", "exec", "s3-server", "cat", s3_mock_dir + "/test_bucket/test_object_key/metadata"]).decode(self.get_stdout_encoding())
server_metadata = json.loads(metadata_json)
return server_metadata["contentType"] == content_type and metadata == server_metadata["userMetadata"]
@retry_check()
def is_s3_bucket_empty(self):
- s3_mock_dir = subprocess.check_output(["docker", "exec", "s3-server", "find", "/tmp/", "-type", "d", "-name", "s3mock*"]).decode(sys.stdout.encoding).strip()
- ls_result = subprocess.check_output(["docker", "exec", "s3-server", "ls", s3_mock_dir + "/test_bucket/"]).decode(sys.stdout.encoding)
+ s3_mock_dir = subprocess.check_output(["docker", "exec", "s3-server", "find", "/tmp/", "-type", "d", "-name", "s3mock*"]).decode(self.get_stdout_encoding()).strip()
+ ls_result = subprocess.check_output(["docker", "exec", "s3-server", "ls", s3_mock_dir + "/test_bucket/"]).decode(self.get_stdout_encoding())
return not ls_result
def wait_for_container_logs(self, container_name, log, timeout, count=1):
diff --git a/docker/test/integration/minifi/core/DockerTestDirectoryBindings.py b/docker/test/integration/minifi/core/DockerTestDirectoryBindings.py
index d9b6a2b..28f9f77 100644
--- a/docker/test/integration/minifi/core/DockerTestDirectoryBindings.py
+++ b/docker/test/integration/minifi/core/DockerTestDirectoryBindings.py
@@ -55,7 +55,6 @@ class DockerTestDirectoryBindings:
@staticmethod
def create_directory(dir):
- logging.info("Creating tmp dir: %s", dir)
os.makedirs(dir)
os.chmod(dir, 0o777)