You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by sz...@apache.org on 2022/04/26 14:39:30 UTC
[nifi-minifi-cpp] 03/03: MINIFICPP-1810 Provide logs to be read from docker logs
This is an automated email from the ASF dual-hosted git repository.
szaszm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit fbf2a349bf10ee36c220eb9f94e1d01ffdd56dd4
Author: Gabor Gyimesi <ga...@gmail.com>
AuthorDate: Tue Apr 26 16:04:46 2022 +0200
MINIFICPP-1810 Provide logs to be read from docker logs
We log to stderr by default in docker containers from now on and let the docker logging engine take care of the logs.
Closes #1315
Signed-off-by: Marton Szasz <sz...@apache.org>
---
.dockerignore | 3 +-
bin/minifi.sh | 5 +-
docker/.dockerignore | 17 -------
docker/Dockerfile | 1 +
docker/conf/minifi-log.properties | 56 ++++++++++++++++++++++
docker/test/integration/minifi/core/Container.py | 3 --
.../integration/minifi/core/DockerTestCluster.py | 22 +--------
.../integration/minifi/core/MinifiContainer.py | 3 --
.../test/integration/minifi/core/NifiContainer.py | 3 --
9 files changed, 64 insertions(+), 49 deletions(-)
diff --git a/.dockerignore b/.dockerignore
index 247ebd57c..5d61cbdb8 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -37,7 +37,8 @@ bin/minifi
*_repository*
logs
**/cmake-build-*
-docker
+docker/*
+!docker/conf
target
*.pyc
*.swp
diff --git a/bin/minifi.sh b/bin/minifi.sh
index fbdea8899..b9ed692f1 100755
--- a/bin/minifi.sh
+++ b/bin/minifi.sh
@@ -211,7 +211,8 @@ case "\$1" in
exit 0;
fi
fi
- echo running
+ pid=$$
+ echo ${pid} > "${pid_file}"
exec "\${minifi_executable}"
;;
status)
@@ -339,6 +340,8 @@ case "$1" in
exit 0;
fi
fi
+ pid=$$
+ echo ${pid} > "${pid_file}"
exec "${minifi_executable}"
;;
status)
diff --git a/docker/.dockerignore b/docker/.dockerignore
deleted file mode 100644
index f6d019fab..000000000
--- a/docker/.dockerignore
+++ /dev/null
@@ -1,17 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Place files you want to exclude from the docker build here similar to .gitignore https://docs.docker.com/engine/reference/builder/#dockerignore-file
-DockerBuild.sh
\ No newline at end of file
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 108764bbd..404a87a0a 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -174,6 +174,7 @@ RUN addgroup -g ${GID} ${USER} && adduser -u ${UID} -D -G ${USER} -g "" ${USER}
# Copy built minifi distribution from builder
COPY --from=build --chown=${USER}:${USER} ${MINIFI_VERSIONED_HOME} ${MINIFI_HOME}
+COPY --from=build --chown=${USER}:${USER} ${MINIFI_BASE_DIR}/docker/conf/minifi-log.properties ${MINIFI_HOME}/conf/minifi-log.properties
USER ${USER}
WORKDIR ${MINIFI_HOME}
diff --git a/docker/conf/minifi-log.properties b/docker/conf/minifi-log.properties
new file mode 100644
index 000000000..459db5085
--- /dev/null
+++ b/docker/conf/minifi-log.properties
@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#More verbose pattern by default
+#Format details at https://github.com/gabime/spdlog/wiki/3.-Custom-formatting
+spdlog.pattern=[%Y-%m-%d %H:%M:%S.%e] [%n] [%l] %v
+# uncomment to prune package names
+#spdlog.shorten_names=true
+
+#Old format
+#spdlog.pattern=[%Y-%m-%d %H:%M:%S.%e] [minifi log] [%l] %v
+
+#More compact format example
+#spdlog.pattern=[%D %H:%M:%S.%e] [%L] %v
+
+#appender.rolling=rollingappender
+#appender.rolling.directory=${MINIFI_HOME}/logs
+#appender.rolling.file_name=minifi-app.log
+#appender.rolling.max_files=3
+#appender.rolling.max_file_size=5242880
+
+#Other possible appenders
+#appender.stdout=stdout
+appender.stderr=stderr
+#appender.null=null
+## The syslog appender will log using syslog(3) on *nix, and to the Windows Event Log on Windows
+#appender.syslog=syslog
+
+logger.root=INFO,stderr
+
+#Logging configurable by namespace
+logger.org::apache::nifi::minifi=INFO,stderr
+
+#Logging configurable by class fully qualified name
+#logger.org::apache::nifi::minifi::core::logging::LoggerConfiguration=DEBUG
+
+# Log compression #
+## Enables the agent to keep a limited chunk of the application
+## logs in memory in compressed format. Note that due to its
+## compressed nature this could mean more logs than the contents
+## of the log files.
+## Setting any of these to 0 disables the in-memory log compression.
+#compression.cached.log.max.size=8 MB
+#compression.compressed.log.max.size=8 MB
diff --git a/docker/test/integration/minifi/core/Container.py b/docker/test/integration/minifi/core/Container.py
index 700dddb21..2e0c40641 100644
--- a/docker/test/integration/minifi/core/Container.py
+++ b/docker/test/integration/minifi/core/Container.py
@@ -64,6 +64,3 @@ class Container:
def get_startup_finished_log_entry(self):
raise NotImplementedError()
-
- def get_log_file_path(self):
- return None
diff --git a/docker/test/integration/minifi/core/DockerTestCluster.py b/docker/test/integration/minifi/core/DockerTestCluster.py
index 1a8a7e5e4..c29e0baa5 100644
--- a/docker/test/integration/minifi/core/DockerTestCluster.py
+++ b/docker/test/integration/minifi/core/DockerTestCluster.py
@@ -16,7 +16,6 @@
import json
import logging
-import subprocess
import sys
import time
import os
@@ -69,26 +68,7 @@ class DockerTestCluster(SingleNodeDockerCluster):
logging.warning('Container segfaulted: %s', container.name)
self.segfault = True
- log_file_path = self.containers[container_name].get_log_file_path()
- if not log_file_path:
- return container.status, container.logs()
-
- try:
- if container.status == 'running':
- app_log_status, app_log = container.exec_run('/bin/sh -c \'cat ' + log_file_path + '\'')
- if app_log_status == 0:
- return container.status, app_log
- elif container.status == 'exited':
- log_file_name = container_name + ".log"
- code = subprocess.run(["docker", "cp", container_name + ":" + log_file_path, log_file_name]).returncode
- if code == 0:
- output = open(log_file_name, 'rb').read()
- os.remove(log_file_name)
- return container.status, output
- except Exception:
- return container.status, None
-
- return container.status, None
+ return container.status, container.logs()
def __wait_for_app_logs_impl(self, container_name, log_entry, timeout_seconds, count, use_regex):
wait_start_time = time.perf_counter()
diff --git a/docker/test/integration/minifi/core/MinifiContainer.py b/docker/test/integration/minifi/core/MinifiContainer.py
index 71cb2c96e..17783eec6 100644
--- a/docker/test/integration/minifi/core/MinifiContainer.py
+++ b/docker/test/integration/minifi/core/MinifiContainer.py
@@ -32,9 +32,6 @@ class MinifiContainer(FlowContainer):
def get_startup_finished_log_entry(self):
return "Starting Flow Controller"
- def get_log_file_path(self):
- return MinifiContainer.MINIFI_ROOT + '/logs/minifi-app.log'
-
def _create_config(self):
serializer = Minifi_flow_yaml_serializer()
test_flow_yaml = serializer.serialize(self.start_nodes)
diff --git a/docker/test/integration/minifi/core/NifiContainer.py b/docker/test/integration/minifi/core/NifiContainer.py
index a9ea539b0..0aadad37f 100644
--- a/docker/test/integration/minifi/core/NifiContainer.py
+++ b/docker/test/integration/minifi/core/NifiContainer.py
@@ -37,9 +37,6 @@ class NifiContainer(FlowContainer):
def get_startup_finished_log_entry(self):
return "Starting Flow Controller"
- def get_log_file_path(self):
- return NifiContainer.NIFI_ROOT + '/logs/nifi-app.log'
-
def __create_config(self):
serializer = Nifi_flow_xml_serializer()
test_flow_xml = serializer.serialize(self.start_nodes, NifiContainer.NIFI_VERSION)