You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by sz...@apache.org on 2022/08/19 10:20:14 UTC
[nifi-minifi-cpp] branch main updated (f39859fda -> 746ade817)
This is an automated email from the ASF dual-hosted git repository.
szaszm pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
from f39859fda MINIFICPP-1896 Change C2 component-level start/stop commands to use UUID
new 5bdb8912c MINIFICPP-1853 Support multiple metrics of the same processor
new a8a427836 MINIFICPP-1888 Move all extension builds to CentOS job
new 746ade817 MINIFICPP-1865 Integrate performance clang-tidy checks in CI
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.clang-tidy | 17 +-
.github/workflows/ci.yml | 38 ++-
C2.md | 321 ++++++++++-----------
cmake/DockerConfig.cmake | 4 +
cmake/MiNiFiOptions.cmake | 3 +-
docker/DockerBuild.sh | 36 ++-
docker/Dockerfile | 3 +-
docker/centos/Dockerfile | 40 ++-
.../test/integration/features/prometheus.feature | 13 +
extensions/aws/processors/PutS3Object.cpp | 4 +-
extensions/civetweb/processors/ListenHTTP.cpp | 4 +-
extensions/expression-language/Expression.cpp | 20 +-
.../impl/expression/Expression.h | 4 +-
extensions/http-curl/protocols/RESTSender.cpp | 2 +-
extensions/http-curl/protocols/RESTSender.h | 2 +-
.../http-curl/tests/C2DescribeMetricsTest.cpp | 161 +++++++++++
extensions/http-curl/tests/C2MetricsTest.cpp | 7 +-
...tyMetricTest.cpp => C2SameProcessorMetrics.cpp} | 30 +-
extensions/http-curl/tests/CMakeLists.txt | 2 +
extensions/http-curl/tests/HTTPSiteToSiteTests.cpp | 2 +-
.../http-curl/tests/unit/InvokeHTTPTests.cpp | 12 +-
extensions/jni/jvm/JniFlowFile.cpp | 24 +-
extensions/libarchive/ArchiveTests.h | 8 +-
extensions/libarchive/BinFiles.cpp | 10 +-
extensions/libarchive/BinFiles.h | 4 +-
extensions/libarchive/MergeContent.cpp | 6 +-
extensions/libarchive/MergeContent.h | 2 +-
.../prometheus/PrometheusMetricsPublisher.cpp | 6 +-
extensions/script/lua/LuaScriptEngine.cpp | 5 +-
extensions/script/lua/LuaScriptEngine.h | 12 +-
extensions/script/python/PyProcessSession.cpp | 22 +-
extensions/script/python/PyProcessSession.h | 20 +-
extensions/script/python/PythonBindings.h | 2 +-
extensions/sftp/client/SFTPClient.cpp | 2 +-
extensions/sftp/tests/ListSFTPTests.cpp | 4 +-
.../processors/DefragmentText.cpp | 2 +-
.../standard-processors/processors/GetFile.h | 10 +-
extensions/standard-processors/processors/GetTCP.h | 6 +-
.../tests/integration/TestExecuteProcess.cpp | 20 +-
.../tests/unit/ProcessorTests.cpp | 6 +-
.../tests/unit/TailFileTests.cpp | 3 +-
libminifi/include/RemoteProcessorGroupPort.h | 14 +-
libminifi/include/c2/C2Client.h | 8 +-
.../controller/StandardControllerServiceNode.h | 19 +-
libminifi/include/core/extension/Utils.h | 7 +-
.../reporting/SiteToSiteProvenanceReportingTask.h | 19 +-
libminifi/include/core/state/nodes/MetricsBase.h | 30 +-
.../include/core/state/nodes/ResponseNodeLoader.h | 9 +-
libminifi/include/core/yaml/YamlConnectionParser.h | 14 +-
libminifi/include/io/tls/TLSServerSocket.h | 16 +-
libminifi/include/io/tls/TLSSocket.h | 21 +-
libminifi/include/utils/ClassUtils.h | 19 +-
libminifi/include/utils/RegexUtils.h | 4 +-
libminifi/include/utils/StringUtils.h | 2 +-
libminifi/src/RemoteProcessorGroupPort.cpp | 2 +-
libminifi/src/c2/C2Agent.cpp | 16 +-
libminifi/src/c2/C2Client.cpp | 90 +++---
.../controller/StandardControllerServiceNode.cpp | 18 +-
.../SiteToSiteProvenanceReportingTask.cpp | 18 +-
.../src/core/state/nodes/ResponseNodeLoader.cpp | 44 +--
libminifi/src/core/yaml/YamlConnectionParser.cpp | 2 +-
libminifi/src/io/tls/TLSServerSocket.cpp | 14 +-
libminifi/src/io/tls/TLSSocket.cpp | 24 +-
libminifi/src/utils/ClassUtils.cpp | 15 +-
libminifi/src/utils/RegexUtils.cpp | 4 +-
libminifi/src/utils/StringUtils.cpp | 6 +-
.../test/archive-tests/ManipulateArchiveTests.cpp | 12 +-
libminifi/test/archive-tests/MergeFileTests.cpp | 2 +-
libminifi/test/archive-tests/util/ArchiveTests.cpp | 16 +-
...reWithPass.yml => TestSameProcessorMetrics.yml} | 85 ++++--
libminifi/test/rocksdb-tests/RocksDBTests.cpp | 1 +
libminifi/test/sql-tests/mocks/MockConnectors.cpp | 2 +
libminifi/test/unit/BackTraceTests.cpp | 6 +-
libminifi/test/unit/CollectionUtilsTests.cpp | 2 +-
libminifi/test/unit/ExpectedTest.cpp | 28 +-
libminifi/test/unit/IdTests.cpp | 2 +-
libminifi/test/unit/ThreadPoolTests.cpp | 6 +-
77 files changed, 836 insertions(+), 660 deletions(-)
create mode 100644 extensions/http-curl/tests/C2DescribeMetricsTest.cpp
copy extensions/http-curl/tests/{C2EmptyMetricTest.cpp => C2SameProcessorMetrics.cpp} (78%)
copy libminifi/test/resources/{TestGetTCPSecureWithPass.yml => TestSameProcessorMetrics.yml} (54%)
[nifi-minifi-cpp] 01/03: MINIFICPP-1853 Support multiple metrics of the same processor
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szaszm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 5bdb8912cde35223c15bddc457512ac1ebd831a1
Author: Gabor Gyimesi <ga...@gmail.com>
AuthorDate: Thu Aug 18 18:27:18 2022 +0200
MINIFICPP-1853 Support multiple metrics of the same processor
Closes #1360
Signed-off-by: Marton Szasz <sz...@apache.org>
---
C2.md | 321 ++++++++++-----------
.../test/integration/features/prometheus.feature | 13 +
.../http-curl/tests/C2DescribeMetricsTest.cpp | 161 +++++++++++
extensions/http-curl/tests/C2MetricsTest.cpp | 7 +-
.../http-curl/tests/C2SameProcessorMetrics.cpp | 115 ++++++++
extensions/http-curl/tests/CMakeLists.txt | 2 +
.../prometheus/PrometheusMetricsPublisher.cpp | 6 +-
.../standard-processors/processors/GetFile.h | 10 +-
extensions/standard-processors/processors/GetTCP.h | 6 +-
libminifi/include/c2/C2Client.h | 8 +-
libminifi/include/core/state/nodes/MetricsBase.h | 30 +-
.../include/core/state/nodes/ResponseNodeLoader.h | 9 +-
libminifi/src/c2/C2Client.cpp | 81 +++---
.../src/core/state/nodes/ResponseNodeLoader.cpp | 44 +--
.../test/resources/TestSameProcessorMetrics.yml | 123 ++++++++
15 files changed, 699 insertions(+), 237 deletions(-)
diff --git a/C2.md b/C2.md
index 2f2af37d5..5394531c5 100644
--- a/C2.md
+++ b/C2.md
@@ -21,14 +21,16 @@ options defined are located in minifi.properties.
## Table of Contents
-- [Description](#description)
-- [Configuration](#configuration)
- - [Base Options](#base-options)
- - [Metrics](#metrics)
- - [Protocols](#protocols)
- - [Triggers](#triggers)
- - [UpdatePolicies](#updatepolicies)
- - [Documentation](#documentation)
+- [Apache NiFi - MiNiFi - C++ C2 Readme.](#apache-nifi---minifi---c-c2-readme)
+ - [Table of Contents](#table-of-contents)
+ - [Description](#description)
+ - [Configuration](#configuration)
+ - [Base Options](#base-options)
+ - [Metrics](#metrics)
+ - [Protocols](#protocols)
+ - [UpdatePolicies](#updatepolicies)
+ - [Triggers](#triggers)
+ - [C2 File triggers](#c2-file-triggers)
## Description
@@ -55,46 +57,46 @@ With this change, heartbeat with agent manifest included is sent only for the fi
light weight heartbeat. If for some reason the C2 server does not receive the first full heartbeat, the manifest can
be requested via C2 DESCRIBE manifest command.
- #in minifi.properties
- # Disable/Enable C2
- nifi.c2.enable=true
+ #in minifi.properties
- # specify which sections the heartbeat message should contain
- # DeviceInfoNode: basic info about the system (OS, number of cores etc)
- # AgentInformation: info about the MiNiFi agent, may include the manifest
- # FlowInformation: information about the current flow, including queue sizes
- # ConfigurationChecksums: hashes of the configuration files; can be used to detect unexpected modifications
- # the default is
- nifi.c2.root.classes=DeviceInfoNode,AgentInformation,FlowInformation
+ # Disable/Enable C2
+ nifi.c2.enable=true
- # specify C2 protocol -- default is RESTSender if not specified
- nifi.c2.agent.protocol.class=RESTSender
- # nifi.c2.agent.protocol.class=CoapProtocol
+ # specify which sections the heartbeat message should contain
+ # DeviceInfoNode: basic info about the system (OS, number of cores etc)
+ # AgentInformation: info about the MiNiFi agent, may include the manifest
+ # FlowInformation: information about the current flow, including queue sizes
+ # ConfigurationChecksums: hashes of the configuration files; can be used to detect unexpected modifications
+ # the default is
+ nifi.c2.root.classes=DeviceInfoNode,AgentInformation,FlowInformation
- # control c2 heartbeat interval in milliseconds
- nifi.c2.agent.heartbeat.period=250
+ # specify C2 protocol -- default is RESTSender if not specified
+ nifi.c2.agent.protocol.class=RESTSender
+ # nifi.c2.agent.protocol.class=CoapProtocol
- # enable reporter classes
- nifi.c2.agent.heartbeat.reporter.class=RESTReciver
+ # control c2 heartbeat interval in milliseconds
+ nifi.c2.agent.heartbeat.period=250
- # specify the rest URIs if using RESTSender
- nifi.c2.rest.url=http://<your-c2-server>/<c2-api-path>/c2-protocol/heartbeat
- nifi.c2.rest.url.ack=http://<your-c2-server>/<c2-api-path>/c2-protocol/acknowledge
- nifi.c2.flow.base.url=http://<your-c2-server>/<c2-api-path>/c2-protocol/
+ # enable reporter classes
+ nifi.c2.agent.heartbeat.reporter.class=RESTReciver
- # c2 agent identifier -- must be defined to run agent
- nifi.c2.agent.identifier=<your identifier>
+ # specify the rest URIs if using RESTSender
+ nifi.c2.rest.url=http://<your-c2-server>/<c2-api-path>/c2-protocol/heartbeat
+ nifi.c2.rest.url.ack=http://<your-c2-server>/<c2-api-path>/c2-protocol/acknowledge
+ nifi.c2.flow.base.url=http://<your-c2-server>/<c2-api-path>/c2-protocol/
- # c2 agent class -- must be defined to run agent
- nifi.c2.agent.class=<your agent class>
+ # c2 agent identifier -- must be defined to run agent
+ nifi.c2.agent.identifier=<your identifier>
- # configure SSL Context service for REST Protocol
- #nifi.c2.rest.ssl.context.service
+ # c2 agent class -- must be defined to run agent
+ nifi.c2.agent.class=<your agent class>
- # specify encoding strategy for c2 requests (gzip, none)
- #nifi.c2.rest.request.encoding=none
+ # configure SSL Context service for REST Protocol
+ #nifi.c2.rest.ssl.context.service
+ # specify encoding strategy for c2 requests (gzip, none)
+ #nifi.c2.rest.request.encoding=none
### Metrics
@@ -103,18 +105,17 @@ operation. Since responses are formed in an AST, metrics can be formed as a sub
are defined apriori and may reference a metrics class specific to a processor. The following example describes
a configuration of an agent
- # in minifi.properties
+ # in minifi.properties
-
- nifi.c2.root.class.definitions=metrics
- nifi.c2.root.class.definitions.metrics.name=metrics
- nifi.c2.root.class.definitions.metrics.metrics=runtimemetrics,loadmetrics,processorMetrics
- nifi.c2.root.class.definitions.metrics.metrics.runtimemetrics.name=RuntimeMetrics
- nifi.c2.root.class.definitions.metrics.metrics.runtimemetrics.classes=DeviceInfoNode,FlowInformation
- nifi.c2.root.class.definitions.metrics.metrics.loadmetrics.name=LoadMetrics
- nifi.c2.root.class.definitions.metrics.metrics.loadmetrics.classes=QueueMetrics,RepositoryMetrics
- nifi.c2.root.class.definitions.metrics.metrics.processorMetrics.name=ProcessorMetric
- nifi.c2.root.class.definitions.metrics.metrics.processorMetrics.classes=GetFileMetrics
+ nifi.c2.root.class.definitions=metrics
+ nifi.c2.root.class.definitions.metrics.name=metrics
+ nifi.c2.root.class.definitions.metrics.metrics=runtimemetrics,loadmetrics,processorMetrics
+ nifi.c2.root.class.definitions.metrics.metrics.runtimemetrics.name=RuntimeMetrics
+ nifi.c2.root.class.definitions.metrics.metrics.runtimemetrics.classes=DeviceInfoNode,FlowInformation
+ nifi.c2.root.class.definitions.metrics.metrics.loadmetrics.name=LoadMetrics
+ nifi.c2.root.class.definitions.metrics.metrics.loadmetrics.classes=QueueMetrics,RepositoryMetrics
+ nifi.c2.root.class.definitions.metrics.metrics.processorMetrics.name=ProcessorMetric
+ nifi.c2.root.class.definitions.metrics.metrics.processorMetrics.classes=GetFileMetrics
This example shows a metrics sub tree defined by the option 'nifi.c2.root.class.definitions'.
@@ -124,85 +125,87 @@ The options below metrics define the sub-trees within metrics: typedmetrics and
The classes sub option will define the metrics classes that are placed within this sub-tree. For the RESTProtocol, the above
configuration produces the following JSON:
- "metrics": {
- "RuntimeMetrics": {
- "deviceInfo": {
- "systemInfo": {
- "cpuUtilization": -1,
- "machinearch": "x86_64",
- "memoryUsage": 13103550464,
- "operatingSystem": "Linux",
- "physicalMem": 67024097280,
- "vCores": 12
- },
- "networkInfo": {
- "hostname": "ggyimesi-5540-ubuntu",
- "ipAddress": "192.168.50.181"
- },
- "identifier": "13396751919892753964"
- },
- "flowInfo": {
- "versionedFlowSnapshotURI": {
- "bucketId": "default",
- "flowId": "8db40550-db5d-11ec-95d7-0433c2c9832b"
- },
- "queues": {
- "2438e3c8-015a-1000-79ca-83af40ec1997": {
- "dataSize": 0,
- "dataSizeMax": 1048576,
- "name": "GetFile/success/LogAttribute",
- "size": 0,
- "sizeMax": 0,
- "uuid": "2438e3c8-015a-1000-79ca-83af40ec1997"
- }
- },
- "components": {
- "FlowController": {
- "running": true,
- "uuid": "2438e3c8-015a-1000-79ca-83af40ec1990"
- },
- "GetFile": {
- "running": false,
- "uuid": "2438e3c8-015a-1000-79ca-83af40ec1991"
- },
- "LogAttribute": {
- "running": true,
- "uuid": "2438e3c8-015a-1000-79ca-83af40ec1992"
- }
- },
- "flowId": "8db40550-db5d-11ec-95d7-0433c2c9832b"
- }
- },
- "LoadMetrics": {
- "QueueMetrics": {
- "GetFile/success/LogAttribute": {
- "datasize": "0",
- "datasizemax": "1048576",
- "queued": "0",
- "queuedmax": "0"
- }
- },
- "RepositoryMetrics": {
- "ff": {
- "full": false,
- "running": false,
- "size": "0"
- },
- "repo_name": {
- "full": false,
- "running": true,
- "size": "0"
- }
- }
- },
- "ProcessorMetrics": {
- "GetFileMetrics": {
- "AcceptedFiles": 0,
- "InputBytes": 0,
- "OnTriggerInvocations": 0
- }
+ "metrics": {
+ "RuntimeMetrics": {
+ "deviceInfo": {
+ "systemInfo": {
+ "cpuUtilization": -1,
+ "machinearch": "x86_64",
+ "memoryUsage": 13103550464,
+ "operatingSystem": "Linux",
+ "physicalMem": 67024097280,
+ "vCores": 12
+ },
+ "networkInfo": {
+ "hostname": "ggyimesi-5540-ubuntu",
+ "ipAddress": "192.168.50.181"
+ },
+ "identifier": "13396751919892753964"
+ },
+ "flowInfo": {
+ "versionedFlowSnapshotURI": {
+ "bucketId": "default",
+ "flowId": "8db40550-db5d-11ec-95d7-0433c2c9832b"
+ },
+ "queues": {
+ "2438e3c8-015a-1000-79ca-83af40ec1997": {
+ "dataSize": 0,
+ "dataSizeMax": 1048576,
+ "name": "GetFile/success/LogAttribute",
+ "size": 0,
+ "sizeMax": 0,
+ "uuid": "2438e3c8-015a-1000-79ca-83af40ec1997"
+ }
+ },
+ "components": {
+ "FlowController": {
+ "running": true,
+ "uuid": "2438e3c8-015a-1000-79ca-83af40ec1990"
+ },
+ "GetFile": {
+ "running": false,
+ "uuid": "2438e3c8-015a-1000-79ca-83af40ec1991"
+ },
+ "LogAttribute": {
+ "running": true,
+ "uuid": "2438e3c8-015a-1000-79ca-83af40ec1992"
+ }
+ },
+ "flowId": "8db40550-db5d-11ec-95d7-0433c2c9832b"
+ }
+ },
+ "LoadMetrics": {
+ "QueueMetrics": {
+ "GetFile/success/LogAttribute": {
+ "datasize": "0",
+ "datasizemax": "1048576",
+ "queued": "0",
+ "queuedmax": "0"
+ }
+ },
+ "RepositoryMetrics": {
+ "ff": {
+ "full": false,
+ "running": false,
+ "size": "0"
+ },
+ "repo_name": {
+ "full": false,
+ "running": true,
+ "size": "0"
+ }
+ }
+ },
+ "ProcessorMetrics": {
+ "GetFileMetrics": {
+ "2438e3c8-015a-1000-79ca-83af40ec1991": {
+ "AcceptedFiles": 0,
+ "InputBytes": 0,
+ "OnTriggerInvocations": 0
+ }
+ }
+ }
}
- }
### Protocols
@@ -213,50 +216,46 @@ Once configured, COAP uses a controller service within the flow OR minifi proper
Note that with CoAP, the payload will be significantly smaller, paring down metrics that are sent in each heartbeat. This will be useful for
constrained environments.
- nifi.c2.agent.coap.host=hostname
- nifi.c2.agent.coap.port=<port number>
-
+ nifi.c2.agent.coap.host=hostname
+ nifi.c2.agent.coap.port=<port number>
If you wish to use the Controller service you made add a controller service named CoapConnectorService with the properties in the example config
below. Note that Max Queue Size is the only non-required property:
- Controller Services:
- - id: 94491a38-015a-1000-0000-000000000001
- name: coapservice
- class: CoapConnectorService
- Properties:
- Remote Server: server
- Remote Port: port
- Max Queue Size: 1000
-
+ Controller Services:
+ - id: 94491a38-015a-1000-0000-000000000001
+ name: coapservice
+ class: CoapConnectorService
+ Properties:
+ Remote Server: server
+ Remote Port: port
+ Max Queue Size: 1000
### UpdatePolicies
Updates to MiNiFi C++ properties can be controlled through an UpdatePolicyControllerService named
C2UpdatePolicy. The service supports several configuration options. They are defined in the following example:
-
- Controller Services:
- - id: 94491a38-015a-1000-0000-000000000001
- name: C2UpdatePolicy
- class: UpdatePolicyControllerService
- Properties:
- # true enables all properties to be updated.
- Allow All Properties: true
- # allowed properties are those which can be updated
- Allowed Properties:
- - value: Property_1
- - value: Property_2
- Disallowed Properties:
- - value: Property_3
- - value: Property_4
-
+ Controller Services:
+ - id: 94491a38-015a-1000-0000-000000000001
+ name: C2UpdatePolicy
+ class: UpdatePolicyControllerService
+ Properties:
+ # true enables all properties to be updated.
+ Allow All Properties: true
+ # allowed properties are those which can be updated
+ Allowed Properties:
+ - value: Property_1
+ - value: Property_2
+ Disallowed Properties:
+ - value: Property_3
+ - value: Property_4
### Triggers
- C2 Triggers can be activated to perform some C2 activity via a local event. Currently only FileUpdateTrigger exists, which monitors
- for C2 File triggers to update the flow configuration. Classes can be defined as a comma separated list of classes to load via the option
- nifi.c2.agent.trigger.classes
+C2 Triggers can be activated to perform some C2 activity via a local event. Currently only FileUpdateTrigger exists, which monitors
+for C2 File triggers to update the flow configuration. Classes can be defined as a comma separated list of classes to load via the option
+nifi.c2.agent.trigger.classes
#### C2 File triggers
@@ -267,8 +266,8 @@ changes while the agent is running, it will be copied into place of the file def
will then be restarted with the new flow configuration. If a failure occurs in reading that file or it is an invalid YAML file, the
update process will be halted.
- in minifi.properties to activate the file update trigger specify
+in minifi.properties to activate the file update trigger specify
- # specifying a trigger
- nifi.c2.agent.trigger.classes=FileUpdateTrigger
- nifi.c2.file.watch=<full path of file to monitor>
+ # specifying a trigger
+ nifi.c2.agent.trigger.classes=FileUpdateTrigger
+ nifi.c2.file.watch=<full path of file to monitor>
diff --git a/docker/test/integration/features/prometheus.feature b/docker/test/integration/features/prometheus.feature
index 7f1f1aa29..e22aa5faf 100644
--- a/docker/test/integration/features/prometheus.feature
+++ b/docker/test/integration/features/prometheus.feature
@@ -15,3 +15,16 @@ Feature: MiNiFi can publish metrics to Prometheus server
And "GetFileMetrics" processor metric is published to the Prometheus server in less than 60 seconds for "GetFile1" processor
And "FlowInformation" is published to the Prometheus server in less than 60 seconds
And "DeviceInfoNode" is published to the Prometheus server in less than 60 seconds
+
+ Scenario: Multiple GetFile metrics are reported by Prometheus
+ Given a GetFile processor with the name "GetFile1" and the "Input Directory" property set to "/tmp/input"
+ And a GetFile processor with the name "GetFile2" and the "Input Directory" property set to "/tmp/input"
+ And "GetFile2" processor is a start node
+ And a file with the content "test" is present in "/tmp/input"
+ And a PutFile processor with the "Directory" property set to "/tmp/output"
+ And the "success" relationship of the GetFile1 processor is connected to the PutFile
+ And the "success" relationship of the GetFile2 processor is connected to the PutFile
+ And a Prometheus server is set up
+ When all instances start up
+ Then "GetFileMetrics" processor metric is published to the Prometheus server in less than 60 seconds for "GetFile1" processor
+ And "GetFileMetrics" processor metric is published to the Prometheus server in less than 60 seconds for "GetFile2" processor
diff --git a/extensions/http-curl/tests/C2DescribeMetricsTest.cpp b/extensions/http-curl/tests/C2DescribeMetricsTest.cpp
new file mode 100644
index 000000000..acf5d80cb
--- /dev/null
+++ b/extensions/http-curl/tests/C2DescribeMetricsTest.cpp
@@ -0,0 +1,161 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#undef NDEBUG
+#include <memory>
+
+#include "TestBase.h"
+#include "HTTPIntegrationBase.h"
+#include "HTTPHandlers.h"
+#include "processors/TailFile.h"
+#include "state/ProcessorController.h"
+#include "utils/file/FileUtils.h"
+#include "utils/TestUtils.h"
+#include "processors/GetTCP.h"
+#include "utils/StringUtils.h"
+#include "utils/file/PathUtils.h"
+
+using namespace std::literals::chrono_literals;
+
+namespace org::apache::nifi::minifi::test {
+
+class VerifyEmptyC2Metric : public VerifyC2Base {
+ public:
+ explicit VerifyEmptyC2Metric(const std::atomic_bool& metrics_found) : metrics_found_(metrics_found) {
+ }
+
+ void testSetup() override {
+ LogTestController::getInstance().setTrace<c2::C2Agent>();
+ LogTestController::getInstance().setTrace<c2::C2Client>();
+ LogTestController::getInstance().setDebug<c2::RESTSender>();
+ LogTestController::getInstance().setDebug<FlowController>();
+ LogTestController::getInstance().setOff<processors::GetTCP>();
+ VerifyC2Base::testSetup();
+ }
+
+ void runAssertions() override {
+ using org::apache::nifi::minifi::utils::verifyEventHappenedInPollTime;
+ assert(verifyEventHappenedInPollTime(40s, [&] { return metrics_found_.load(); }, 1s));
+ }
+
+ private:
+ const std::atomic_bool& metrics_found_;
+};
+
+class MetricsHandler: public HeartbeatHandler {
+ public:
+ explicit MetricsHandler(std::atomic_bool& metrics_found, std::shared_ptr<Configure> configuration)
+ : HeartbeatHandler(std::move(configuration)),
+ metrics_found_(metrics_found) {
+ }
+
+ void handleHeartbeat(const rapidjson::Document&, struct mg_connection* conn) override {
+ switch (state_) {
+ case TestState::DESCRIBE_SPECIFIC_METRIC: {
+ sendHeartbeatResponse("DESCRIBE", "metrics", "889347", conn, {{"metricsClass", "GetFileMetrics"}});
+ break;
+ }
+ case TestState::DESCRIBE_ALL_METRICS: {
+ sendHeartbeatResponse("DESCRIBE", "metrics", "889347", conn);
+ break;
+ }
+ default:
+ throw std::runtime_error("Unhandled test state");
+ }
+ }
+
+ void handleAcknowledge(const rapidjson::Document& root) override {
+ switch (state_) {
+ case TestState::DESCRIBE_SPECIFIC_METRIC: {
+ verifySpecificMetrics(root);
+ break;
+ }
+ case TestState::DESCRIBE_ALL_METRICS: {
+ verifyAllMetrics(root);
+ break;
+ }
+ default:
+ throw std::runtime_error("Unhandled test state");
+ }
+ }
+
+ private:
+ enum class TestState {
+ DESCRIBE_SPECIFIC_METRIC,
+ DESCRIBE_ALL_METRICS
+ };
+
+ static constexpr const char* GETFILE1_UUID = "471deef6-2a6e-4a7d-912a-81cc17e3a206";
+ static constexpr const char* GETFILE2_UUID = "471deef6-2a6e-4a7d-912a-81cc17e3a207";
+ static constexpr const char* GETTCP1_UUID = "2438e3c8-015a-1000-79ca-83af40ec1995";
+ static constexpr const char* GETTCP2_UUID = "2438e3c8-015a-1000-79ca-83af40ec1996";
+
+ static void sendEmptyHeartbeatResponse(struct mg_connection* conn) {
+ mg_printf(conn, "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 0\r\nConnection: close\r\n\r\n");
+ }
+
+ void verifySpecificMetrics(const rapidjson::Document& root) {
+ auto getfile_metrics_verified =
+ !root.HasMember("metrics") &&
+ root.HasMember("GetFileMetrics") &&
+ root["GetFileMetrics"].HasMember(GETFILE1_UUID) &&
+ root["GetFileMetrics"].HasMember(GETFILE2_UUID);
+ if (getfile_metrics_verified) {
+ state_ = TestState::DESCRIBE_ALL_METRICS;
+ }
+ }
+
+ void verifyAllMetrics(const rapidjson::Document& root) {
+ auto all_metrics_verified =
+ root.HasMember("metrics") &&
+ root["metrics"].HasMember("ProcessorMetrics") &&
+ root["metrics"]["ProcessorMetrics"].HasMember("GetFileMetrics") &&
+ root["metrics"]["ProcessorMetrics"]["GetFileMetrics"].HasMember(GETFILE1_UUID) &&
+ root["metrics"]["ProcessorMetrics"]["GetFileMetrics"].HasMember(GETFILE2_UUID) &&
+ root["metrics"]["ProcessorMetrics"]["GetTCPMetrics"].HasMember(GETTCP1_UUID) &&
+ root["metrics"]["ProcessorMetrics"]["GetTCPMetrics"].HasMember(GETTCP2_UUID) &&
+ root["metrics"].HasMember("SystemMetrics") &&
+ root["metrics"]["SystemMetrics"].HasMember("QueueMetrics");
+ if (all_metrics_verified) {
+ metrics_found_ = true;
+ }
+ }
+
+ TestState state_ = TestState::DESCRIBE_SPECIFIC_METRIC;
+ std::atomic_bool& metrics_found_;
+};
+
+} // namespace org::apache::nifi::minifi::test
+
+int main(int argc, char **argv) {
+ std::atomic_bool metrics_found{false};
+ const cmd_args args = parse_cmdline_args(argc, argv, "api/heartbeat");
+ org::apache::nifi::minifi::test::VerifyEmptyC2Metric harness(metrics_found);
+ harness.getConfiguration()->set("nifi.c2.root.class.definitions", "metrics");
+ harness.getConfiguration()->set("nifi.c2.root.class.definitions.metrics.name", "metrics");
+ harness.getConfiguration()->set("nifi.c2.root.class.definitions.metrics.metrics", "processormetrics,systemmetrics");
+ harness.getConfiguration()->set("nifi.c2.root.class.definitions.metrics.metrics.processormetrics.name", "ProcessorMetrics");
+ harness.getConfiguration()->set("nifi.c2.root.class.definitions.metrics.metrics.processormetrics.classes", "GetFileMetrics,GetTCPMetrics");
+ harness.getConfiguration()->set("nifi.c2.root.class.definitions.metrics.metrics.systemmetrics.name", "SystemMetrics");
+ harness.getConfiguration()->set("nifi.c2.root.class.definitions.metrics.metrics.systemmetrics.classes", "QueueMetrics");
+ harness.setKeyDir(args.key_dir);
+ org::apache::nifi::minifi::test::MetricsHandler handler(metrics_found, harness.getConfiguration());
+ harness.setUrl(args.url, &handler);
+ harness.run(args.test_file);
+ return 0;
+}
diff --git a/extensions/http-curl/tests/C2MetricsTest.cpp b/extensions/http-curl/tests/C2MetricsTest.cpp
index 7661fdd84..0e06d7478 100644
--- a/extensions/http-curl/tests/C2MetricsTest.cpp
+++ b/extensions/http-curl/tests/C2MetricsTest.cpp
@@ -94,6 +94,8 @@ class MetricsHandler: public HeartbeatHandler {
VERIFY_UPDATED_METRICS
};
+ static constexpr const char* GETTCP1_UUID = "2438e3c8-015a-1000-79ca-83af40ec1991";
+
static void sendEmptyHeartbeatResponse(struct mg_connection* conn) {
mg_printf(conn, "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 0\r\nConnection: close\r\n\r\n");
}
@@ -169,8 +171,9 @@ class MetricsHandler: public HeartbeatHandler {
static bool verifyProcessorMetrics(const rapidjson::Value& processor_metrics) {
return processor_metrics.HasMember("GetTCPMetrics") &&
- processor_metrics["GetTCPMetrics"].HasMember("OnTriggerInvocations") &&
- processor_metrics["GetTCPMetrics"]["OnTriggerInvocations"].GetUint() > 0;
+ processor_metrics["GetTCPMetrics"].HasMember(GETTCP1_UUID) &&
+ processor_metrics["GetTCPMetrics"][GETTCP1_UUID].HasMember("OnTriggerInvocations") &&
+ processor_metrics["GetTCPMetrics"][GETTCP1_UUID]["OnTriggerInvocations"].GetUint() > 0;
}
[[nodiscard]] static std::string getReplacementConfigAsJsonValue(const std::string& replacement_config_path) {
diff --git a/extensions/http-curl/tests/C2SameProcessorMetrics.cpp b/extensions/http-curl/tests/C2SameProcessorMetrics.cpp
new file mode 100644
index 000000000..c7d1b29aa
--- /dev/null
+++ b/extensions/http-curl/tests/C2SameProcessorMetrics.cpp
@@ -0,0 +1,115 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#undef NDEBUG
+#include <memory>
+
+#include "TestBase.h"
+#include "HTTPIntegrationBase.h"
+#include "HTTPHandlers.h"
+#include "processors/TailFile.h"
+#include "state/ProcessorController.h"
+#include "utils/file/FileUtils.h"
+#include "utils/TestUtils.h"
+#include "processors/GetTCP.h"
+#include "utils/StringUtils.h"
+#include "utils/file/PathUtils.h"
+
+using namespace std::literals::chrono_literals;
+
+namespace org::apache::nifi::minifi::test {
+
+class VerifyEmptyC2Metric : public VerifyC2Base {
+ public:
+ explicit VerifyEmptyC2Metric(const std::atomic_bool& metrics_found) : metrics_found_(metrics_found) {
+ }
+
+ void testSetup() override {
+ LogTestController::getInstance().setTrace<minifi::c2::C2Agent>();
+ LogTestController::getInstance().setTrace<minifi::c2::C2Client>();
+ LogTestController::getInstance().setDebug<minifi::c2::RESTSender>();
+ LogTestController::getInstance().setDebug<minifi::FlowController>();
+ LogTestController::getInstance().setOff<minifi::processors::GetTCP>();
+ VerifyC2Base::testSetup();
+ }
+
+ void runAssertions() override {
+ using org::apache::nifi::minifi::utils::verifyEventHappenedInPollTime;
+ assert(verifyEventHappenedInPollTime(40s, [&] { return metrics_found_.load(); }, 1s));
+ }
+
+ private:
+ const std::atomic_bool& metrics_found_;
+};
+
+class MetricsHandler: public HeartbeatHandler {
+ public:
+ explicit MetricsHandler(std::atomic_bool& metrics_found, std::shared_ptr<minifi::Configure> configuration)
+ : HeartbeatHandler(std::move(configuration)),
+ metrics_found_(metrics_found) {
+ }
+
+ void handleHeartbeat(const rapidjson::Document& root, struct mg_connection* conn) override {
+ verifyMetrics(root);
+ sendEmptyHeartbeatResponse(conn);
+ }
+
+ private:
+ static constexpr const char* GETFILE1_UUID = "471deef6-2a6e-4a7d-912a-81cc17e3a206";
+ static constexpr const char* GETFILE2_UUID = "471deef6-2a6e-4a7d-912a-81cc17e3a207";
+ static constexpr const char* GETTCP1_UUID = "2438e3c8-015a-1000-79ca-83af40ec1995";
+ static constexpr const char* GETTCP2_UUID = "2438e3c8-015a-1000-79ca-83af40ec1996";
+
+ static void sendEmptyHeartbeatResponse(struct mg_connection* conn) {
+ mg_printf(conn, "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 0\r\nConnection: close\r\n\r\n");
+ }
+
+ void verifyMetrics(const rapidjson::Document& root) {
+ auto initial_metrics_verified =
+ root.HasMember("metrics") &&
+ root["metrics"].HasMember("ProcessorMetrics") &&
+ root["metrics"]["ProcessorMetrics"].HasMember("GetFileMetrics") &&
+ root["metrics"]["ProcessorMetrics"]["GetFileMetrics"].HasMember(GETFILE1_UUID) &&
+ root["metrics"]["ProcessorMetrics"]["GetFileMetrics"].HasMember(GETFILE2_UUID) &&
+ root["metrics"]["ProcessorMetrics"]["GetTCPMetrics"].HasMember(GETTCP1_UUID) &&
+ root["metrics"]["ProcessorMetrics"]["GetTCPMetrics"].HasMember(GETTCP2_UUID);
+ if (initial_metrics_verified) {
+ metrics_found_ = true;
+ }
+ }
+
+ std::atomic_bool& metrics_found_;
+};
+
+} // namespace org::apache::nifi::minifi::test
+
+int main(int argc, char **argv) {
+ std::atomic_bool metrics_found{false};
+ const cmd_args args = parse_cmdline_args(argc, argv, "api/heartbeat");
+ org::apache::nifi::minifi::test::VerifyEmptyC2Metric harness(metrics_found);
+ harness.getConfiguration()->set("nifi.c2.root.class.definitions", "metrics");
+ harness.getConfiguration()->set("nifi.c2.root.class.definitions.metrics.name", "metrics");
+ harness.getConfiguration()->set("nifi.c2.root.class.definitions.metrics.metrics", "processormetrics");
+ harness.getConfiguration()->set("nifi.c2.root.class.definitions.metrics.metrics.processormetrics.name", "ProcessorMetrics");
+ harness.getConfiguration()->set("nifi.c2.root.class.definitions.metrics.metrics.processormetrics.classes", "GetFileMetrics,GetTCPMetrics");
+ harness.setKeyDir(args.key_dir);
+ org::apache::nifi::minifi::test::MetricsHandler handler(metrics_found, harness.getConfiguration());
+ harness.setUrl(args.url, &handler);
+ harness.run(args.test_file);
+ return 0;
+}
diff --git a/extensions/http-curl/tests/CMakeLists.txt b/extensions/http-curl/tests/CMakeLists.txt
index d24d0ac9e..3de352c0a 100644
--- a/extensions/http-curl/tests/CMakeLists.txt
+++ b/extensions/http-curl/tests/CMakeLists.txt
@@ -106,3 +106,5 @@ add_test(NAME C2UpdateAssetTest COMMAND C2UpdateAssetTest)
add_test(NAME C2CompressTest COMMAND C2CompressTest)
add_test(NAME C2MetricsTest COMMAND C2MetricsTest "${TEST_RESOURCES}/TestC2Metrics.yml")
add_test(NAME C2EmptyMetricTest COMMAND C2EmptyMetricTest "${TEST_RESOURCES}/TestEmpty.yml")
+add_test(NAME C2SameProcessorMetrics COMMAND C2SameProcessorMetrics "${TEST_RESOURCES}/TestSameProcessorMetrics.yml")
+add_test(NAME C2DescribeMetricsTest COMMAND C2DescribeMetricsTest "${TEST_RESOURCES}/TestSameProcessorMetrics.yml")
diff --git a/extensions/prometheus/PrometheusMetricsPublisher.cpp b/extensions/prometheus/PrometheusMetricsPublisher.cpp
index 007723367..85f122120 100644
--- a/extensions/prometheus/PrometheusMetricsPublisher.cpp
+++ b/extensions/prometheus/PrometheusMetricsPublisher.cpp
@@ -71,12 +71,12 @@ std::vector<std::shared_ptr<state::response::ResponseNode>> PrometheusMetricsPub
if (auto metric_classes_str = configuration_->get(minifi::Configuration::nifi_metrics_publisher_metrics)) {
auto metric_classes = utils::StringUtils::split(*metric_classes_str, ",");
for (const std::string& clazz : metric_classes) {
- auto response_node = response_node_loader_->loadResponseNode(clazz, root);
- if (!response_node) {
+ auto response_nodes = response_node_loader_->loadResponseNodes(clazz, root);
+ if (response_nodes.empty()) {
logger_->log_warn("Metric class '%s' could not be loaded.", clazz);
continue;
}
- nodes.push_back(response_node);
+ nodes.insert(nodes.end(), response_nodes.begin(), response_nodes.end());
}
}
return nodes;
diff --git a/extensions/standard-processors/processors/GetFile.h b/extensions/standard-processors/processors/GetFile.h
index 004427982..086ec41bd 100644
--- a/extensions/standard-processors/processors/GetFile.h
+++ b/extensions/standard-processors/processors/GetFile.h
@@ -62,23 +62,27 @@ class GetFileMetrics : public state::response::ResponseNode {
std::vector<state::response::SerializedResponseNode> serialize() override {
std::vector<state::response::SerializedResponseNode> resp;
+ state::response::SerializedResponseNode root_node;
+ root_node.name = source_component_.getUUIDStr();
+
state::response::SerializedResponseNode iter;
iter.name = "OnTriggerInvocations";
iter.value = (uint32_t)iterations_.load();
- resp.push_back(iter);
+ root_node.children.push_back(iter);
state::response::SerializedResponseNode accepted_files;
accepted_files.name = "AcceptedFiles";
accepted_files.value = (uint32_t)accepted_files_.load();
- resp.push_back(accepted_files);
+ root_node.children.push_back(accepted_files);
state::response::SerializedResponseNode input_bytes;
input_bytes.name = "InputBytes";
input_bytes.value = (uint32_t)input_bytes_.load();
- resp.push_back(input_bytes);
+ root_node.children.push_back(input_bytes);
+ resp.push_back(root_node);
return resp;
}
diff --git a/extensions/standard-processors/processors/GetTCP.h b/extensions/standard-processors/processors/GetTCP.h
index a2fa3b819..7e326eb19 100644
--- a/extensions/standard-processors/processors/GetTCP.h
+++ b/extensions/standard-processors/processors/GetTCP.h
@@ -111,11 +111,15 @@ class GetTCPMetrics : public state::response::ResponseNode {
std::vector<state::response::SerializedResponseNode> serialize() override {
std::vector<state::response::SerializedResponseNode> resp;
+ state::response::SerializedResponseNode root_node;
+ root_node.name = source_component_.getUUIDStr();
+
state::response::SerializedResponseNode iter;
iter.name = "OnTriggerInvocations";
iter.value = (uint32_t)iterations_.load();
- resp.push_back(iter);
+ root_node.children.push_back(iter);
+ resp.push_back(root_node);
return resp;
}
diff --git a/libminifi/include/c2/C2Client.h b/libminifi/include/c2/C2Client.h
index 26da4605b..816474d76 100644
--- a/libminifi/include/c2/C2Client.h
+++ b/libminifi/include/c2/C2Client.h
@@ -18,7 +18,7 @@
#pragma once
-#include <map>
+#include <unordered_map>
#include <memory>
#include <mutex>
#include <optional>
@@ -77,7 +77,11 @@ class C2Client : public core::Flow, public state::response::NodeReporter {
bool initialized_ = false;
std::shared_ptr<core::logging::Logger> logger_;
mutable std::mutex metrics_mutex_;
- std::map<std::string, std::shared_ptr<state::response::ResponseNode>> root_response_nodes_;
+
+ // Name and response node value of the root response nodes defined in nifi.c2.root.classes and nifi.c2.root.class.definitions
+ // In case a root class is defined to be a processor metric there can be multiple response nodes if the same processor is defined
+ // multiple times in the flow
+ std::unordered_map<std::string, std::vector<std::shared_ptr<state::response::ResponseNode>>> root_response_nodes_;
protected:
std::atomic<bool> flow_update_{false};
diff --git a/libminifi/include/core/state/nodes/MetricsBase.h b/libminifi/include/core/state/nodes/MetricsBase.h
index 6fe7dcb54..da545fb14 100644
--- a/libminifi/include/core/state/nodes/MetricsBase.h
+++ b/libminifi/include/core/state/nodes/MetricsBase.h
@@ -22,6 +22,7 @@
#include <memory>
#include <string>
#include <optional>
+#include <unordered_map>
#include "../Value.h"
#include "../PublishedMetricProvider.h"
@@ -51,6 +52,19 @@ class ResponseNode : public core::Connectable, public PublishedMetricProvider {
}
virtual ~ResponseNode() = default;
+ static std::vector<state::response::SerializedResponseNode> serializeAndMergeResponseNodes(const std::vector<std::shared_ptr<ResponseNode>>& nodes) {
+ if (nodes.empty()) {
+ return {};
+ }
+
+ std::vector<state::response::SerializedResponseNode> result;
+ for (const auto& node: nodes) {
+ auto serialized = node->serialize();
+ result.insert(result.end(), serialized.begin(), serialized.end());
+ }
+ return result;
+ }
+
virtual std::vector<SerializedResponseNode> serialize() = 0;
virtual void yield() {
@@ -101,7 +115,7 @@ class ObjectNode : public ResponseNode {
}
void add_node(const std::shared_ptr<ResponseNode> &node) {
- nodes_.push_back(node);
+ nodes_[node->getName()].push_back(node);
}
std::string getName() const override {
@@ -110,19 +124,19 @@ class ObjectNode : public ResponseNode {
std::vector<SerializedResponseNode> serialize() override {
std::vector<SerializedResponseNode> serialized;
-// SerializedResponseNode outer_node;
- // outer_node.name = getName();
- for (auto &node : nodes_) {
+ for (const auto& [name, nodes] : nodes_) {
+ if (nodes.empty()) {
+ continue;
+ }
SerializedResponseNode inner_node;
- inner_node.name = node->getName();
- for (auto &embed : node->serialize()) {
+ inner_node.name = nodes[0]->getName();
+ for (auto &embed : ResponseNode::serializeAndMergeResponseNodes(nodes)) {
if (!embed.empty() || embed.keep_empty) {
inner_node.children.push_back(std::move(embed));
}
}
serialized.push_back(std::move(inner_node));
}
- // serialized.push_back(std::move(outer_node));
return serialized;
}
@@ -131,7 +145,7 @@ class ObjectNode : public ResponseNode {
}
protected:
- std::vector<std::shared_ptr<ResponseNode>> nodes_;
+ std::unordered_map<std::string, std::vector<std::shared_ptr<ResponseNode>>> nodes_;
};
/**
diff --git a/libminifi/include/core/state/nodes/ResponseNodeLoader.h b/libminifi/include/core/state/nodes/ResponseNodeLoader.h
index ffd7a57f9..58d2c8d85 100644
--- a/libminifi/include/core/state/nodes/ResponseNodeLoader.h
+++ b/libminifi/include/core/state/nodes/ResponseNodeLoader.h
@@ -23,6 +23,7 @@
#include <memory>
#include <mutex>
#include <functional>
+#include <vector>
#include "MetricsBase.h"
#include "core/ProcessGroup.h"
@@ -38,14 +39,14 @@ class ResponseNodeLoader {
public:
ResponseNodeLoader(std::shared_ptr<Configure> configuration, std::shared_ptr<core::Repository> provenance_repo,
std::shared_ptr<core::Repository> flow_file_repo, core::FlowConfiguration* flow_configuration);
- std::shared_ptr<ResponseNode> loadResponseNode(const std::string& clazz, core::ProcessGroup* root);
- std::shared_ptr<state::response::ResponseNode> getComponentMetricsNode(const std::string& metrics_class) const;
+ std::vector<std::shared_ptr<ResponseNode>> loadResponseNodes(const std::string& clazz, core::ProcessGroup* root);
+ std::vector<std::shared_ptr<ResponseNode>> getComponentMetricsNodes(const std::string& metrics_class) const;
void setControllerServiceProvider(core::controller::ControllerServiceProvider* controller);
void setStateMonitor(state::StateMonitor* update_sink);
void initializeComponentMetrics(core::ProcessGroup* root);
private:
- std::shared_ptr<ResponseNode> getResponseNode(const std::string& clazz) const;
+ std::vector<std::shared_ptr<ResponseNode>> getResponseNodes(const std::string& clazz) const;
void initializeRepositoryMetrics(const std::shared_ptr<ResponseNode>& response_node);
static void initializeQueueMetrics(const std::shared_ptr<ResponseNode>& response_node, core::ProcessGroup* root);
void initializeAgentIdentifier(const std::shared_ptr<ResponseNode>& response_node);
@@ -55,7 +56,7 @@ class ResponseNodeLoader {
void initializeFlowMonitor(const std::shared_ptr<ResponseNode>& response_node, core::ProcessGroup* root);
mutable std::mutex component_metrics_mutex_;
- std::unordered_map<std::string, std::shared_ptr<ResponseNode>> component_metrics_;
+ std::unordered_map<std::string, std::vector<std::shared_ptr<ResponseNode>>> component_metrics_;
std::shared_ptr<Configure> configuration_;
std::shared_ptr<core::Repository> provenance_repo_;
std::shared_ptr<core::Repository> flow_file_repo_;
diff --git a/libminifi/src/c2/C2Client.cpp b/libminifi/src/c2/C2Client.cpp
index 14829aa75..c7adee079 100644
--- a/libminifi/src/c2/C2Client.cpp
+++ b/libminifi/src/c2/C2Client.cpp
@@ -32,6 +32,7 @@
#include "core/state/nodes/FlowInformation.h"
#include "utils/file/FileSystem.h"
#include "utils/file/FileUtils.h"
+#include "utils/gsl.h"
namespace org::apache::nifi::minifi::c2 {
@@ -102,11 +103,13 @@ std::optional<std::string> C2Client::fetchFlow(const std::string& uri) const {
void C2Client::loadNodeClasses(const std::string& class_definitions, const std::shared_ptr<state::response::ResponseNode>& new_node) {
auto classes = utils::StringUtils::split(class_definitions, ",");
for (const std::string& clazz : classes) {
- auto response_node = response_node_loader_.loadResponseNode(clazz, root_.get());
- if (!response_node) {
+ auto response_nodes = response_node_loader_.loadResponseNodes(clazz, root_.get());
+ if (response_nodes.empty()) {
continue;
}
- std::static_pointer_cast<state::response::ObjectNode>(new_node)->add_node(response_node);
+ for (const auto& response_node : response_nodes) {
+ std::static_pointer_cast<state::response::ObjectNode>(new_node)->add_node(response_node);
+ }
}
}
@@ -137,7 +140,7 @@ void C2Client::loadC2ResponseConfiguration(const std::string &prefix) {
}
// We don't need to lock here, we already do it in the initializeResponseNodes member function
- root_response_nodes_[name] = new_node;
+ root_response_nodes_[name].push_back(new_node);
} catch (...) {
logger_->log_error("Could not create metrics class %s", metricsClass);
}
@@ -191,23 +194,24 @@ std::shared_ptr<state::response::ResponseNode> C2Client::loadC2ResponseConfigura
std::optional<state::response::NodeReporter::ReportedNode> C2Client::getMetricsNode(const std::string& metrics_class) const {
std::lock_guard<std::mutex> guard{metrics_mutex_};
+ const auto createReportedNode = [](const std::vector<std::shared_ptr<state::response::ResponseNode>>& nodes) {
+ gsl_Expects(!nodes.empty());
+ state::response::NodeReporter::ReportedNode reported_node;
+ reported_node.is_array = nodes[0]->isArray();
+ reported_node.name = nodes[0]->getName();
+ reported_node.serialized_nodes = state::response::ResponseNode::serializeAndMergeResponseNodes(nodes);
+ return reported_node;
+ };
+
if (!metrics_class.empty()) {
- auto metrics_node = response_node_loader_.getComponentMetricsNode(metrics_class);
- if (metrics_node) {
- state::response::NodeReporter::ReportedNode reported_node;
- reported_node.is_array = metrics_node->isArray();
- reported_node.name = metrics_node->getName();
- reported_node.serialized_nodes = metrics_node->serialize();
- return reported_node;
+ auto metrics_nodes = response_node_loader_.getComponentMetricsNodes(metrics_class);
+ if (!metrics_nodes.empty()) {
+ return createReportedNode(metrics_nodes);
}
} else {
- const auto iter = root_response_nodes_.find("metrics");
- if (iter != root_response_nodes_.end()) {
- state::response::NodeReporter::ReportedNode reported_node;
- reported_node.is_array = iter->second->isArray();
- reported_node.name = iter->second->getName();
- reported_node.serialized_nodes = iter->second->serialize();
- return reported_node;
+ const auto metrics_it = root_response_nodes_.find("metrics");
+ if (metrics_it != root_response_nodes_.end()) {
+ return createReportedNode(metrics_it->second);
}
}
return std::nullopt;
@@ -218,38 +222,45 @@ std::vector<state::response::NodeReporter::ReportedNode> C2Client::getHeartbeatN
configuration_->get(minifi::Configuration::nifi_c2_full_heartbeat, fullHb);
const bool include = include_manifest || fullHb == "true";
- std::vector<state::response::NodeReporter::ReportedNode> nodes;
+ std::vector<state::response::NodeReporter::ReportedNode> reported_nodes;
std::lock_guard<std::mutex> guard{metrics_mutex_};
- nodes.reserve(root_response_nodes_.size());
- for (const auto &entry : root_response_nodes_) {
- auto identifier = std::dynamic_pointer_cast<state::response::AgentIdentifier>(entry.second);
- if (identifier) {
- identifier->includeAgentManifest(include);
- }
- if (entry.second) {
- state::response::NodeReporter::ReportedNode reported_node;
- reported_node.name = entry.second->getName();
- reported_node.is_array = entry.second->isArray();
- reported_node.serialized_nodes = entry.second->serialize();
- nodes.push_back(reported_node);
+ reported_nodes.reserve(root_response_nodes_.size());
+ for (const auto& [name, node_values] : root_response_nodes_) {
+ for (const auto& node : node_values) {
+ auto identifier = std::dynamic_pointer_cast<state::response::AgentIdentifier>(node);
+ if (identifier) {
+ identifier->includeAgentManifest(include);
+ }
+ if (node) {
+ state::response::NodeReporter::ReportedNode reported_node;
+ reported_node.name = node->getName();
+ reported_node.is_array = node->isArray();
+ reported_node.serialized_nodes = node->serialize();
+ reported_nodes.push_back(reported_node);
+ }
}
}
- return nodes;
+ return reported_nodes;
}
void C2Client::initializeResponseNodes(core::ProcessGroup* root) {
+ if (!root_response_nodes_.empty()) {
+ return;
+ }
std::string class_csv;
std::lock_guard<std::mutex> guard{metrics_mutex_};
if (configuration_->get(minifi::Configuration::nifi_c2_root_classes, class_csv)) {
std::vector<std::string> classes = utils::StringUtils::split(class_csv, ",");
for (const std::string& clazz : classes) {
- auto response_node = response_node_loader_.loadResponseNode(clazz, root);
- if (!response_node) {
+ auto response_nodes = response_node_loader_.loadResponseNodes(clazz, root);
+ if (response_nodes.empty()) {
continue;
}
- root_response_nodes_[response_node->getName()] = std::move(response_node);
+ for (auto&& response_node: response_nodes) {
+ root_response_nodes_[response_node->getName()].push_back(std::move(response_node));
+ }
}
}
diff --git a/libminifi/src/core/state/nodes/ResponseNodeLoader.cpp b/libminifi/src/core/state/nodes/ResponseNodeLoader.cpp
index f98d29379..feb44a00e 100644
--- a/libminifi/src/core/state/nodes/ResponseNodeLoader.cpp
+++ b/libminifi/src/core/state/nodes/ResponseNodeLoader.cpp
@@ -26,6 +26,7 @@
#include "core/state/nodes/AgentInformation.h"
#include "core/state/nodes/ConfigurationChecksums.h"
#include "c2/C2Agent.h"
+#include "utils/gsl.h"
namespace org::apache::nifi::minifi::state::response {
@@ -59,17 +60,22 @@ void ResponseNodeLoader::initializeComponentMetrics(core::ProcessGroup* root) {
node_source->getResponseNodes(metric_vector);
std::lock_guard<std::mutex> guard(component_metrics_mutex_);
for (const auto& metric : metric_vector) {
- component_metrics_[metric->getName()] = metric;
+ component_metrics_[metric->getName()].push_back(metric);
}
}
}
-std::shared_ptr<ResponseNode> ResponseNodeLoader::getResponseNode(const std::string& clazz) const {
+std::vector<std::shared_ptr<ResponseNode>> ResponseNodeLoader::getResponseNodes(const std::string& clazz) const {
std::shared_ptr<core::CoreComponent> ptr = core::ClassLoader::getDefaultClassLoader().instantiate(clazz, clazz);
if (ptr == nullptr) {
- return getComponentMetricsNode(clazz);
+ return getComponentMetricsNodes(clazz);
}
- return std::dynamic_pointer_cast<ResponseNode>(ptr);
+ auto response_node = std::dynamic_pointer_cast<ResponseNode>(ptr);
+ if (!response_node) {
+ logger_->log_error("Instantiated class '%s' is not a ResponseNode!", clazz);
+ return {};
+ }
+ return {response_node};
}
void ResponseNodeLoader::initializeRepositoryMetrics(const std::shared_ptr<ResponseNode>& response_node) {
@@ -153,24 +159,26 @@ void ResponseNodeLoader::initializeFlowMonitor(const std::shared_ptr<ResponseNod
}
}
-std::shared_ptr<ResponseNode> ResponseNodeLoader::loadResponseNode(const std::string& clazz, core::ProcessGroup* root) {
- auto response_node = getResponseNode(clazz);
- if (!response_node) {
+std::vector<std::shared_ptr<ResponseNode>> ResponseNodeLoader::loadResponseNodes(const std::string& clazz, core::ProcessGroup* root) {
+ auto response_nodes = getResponseNodes(clazz);
+ if (response_nodes.empty()) {
logger_->log_error("No metric defined for %s", clazz);
- return nullptr;
+ return {};
}
- initializeRepositoryMetrics(response_node);
- initializeQueueMetrics(response_node, root);
- initializeAgentIdentifier(response_node);
- initializeAgentMonitor(response_node);
- initializeAgentNode(response_node);
- initializeConfigurationChecksums(response_node);
- initializeFlowMonitor(response_node, root);
- return response_node;
+ for (const auto& response_node : response_nodes) {
+ initializeRepositoryMetrics(response_node);
+ initializeQueueMetrics(response_node, root);
+ initializeAgentIdentifier(response_node);
+ initializeAgentMonitor(response_node);
+ initializeAgentNode(response_node);
+ initializeConfigurationChecksums(response_node);
+ initializeFlowMonitor(response_node, root);
+ }
+ return response_nodes;
}
-std::shared_ptr<state::response::ResponseNode> ResponseNodeLoader::getComponentMetricsNode(const std::string& metrics_class) const {
+std::vector<std::shared_ptr<ResponseNode>> ResponseNodeLoader::getComponentMetricsNodes(const std::string& metrics_class) const {
if (!metrics_class.empty()) {
std::lock_guard<std::mutex> lock(component_metrics_mutex_);
const auto citer = component_metrics_.find(metrics_class);
@@ -178,7 +186,7 @@ std::shared_ptr<state::response::ResponseNode> ResponseNodeLoader::getComponentM
return citer->second;
}
}
- return nullptr;
+ return {};
}
void ResponseNodeLoader::setControllerServiceProvider(core::controller::ControllerServiceProvider* controller) {
diff --git a/libminifi/test/resources/TestSameProcessorMetrics.yml b/libminifi/test/resources/TestSameProcessorMetrics.yml
new file mode 100644
index 000000000..2c842b8b8
--- /dev/null
+++ b/libminifi/test/resources/TestSameProcessorMetrics.yml
@@ -0,0 +1,123 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+Flow Controller:
+ name: MiNiFi Flow
+ id: 2438e3c8-015a-1000-79ca-83af40ec1990
+Processors:
+ - name: GetFile1
+ id: 471deef6-2a6e-4a7d-912a-81cc17e3a206
+ class: org.apache.nifi.processors.standard.GetFile
+ max concurrent tasks: 1
+ scheduling strategy: TIMER_DRIVEN
+ scheduling period: 1 sec
+ penalization period: 30 sec
+ yield period: 1 sec
+ run duration nanos: 0
+ auto-terminated relationships list:
+ Properties:
+ Input Directory: /tmp/getfile1
+ Keep Source File: true
+ - name: GetFile2
+ id: 471deef6-2a6e-4a7d-912a-81cc17e3a207
+ class: org.apache.nifi.processors.standard.GetFile
+ max concurrent tasks: 1
+ scheduling strategy: TIMER_DRIVEN
+ scheduling period: 1 sec
+ penalization period: 30 sec
+ yield period: 1 sec
+ run duration nanos: 0
+ auto-terminated relationships list:
+ Properties:
+ Input Directory: /tmp/getfile2
+ Keep Source File: true
+ - name: GetTCP1
+ id: 2438e3c8-015a-1000-79ca-83af40ec1995
+ class: org.apache.nifi.processors.standard.GetTCP
+ max concurrent tasks: 1
+ scheduling strategy: TIMER_DRIVEN
+ scheduling period: 10 msec
+ penalization period: 30 msec
+ yield period: 10 msec
+ run duration nanos: 0
+ auto-terminated relationships list:
+ - success
+ - partial
+ Properties:
+ SSL Context Service: SSLContextService
+ endpoint-list: localhost:8776
+ end-of-message-byte: d
+ reconnect-interval: 100ms
+ connection-attempt-timeout: 2000
+ - name: GetTCP2
+ id: 2438e3c8-015a-1000-79ca-83af40ec1996
+ class: org.apache.nifi.processors.standard.GetTCP
+ max concurrent tasks: 1
+ scheduling strategy: TIMER_DRIVEN
+ scheduling period: 10 msec
+ penalization period: 30 msec
+ yield period: 10 msec
+ run duration nanos: 0
+ auto-terminated relationships list:
+ - success
+ - partial
+ Properties:
+ SSL Context Service: SSLContextService
+ endpoint-list: localhost:8776
+ end-of-message-byte: d
+ reconnect-interval: 100ms
+ connection-attempt-timeout: 2000
+ - name: LogAttribute
+ id: 2438e3c8-015a-1000-79ca-83af40ec1992
+ class: org.apache.nifi.processors.standard.LogAttribute
+ max concurrent tasks: 1
+ scheduling strategy: TIMER_DRIVEN
+ scheduling period: 1 sec
+ penalization period: 30 sec
+ yield period: 1 sec
+ run duration nanos: 0
+ auto-terminated relationships list:
+ - success
+ Properties:
+ Log Level: info
+ Log Payload: true
+
+Connections:
+ - name: GetFile1/success/LogAttribute
+ id: 2438e3c8-015a-1000-79ca-83af40ec1997
+ source name: GetFile1
+ source id: 471deef6-2a6e-4a7d-912a-81cc17e3a206
+ source relationship name: success
+ destination name: LogAttribute
+ destination id: 2438e3c8-015a-1000-79ca-83af40ec1992
+ max work queue size: 0
+ max work queue data size: 1 MB
+ flowfile expiration: 60 sec
+ - name: GetFile2/success/LogAttribute
+ id: 2438e3c8-015a-1000-79ca-83af40ec1998
+ source name: GetFile2
+ source id: 471deef6-2a6e-4a7d-912a-81cc17e3a207
+ source relationship name: success
+ destination name: LogAttribute
+ destination id: 2438e3c8-015a-1000-79ca-83af40ec1992
+ max work queue size: 0
+ max work queue data size: 1 MB
+ flowfile expiration: 60 sec
+
+Remote Processing Groups:
+
[nifi-minifi-cpp] 03/03: MINIFICPP-1865 Integrate performance clang-tidy checks in CI
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szaszm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 746ade8170f445c3eb7c369e736ff439b87db1c6
Author: Gabor Gyimesi <ga...@gmail.com>
AuthorDate: Thu Aug 18 18:30:31 2022 +0200
MINIFICPP-1865 Integrate performance clang-tidy checks in CI
Closes #1377
Signed-off-by: Marton Szasz <sz...@apache.org>
---
.clang-tidy | 17 ++++++++++++-
extensions/aws/processors/PutS3Object.cpp | 4 ++--
extensions/civetweb/processors/ListenHTTP.cpp | 4 ++--
extensions/expression-language/Expression.cpp | 20 ++++++++--------
.../impl/expression/Expression.h | 4 ++--
extensions/http-curl/protocols/RESTSender.cpp | 2 +-
extensions/http-curl/protocols/RESTSender.h | 2 +-
extensions/http-curl/tests/HTTPSiteToSiteTests.cpp | 2 +-
.../http-curl/tests/unit/InvokeHTTPTests.cpp | 12 +++++-----
extensions/jni/jvm/JniFlowFile.cpp | 24 +++++++++----------
extensions/libarchive/ArchiveTests.h | 8 +++----
extensions/libarchive/BinFiles.cpp | 10 ++++----
extensions/libarchive/BinFiles.h | 4 ++--
extensions/libarchive/MergeContent.cpp | 6 ++---
extensions/libarchive/MergeContent.h | 2 +-
extensions/script/lua/LuaScriptEngine.cpp | 5 ++--
extensions/script/lua/LuaScriptEngine.h | 12 ++--------
extensions/script/python/PyProcessSession.cpp | 22 ++++++-----------
extensions/script/python/PyProcessSession.h | 20 +++++-----------
extensions/script/python/PythonBindings.h | 2 +-
extensions/sftp/client/SFTPClient.cpp | 2 +-
.../processors/DefragmentText.cpp | 2 +-
.../tests/integration/TestExecuteProcess.cpp | 20 +++++++---------
.../tests/unit/ProcessorTests.cpp | 6 ++---
.../tests/unit/TailFileTests.cpp | 3 ++-
libminifi/include/RemoteProcessorGroupPort.h | 14 +++--------
.../controller/StandardControllerServiceNode.h | 19 +++------------
.../reporting/SiteToSiteProvenanceReportingTask.h | 19 +++------------
libminifi/include/core/yaml/YamlConnectionParser.h | 14 ++---------
libminifi/include/io/tls/TLSServerSocket.h | 16 +++----------
libminifi/include/io/tls/TLSSocket.h | 21 ++++------------
libminifi/include/utils/ClassUtils.h | 19 +++------------
libminifi/include/utils/RegexUtils.h | 4 ++--
libminifi/include/utils/StringUtils.h | 2 +-
libminifi/src/RemoteProcessorGroupPort.cpp | 2 +-
libminifi/src/c2/C2Agent.cpp | 16 ++++++-------
libminifi/src/c2/C2Client.cpp | 9 +++----
.../controller/StandardControllerServiceNode.cpp | 18 ++++----------
.../SiteToSiteProvenanceReportingTask.cpp | 18 ++++----------
libminifi/src/core/yaml/YamlConnectionParser.cpp | 2 +-
libminifi/src/io/tls/TLSServerSocket.cpp | 14 +++--------
libminifi/src/io/tls/TLSSocket.cpp | 24 +++++++------------
libminifi/src/utils/ClassUtils.cpp | 15 +++---------
libminifi/src/utils/RegexUtils.cpp | 4 ++--
libminifi/src/utils/StringUtils.cpp | 6 ++---
.../test/archive-tests/ManipulateArchiveTests.cpp | 12 +++++-----
libminifi/test/archive-tests/MergeFileTests.cpp | 2 +-
libminifi/test/archive-tests/util/ArchiveTests.cpp | 16 ++++++-------
libminifi/test/rocksdb-tests/RocksDBTests.cpp | 1 +
libminifi/test/sql-tests/mocks/MockConnectors.cpp | 2 ++
libminifi/test/unit/BackTraceTests.cpp | 6 ++---
libminifi/test/unit/ExpectedTest.cpp | 28 +++++++++++-----------
libminifi/test/unit/IdTests.cpp | 2 +-
libminifi/test/unit/ThreadPoolTests.cpp | 6 ++---
54 files changed, 209 insertions(+), 337 deletions(-)
diff --git a/.clang-tidy b/.clang-tidy
index 224d7c595..6dac184f4 100644
--- a/.clang-tidy
+++ b/.clang-tidy
@@ -90,5 +90,20 @@ Checks: >
readability-string-compare,
readability-uniqueptr-delete-release,
readability-uppercase-literal-suffix,
- readability-use-anyofallof
+ readability-use-anyofallof,
+ performance-faster-string-find,
+ performance-for-range-copy,
+ performance-implicit-conversion-in-loop,
+ performance-inefficient-algorithm,
+ performance-inefficient-string-concatenation,
+ performance-inefficient-vector-operation,
+ performance-move-const-arg,
+ performance-move-constructor-init,
+ performance-no-automatic-move,
+ performance-no-int-to-ptr,
+ performance-noexcept-move-constructor,
+ performance-trivially-destructible,
+ performance-type-promotion-in-math-fn,
+ performance-unnecessary-copy-initialization,
+ performance-unnecessary-value-param
FormatStyle: 'file'
diff --git a/extensions/aws/processors/PutS3Object.cpp b/extensions/aws/processors/PutS3Object.cpp
index b312e9b53..22407b896 100644
--- a/extensions/aws/processors/PutS3Object.cpp
+++ b/extensions/aws/processors/PutS3Object.cpp
@@ -46,10 +46,10 @@ void PutS3Object::fillUserMetadata(const std::shared_ptr<core::ProcessContext> &
logger_->log_debug("PutS3Object: DynamicProperty: [%s] -> [%s]", prop_key, prop_value);
user_metadata_map_.emplace(prop_key, prop_value);
if (first_property) {
- user_metadata_ = prop_key + "=" + prop_value;
+ user_metadata_ = minifi::utils::StringUtils::join_pack(prop_key, "=", prop_value);
first_property = false;
} else {
- user_metadata_ += "," + prop_key + "=" + prop_value;
+ user_metadata_ += minifi::utils::StringUtils::join_pack(",", prop_key, "=", prop_value);
}
}
}
diff --git a/extensions/civetweb/processors/ListenHTTP.cpp b/extensions/civetweb/processors/ListenHTTP.cpp
index a790d0436..127ac332a 100644
--- a/extensions/civetweb/processors/ListenHTTP.cpp
+++ b/extensions/civetweb/processors/ListenHTTP.cpp
@@ -253,7 +253,7 @@ bool ListenHTTP::processIncomingFlowFile(core::ProcessSession &session) {
response.mime_type = "application/octet-stream";
}
response.body = to_string(session.readBuffer(flow_file));
- handler_->setResponseBody(std::move(response));
+ handler_->setResponseBody(response);
}
session.remove(flow_file);
@@ -435,7 +435,7 @@ void ListenHTTP::Handler::setResponseBody(const ResponseBody& response) {
logger_->log_info("Registering response body for URI '%s' of length %lu",
response.uri,
response.body.size());
- response_uri_map_[response.uri] = std::move(response);
+ response_uri_map_[response.uri] = response;
}
}
diff --git a/extensions/expression-language/Expression.cpp b/extensions/expression-language/Expression.cpp
index f25da3f9b..c52411c74 100644
--- a/extensions/expression-language/Expression.cpp
+++ b/extensions/expression-language/Expression.cpp
@@ -76,7 +76,7 @@ Expression compile(const std::string &expr_str) {
}
Expression make_static(std::string val) {
- return Expression(Value(val));
+ return Expression(Value(std::move(val)));
}
Expression make_dynamic(const std::function<Value(const Parameters ¶ms, const std::vector<Expression> &sub_exprs)> &val_fn) {
@@ -910,7 +910,7 @@ Expression make_dynamic_function_incomplete(const std::string &function_name, co
} else {
return make_dynamic([=](const Parameters ¶ms, const std::vector<Expression>& /*sub_exprs*/) -> Value {
std::vector<Value> evaluated_args;
-
+ evaluated_args.reserve(args.size());
for (const auto &arg : args) {
evaluated_args.emplace_back(arg(params));
}
@@ -1035,7 +1035,7 @@ Expression make_allAttributes(const std::string &function_name, const std::vecto
result.make_multi([=](const Parameters& /*params*/) -> std::vector<Expression> {
std::vector<Expression> out_exprs;
-
+ out_exprs.reserve(args.size());
for (const auto &arg : args) {
out_exprs.emplace_back(make_dynamic([=](const Parameters ¶ms,
const std::vector<Expression>& /*sub_exprs*/) -> Value {
@@ -1080,7 +1080,7 @@ Expression make_anyAttribute(const std::string &function_name, const std::vector
result.make_multi([=](const Parameters& /*params*/) -> std::vector<Expression> {
std::vector<Expression> out_exprs;
-
+ out_exprs.reserve(args.size());
for (const auto &arg : args) {
out_exprs.emplace_back(make_dynamic([=](const Parameters ¶ms,
const std::vector<Expression>& /*sub_exprs*/) -> Value {
@@ -1312,7 +1312,7 @@ Expression make_join(const std::string &function_name, const std::vector<Express
throw std::runtime_error(message_ss.str());
}
- auto delim_expr = args[1];
+ const auto& delim_expr = args[1];
return args[0].make_aggregate([delim_expr](const Parameters ¶ms,
const std::vector<Expression> &sub_exprs) -> Value {
@@ -1541,8 +1541,7 @@ Expression Expression::operator+(const Expression &other_expr) const {
other_val_fn,
other_sub_expr_generator](const Parameters ¶ms,
const std::vector<Expression>& /*sub_exprs*/) -> Value {
- Value result(val);
- return Value(result.asString().append(other_val_fn(params, other_sub_expr_generator(params)).asString()));
+ return Value(val.asString().append(other_val_fn(params, other_sub_expr_generator(params)).asString()));
});
} else if (!is_dynamic() && !other_expr.is_dynamic()) {
std::string result(val_.asString());
@@ -1561,13 +1560,14 @@ Value Expression::operator()(const Parameters ¶ms) const {
}
}
-Expression Expression::compose_multi(const std::function<Value(const std::vector<Value> &)> fn, const std::vector<Expression> &args) const {
+Expression Expression::compose_multi(const std::function<Value(const std::vector<Value> &)>& fn, const std::vector<Expression> &args) const {
auto result = make_dynamic(val_fn_);
auto compose_expr_generator = sub_expr_generator_;
result.sub_expr_generator_ = [=](const Parameters ¶ms) -> std::vector<Expression> {
auto sub_exprs = compose_expr_generator(params);
- std::vector<Expression> out_exprs {};
+ std::vector<Expression> out_exprs;
+ out_exprs.reserve(sub_exprs.size());
for (const auto &sub_expr : sub_exprs) {
out_exprs.emplace_back(make_dynamic([=](const Parameters ¶ms,
const std::vector<Expression>& /*sub_exprs*/) {
@@ -1590,7 +1590,7 @@ Expression Expression::compose_multi(const std::function<Value(const std::vector
return result;
}
-Expression Expression::make_aggregate(std::function<Value(const Parameters ¶ms, const std::vector<Expression> &sub_exprs)> val_fn) const {
+Expression Expression::make_aggregate(const std::function<Value(const Parameters ¶ms, const std::vector<Expression> &sub_exprs)>& val_fn) const {
auto sub_expr_generator = sub_expr_generator_;
return make_dynamic([sub_expr_generator,
val_fn](const Parameters ¶ms,
diff --git a/extensions/expression-language/impl/expression/Expression.h b/extensions/expression-language/impl/expression/Expression.h
index b9187f2ec..1fcdd173b 100644
--- a/extensions/expression-language/impl/expression/Expression.h
+++ b/extensions/expression-language/impl/expression/Expression.h
@@ -113,9 +113,9 @@ class Expression {
* @param args function arguments
* @return composed multi-expression
*/
- Expression compose_multi(std::function<Value(const std::vector<Value> &)> fn, const std::vector<Expression> &args) const;
+ Expression compose_multi(const std::function<Value(const std::vector<Value> &)>& fn, const std::vector<Expression> &args) const;
- Expression make_aggregate(std::function<Value(const Parameters ¶ms, const std::vector<Expression> &sub_exprs)> val_fn) const;
+ Expression make_aggregate(const std::function<Value(const Parameters ¶ms, const std::vector<Expression> &sub_exprs)>& val_fn) const;
protected:
Value val_;
diff --git a/extensions/http-curl/protocols/RESTSender.cpp b/extensions/http-curl/protocols/RESTSender.cpp
index a6cbe7d27..d39dc158f 100644
--- a/extensions/http-curl/protocols/RESTSender.cpp
+++ b/extensions/http-curl/protocols/RESTSender.cpp
@@ -92,7 +92,7 @@ void RESTSender::setSecurityContext(utils::HTTPClient &client, const std::string
client.initialize(type, url, generatedService);
}
-C2Payload RESTSender::sendPayload(const std::string url, const Direction direction, const C2Payload &payload, std::optional<std::string> data) {
+C2Payload RESTSender::sendPayload(const std::string& url, const Direction direction, const C2Payload &payload, std::optional<std::string> data) {
if (url.empty()) {
return {payload.getOperation(), state::UpdateState::READ_ERROR};
}
diff --git a/extensions/http-curl/protocols/RESTSender.h b/extensions/http-curl/protocols/RESTSender.h
index ff5c9f561..9fb96f35d 100644
--- a/extensions/http-curl/protocols/RESTSender.h
+++ b/extensions/http-curl/protocols/RESTSender.h
@@ -55,7 +55,7 @@ class RESTSender : public RESTProtocol, public C2Protocol {
void initialize(core::controller::ControllerServiceProvider* controller, const std::shared_ptr<Configure> &configure) override;
protected:
- C2Payload sendPayload(const std::string url, const Direction direction, const C2Payload &payload, std::optional<std::string> data);
+ C2Payload sendPayload(const std::string& url, const Direction direction, const C2Payload &payload, std::optional<std::string> data);
/**
* Initializes the SSLContextService onto the HTTP client if one is needed
diff --git a/extensions/http-curl/tests/HTTPSiteToSiteTests.cpp b/extensions/http-curl/tests/HTTPSiteToSiteTests.cpp
index fbb33db7b..b14dacb21 100644
--- a/extensions/http-curl/tests/HTTPSiteToSiteTests.cpp
+++ b/extensions/http-curl/tests/HTTPSiteToSiteTests.cpp
@@ -93,7 +93,7 @@ struct test_profile {
bool invalid_checksum{false};
};
-void run_variance(std::string test_file_location, bool isSecure, std::string url, const struct test_profile &profile) {
+void run_variance(const std::string& test_file_location, bool isSecure, const std::string& url, const struct test_profile &profile) {
SiteToSiteTestHarness harness(isSecure);
std::string in_port = "471deef6-2a6e-4a7d-912a-81cc17e3a204";
diff --git a/extensions/http-curl/tests/unit/InvokeHTTPTests.cpp b/extensions/http-curl/tests/unit/InvokeHTTPTests.cpp
index 45938306a..37a2d85e9 100644
--- a/extensions/http-curl/tests/unit/InvokeHTTPTests.cpp
+++ b/extensions/http-curl/tests/unit/InvokeHTTPTests.cpp
@@ -103,7 +103,7 @@ TEST_CASE("HTTPTestsWithNoResourceClaimPOST", "[httptest1]") {
auto records = reporter->getEvents();
auto record = session->get();
REQUIRE(record == nullptr);
- REQUIRE(records.size() == 0);
+ REQUIRE(records.empty());
reporter = session->getProvenanceReporter();
@@ -117,7 +117,7 @@ TEST_CASE("HTTPTestsWithNoResourceClaimPOST", "[httptest1]") {
session->commit();
records = reporter->getEvents();
// FIXME(fgerlits): this test is very weak, as `records` is empty
- for (auto provEventRecord : records) {
+ for (const auto& provEventRecord : records) {
REQUIRE(provEventRecord->getComponentType() == TestHTTPServer::PROCESSOR_NAME);
}
@@ -197,7 +197,7 @@ TEST_CASE("HTTPTestsWithResourceClaimPOST", "[httptest1]") {
auto records = reporter->getEvents();
record = session->get();
REQUIRE(record == nullptr);
- REQUIRE(records.size() == 0);
+ REQUIRE(records.empty());
listenhttp->incrementActiveTasks();
listenhttp->setScheduledState(core::ScheduledState::RUNNING);
@@ -215,7 +215,7 @@ TEST_CASE("HTTPTestsWithResourceClaimPOST", "[httptest1]") {
session2->commit();
records = reporter->getEvents();
// FIXME(fgerlits): this test is very weak, as `records` is empty
- for (auto provEventRecord : records) {
+ for (const auto& provEventRecord : records) {
REQUIRE(provEventRecord->getComponentType() == listenhttp->getName());
}
@@ -238,14 +238,14 @@ TEST_CASE("HTTPTestsPostNoResourceClaim", "[httptest1]") {
auto records = plan->getProvenanceRecords();
std::shared_ptr<core::FlowFile> record = plan->getCurrentFlowFile();
REQUIRE(record == nullptr);
- REQUIRE(records.size() == 0);
+ REQUIRE(records.empty());
plan->reset();
testController.runSession(plan);
records = plan->getProvenanceRecords();
// FIXME(fgerlits): this test is very weak, as `records` is empty
- for (auto provEventRecord : records) {
+ for (const auto& provEventRecord : records) {
REQUIRE(provEventRecord->getComponentType() == TestHTTPServer::PROCESSOR_NAME);
}
diff --git a/extensions/jni/jvm/JniFlowFile.cpp b/extensions/jni/jvm/JniFlowFile.cpp
index 523ec9f8a..a4861e0ed 100644
--- a/extensions/jni/jvm/JniFlowFile.cpp
+++ b/extensions/jni/jvm/JniFlowFile.cpp
@@ -41,7 +41,7 @@ extern "C" {
namespace minifi = org::apache::nifi::minifi;
JNIEXPORT jlong JNICALL Java_org_apache_nifi_processor_JniFlowFile_getId(JNIEnv *env, jobject obj) {
- minifi::jni::JniFlowFile *ptr = minifi::jni::JVMLoader::getInstance()->getReference<minifi::jni::JniFlowFile>(env, obj);
+ auto *ptr = minifi::jni::JVMLoader::getInstance()->getReference<minifi::jni::JniFlowFile>(env, obj);
auto ff = ptr->get();
THROW_IF_NULL(ff, env, NO_FF_OBJECT);
@@ -49,7 +49,7 @@ JNIEXPORT jlong JNICALL Java_org_apache_nifi_processor_JniFlowFile_getId(JNIEnv
return id;
}
JNIEXPORT jlong JNICALL Java_org_apache_nifi_processor_JniFlowFile_getEntryDate(JNIEnv *env, jobject obj) {
- minifi::jni::JniFlowFile *ptr = minifi::jni::JVMLoader::getInstance()->getReference<minifi::jni::JniFlowFile>(env, obj);
+ auto *ptr = minifi::jni::JVMLoader::getInstance()->getReference<minifi::jni::JniFlowFile>(env, obj);
auto ff = ptr->get();
THROW_IF_NULL(ff, env, NO_FF_OBJECT);
@@ -57,7 +57,7 @@ JNIEXPORT jlong JNICALL Java_org_apache_nifi_processor_JniFlowFile_getEntryDate(
return entryDate;
}
JNIEXPORT jlong JNICALL Java_org_apache_nifi_processor_JniFlowFile_getLineageStartDate(JNIEnv *env, jobject obj) {
- minifi::jni::JniFlowFile *ptr = minifi::jni::JVMLoader::getInstance()->getReference<minifi::jni::JniFlowFile>(env, obj);
+ auto *ptr = minifi::jni::JVMLoader::getInstance()->getReference<minifi::jni::JniFlowFile>(env, obj);
auto ff = ptr->get();
THROW_IF_NULL(ff, env, NO_FF_OBJECT);
@@ -65,7 +65,7 @@ JNIEXPORT jlong JNICALL Java_org_apache_nifi_processor_JniFlowFile_getLineageSta
return val;
}
JNIEXPORT jlong JNICALL Java_org_apache_nifi_processor_JniFlowFile_getLineageStartIndex(JNIEnv *env, jobject obj) {
- minifi::jni::JniFlowFile *ptr = minifi::jni::JVMLoader::getInstance()->getReference<minifi::jni::JniFlowFile>(env, obj);
+ auto *ptr = minifi::jni::JVMLoader::getInstance()->getReference<minifi::jni::JniFlowFile>(env, obj);
auto ff = ptr->get();
THROW_IF_NULL(ff, env, NO_FF_OBJECT);
@@ -73,7 +73,7 @@ JNIEXPORT jlong JNICALL Java_org_apache_nifi_processor_JniFlowFile_getLineageSta
return val;
}
JNIEXPORT jlong JNICALL Java_org_apache_nifi_processor_JniFlowFile_getLastQueueDatePrim(JNIEnv *env, jobject obj) {
- minifi::jni::JniFlowFile *ptr = minifi::jni::JVMLoader::getInstance()->getReference<minifi::jni::JniFlowFile>(env, obj);
+ auto *ptr = minifi::jni::JVMLoader::getInstance()->getReference<minifi::jni::JniFlowFile>(env, obj);
auto ff = ptr->get();
THROW_IF_NULL(ff, env, NO_FF_OBJECT);
@@ -81,7 +81,7 @@ JNIEXPORT jlong JNICALL Java_org_apache_nifi_processor_JniFlowFile_getLastQueueD
return val;
}
JNIEXPORT jlong JNICALL Java_org_apache_nifi_processor_JniFlowFile_getQueueDateIndex(JNIEnv *env, jobject obj) {
- minifi::jni::JniFlowFile *ptr = minifi::jni::JVMLoader::getInstance()->getReference<minifi::jni::JniFlowFile>(env, obj);
+ auto *ptr = minifi::jni::JVMLoader::getInstance()->getReference<minifi::jni::JniFlowFile>(env, obj);
auto ff = ptr->get();
THROW_IF_NULL(ff, env, NO_FF_OBJECT);
@@ -89,7 +89,7 @@ JNIEXPORT jlong JNICALL Java_org_apache_nifi_processor_JniFlowFile_getQueueDateI
return val;
}
JNIEXPORT jboolean JNICALL Java_org_apache_nifi_processor_JniFlowFile_isPenalized(JNIEnv *env, jobject obj) {
- minifi::jni::JniFlowFile *ptr = minifi::jni::JVMLoader::getInstance()->getReference<minifi::jni::JniFlowFile>(env, obj);
+ auto *ptr = minifi::jni::JVMLoader::getInstance()->getReference<minifi::jni::JniFlowFile>(env, obj);
auto ff = ptr->get();
THROW_IF_NULL(ff, env, NO_FF_OBJECT);
@@ -97,7 +97,7 @@ JNIEXPORT jboolean JNICALL Java_org_apache_nifi_processor_JniFlowFile_isPenalize
return val;
}
JNIEXPORT jstring JNICALL Java_org_apache_nifi_processor_JniFlowFile_getAttribute(JNIEnv *env, jobject obj, jstring key) {
- minifi::jni::JniFlowFile *ptr = minifi::jni::JVMLoader::getInstance()->getReference<minifi::jni::JniFlowFile>(env, obj);
+ auto *ptr = minifi::jni::JVMLoader::getInstance()->getReference<minifi::jni::JniFlowFile>(env, obj);
auto ff = ptr->get();
THROW_IF_NULL(ff, env, NO_FF_OBJECT);
@@ -106,14 +106,14 @@ JNIEXPORT jstring JNICALL Java_org_apache_nifi_processor_JniFlowFile_getAttribut
return env->NewStringUTF(value.c_str());
}
JNIEXPORT jlong JNICALL Java_org_apache_nifi_processor_JniFlowFile_getSize(JNIEnv *env, jobject obj) {
- minifi::jni::JniFlowFile *ptr = minifi::jni::JVMLoader::getInstance()->getReference<minifi::jni::JniFlowFile>(env, obj);
+ auto *ptr = minifi::jni::JVMLoader::getInstance()->getReference<minifi::jni::JniFlowFile>(env, obj);
auto ff = ptr->get();
THROW_IF_NULL(ff, env, NO_FF_OBJECT);
jlong val = ff->getSize();
return val;
}
JNIEXPORT jstring JNICALL Java_org_apache_nifi_processor_JniFlowFile_getUUIDStr(JNIEnv *env, jobject obj) {
- minifi::jni::JniFlowFile *ptr = minifi::jni::JVMLoader::getInstance()->getReference<minifi::jni::JniFlowFile>(env, obj);
+ auto *ptr = minifi::jni::JVMLoader::getInstance()->getReference<minifi::jni::JniFlowFile>(env, obj);
auto ff = ptr->get();
THROW_IF_NULL(ff, env, NO_FF_OBJECT);
@@ -121,7 +121,7 @@ JNIEXPORT jstring JNICALL Java_org_apache_nifi_processor_JniFlowFile_getUUIDStr
}
JNIEXPORT jobject JNICALL Java_org_apache_nifi_processor_JniFlowFile_getAttributes(JNIEnv *env, jobject obj) {
- minifi::jni::JniFlowFile *ptr = minifi::jni::JVMLoader::getInstance()->getReference<minifi::jni::JniFlowFile>(env, obj);
+ auto *ptr = minifi::jni::JVMLoader::getInstance()->getReference<minifi::jni::JniFlowFile>(env, obj);
auto ff = ptr->get();
jclass mapClass = env->FindClass("java/util/HashMap");
@@ -136,7 +136,7 @@ JNIEXPORT jobject JNICALL Java_org_apache_nifi_processor_JniFlowFile_getAttribut
jmethodID put = env->GetMethodID(mapClass, "put", "(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;");
- for (auto kf : ff->getAttributes()) {
+ for (const auto& kf : ff->getAttributes()) {
env->CallObjectMethod(hashMap, put, env->NewStringUTF(kf.first.c_str()), env->NewStringUTF(kf.second.c_str()));
minifi::jni::ThrowIf(env);
}
diff --git a/extensions/libarchive/ArchiveTests.h b/extensions/libarchive/ArchiveTests.h
index a8d60cd5f..36fdb1955 100644
--- a/extensions/libarchive/ArchiveTests.h
+++ b/extensions/libarchive/ArchiveTests.h
@@ -55,9 +55,9 @@ FN_VEC_T build_test_archive_order(int, const char**);
OrderedTestArchive build_ordered_test_archive(int, const char**, const char**);
-void build_test_archive(std::string, TAE_MAP_T entries, FN_VEC_T order = FN_VEC_T());
-void build_test_archive(std::string, OrderedTestArchive);
+void build_test_archive(const std::string&, const TAE_MAP_T& entries, FN_VEC_T order = FN_VEC_T());
+void build_test_archive(const std::string&, OrderedTestArchive&);
-bool check_archive_contents(std::string, TAE_MAP_T entries, bool check_attributes = true, FN_VEC_T order = FN_VEC_T());
-bool check_archive_contents(std::string, OrderedTestArchive, bool check_attributes = true);
+bool check_archive_contents(const std::string&, const TAE_MAP_T& entries, bool check_attributes = true, const FN_VEC_T& order = FN_VEC_T());
+bool check_archive_contents(const std::string&, const OrderedTestArchive&, bool check_attributes = true);
diff --git a/extensions/libarchive/BinFiles.cpp b/extensions/libarchive/BinFiles.cpp
index 9dd0084b8..82b31c282 100644
--- a/extensions/libarchive/BinFiles.cpp
+++ b/extensions/libarchive/BinFiles.cpp
@@ -80,7 +80,7 @@ void BinFiles::onSchedule(core::ProcessContext *context, core::ProcessSessionFac
}
}
-void BinFiles::preprocessFlowFile(core::ProcessContext* /*context*/, core::ProcessSession* /*session*/, std::shared_ptr<core::FlowFile> flow) {
+void BinFiles::preprocessFlowFile(core::ProcessContext* /*context*/, core::ProcessSession* /*session*/, const std::shared_ptr<core::FlowFile>& flow) {
// handle backward compatibility with old segment attributes
std::string value;
if (!flow->getAttribute(BinFiles::FRAGMENT_COUNT_ATTRIBUTE, value) && flow->getAttribute(BinFiles::SEGMENT_COUNT_ATTRIBUTE, value)) {
@@ -113,7 +113,7 @@ void BinManager::gatherReadyBins() {
emptyQueue.push_back(group_id);
}
}
- for (auto group : emptyQueue) {
+ for (const auto& group : emptyQueue) {
// erase from the map if the queue is empty for the group
groupBinMap_.erase(group);
}
@@ -156,7 +156,7 @@ void BinManager::getReadyBin(std::deque<std::unique_ptr<Bin>> &retBins) {
}
}
-bool BinManager::offer(const std::string &group, std::shared_ptr<core::FlowFile> flow) {
+bool BinManager::offer(const std::string &group, const std::shared_ptr<core::FlowFile>& flow) {
std::lock_guard < std::mutex > lock(mutex_);
if (flow->getSize() > maxSize_) {
// could not be added to a bin -- too large by itself, so create a separate bin for just this guy.
@@ -279,7 +279,7 @@ void BinFiles::onTrigger(const std::shared_ptr<core::ProcessContext> &context, c
void BinFiles::transferFlowsToFail(core::ProcessContext* /*context*/, core::ProcessSession *session, std::unique_ptr<Bin> &bin) {
std::deque<std::shared_ptr<core::FlowFile>> &flows = bin->getFlowFile();
- for (auto flow : flows) {
+ for (const auto& flow : flows) {
session->transfer(flow, Failure);
}
flows.clear();
@@ -287,7 +287,7 @@ void BinFiles::transferFlowsToFail(core::ProcessContext* /*context*/, core::Proc
void BinFiles::addFlowsToSession(core::ProcessContext* /*context*/, core::ProcessSession *session, std::unique_ptr<Bin> &bin) {
std::deque<std::shared_ptr<core::FlowFile>> &flows = bin->getFlowFile();
- for (auto flow : flows) {
+ for (const auto& flow : flows) {
session->add(flow);
}
}
diff --git a/extensions/libarchive/BinFiles.h b/extensions/libarchive/BinFiles.h
index fc72ec147..71cece114 100644
--- a/extensions/libarchive/BinFiles.h
+++ b/extensions/libarchive/BinFiles.h
@@ -162,7 +162,7 @@ class BinManager {
binCount_ = 0;
}
// Adds the given flowFile to the first available bin in which it fits for the given group or creates a new bin in the specified group if necessary.
- bool offer(const std::string &group, std::shared_ptr<core::FlowFile> flow);
+ bool offer(const std::string &group, const std::shared_ptr<core::FlowFile>& flow);
// gather ready bins once the bin are full enough or exceed bin age
void gatherReadyBins();
// marks oldest bin as ready
@@ -253,7 +253,7 @@ class BinFiles : public core::Processor {
protected:
// Allows general pre-processing of a flow file before it is offered to a bin. This is called before getGroupId().
- virtual void preprocessFlowFile(core::ProcessContext *context, core::ProcessSession *session, std::shared_ptr<core::FlowFile> flow);
+ virtual void preprocessFlowFile(core::ProcessContext *context, core::ProcessSession *session, const std::shared_ptr<core::FlowFile>& flow);
// Returns a group ID representing a bin. This allows flow files to be binned into like groups
virtual std::string getGroupId(core::ProcessContext* /*context*/, std::shared_ptr<core::FlowFile> /*flow*/) {
return "";
diff --git a/extensions/libarchive/MergeContent.cpp b/extensions/libarchive/MergeContent.cpp
index c067a5a00..4a2d69f7d 100644
--- a/extensions/libarchive/MergeContent.cpp
+++ b/extensions/libarchive/MergeContent.cpp
@@ -36,7 +36,7 @@ void MergeContent::initialize() {
setSupportedRelationships(relationships());
}
-std::string MergeContent::readContent(std::string path) {
+std::string MergeContent::readContent(const std::string& path) {
std::string contents;
std::ifstream in(path.c_str(), std::ios::in | std::ios::binary);
if (in) {
@@ -159,7 +159,7 @@ bool MergeContent::checkDefragment(std::unique_ptr<Bin> &bin) {
catch (...) {
return false;
}
- for (auto flow : flows) {
+ for (const auto& flow : flows) {
std::string value;
if (!flow->getAttribute(BinFiles::FRAGMENT_ID_ATTRIBUTE, value))
return false;
@@ -262,7 +262,7 @@ bool MergeContent::processBin(core::ProcessContext *context, core::ProcessSessio
// we successfully merge the flow
session->transfer(merge_flow, Merge);
std::deque<std::shared_ptr<core::FlowFile>> &flows = bin->getFlowFile();
- for (auto flow : flows) {
+ for (const auto& flow : flows) {
session->transfer(flow, Original);
}
logger_->log_info("Merge FlowFile record UUID %s, payload length %d", merge_flow->getUUIDStr(), merge_flow->getSize());
diff --git a/extensions/libarchive/MergeContent.h b/extensions/libarchive/MergeContent.h
index 5d986280d..40bf1096b 100644
--- a/extensions/libarchive/MergeContent.h
+++ b/extensions/libarchive/MergeContent.h
@@ -363,7 +363,7 @@ class MergeContent : public processors::BinFiles {
std::string footerContent_;
std::string demarcatorContent_;
std::string attributeStrategy_;
- static std::string readContent(std::string path);
+ static std::string readContent(const std::string& path);
};
} // namespace org::apache::nifi::minifi::processors
diff --git a/extensions/script/lua/LuaScriptEngine.cpp b/extensions/script/lua/LuaScriptEngine.cpp
index 6e82d592c..48183c7b1 100644
--- a/extensions/script/lua/LuaScriptEngine.cpp
+++ b/extensions/script/lua/LuaScriptEngine.cpp
@@ -23,6 +23,7 @@
#include "LuaScriptEngine.h"
#include "LuaProcessSession.h"
+#include "utils/StringUtils.h"
namespace org::apache::nifi::minifi::lua {
@@ -62,9 +63,9 @@ LuaScriptEngine::LuaScriptEngine() {
void LuaScriptEngine::executeScriptWithAppendedModulePaths(std::string& script) {
for (const auto& module_path : module_paths_) {
if (std::filesystem::is_regular_file(std::filesystem::status(module_path))) {
- script = "package.path = package.path .. \";" + module_path + "\"\n" + script;
+ script = utils::StringUtils::join_pack("package.path = package.path .. \";", module_path, "\"\n", script);
} else {
- script = "package.path = package.path .. \";" + module_path + "/?.lua\"\n" + script;
+ script = utils::StringUtils::join_pack("package.path = package.path .. \";", module_path, "/?.lua\"\n", script);
}
}
lua_.script(script, sol::script_throw_on_error);
diff --git a/extensions/script/lua/LuaScriptEngine.h b/extensions/script/lua/LuaScriptEngine.h
index 8873335c9..0838d121f 100644
--- a/extensions/script/lua/LuaScriptEngine.h
+++ b/extensions/script/lua/LuaScriptEngine.h
@@ -31,11 +31,7 @@
#include "LuaProcessSession.h"
-namespace org {
-namespace apache {
-namespace nifi {
-namespace minifi {
-namespace lua {
+namespace org::apache::nifi::minifi::lua {
class LuaScriptEngine : public script::ScriptEngine {
public:
@@ -115,8 +111,4 @@ class LuaScriptEngine : public script::ScriptEngine {
sol::state lua_;
};
-} /* namespace lua */
-} /* namespace minifi */
-} /* namespace nifi */
-} /* namespace apache */
-} /* namespace org */
+} // namespace org::apache::nifi::minifi::lua
diff --git a/extensions/script/python/PyProcessSession.cpp b/extensions/script/python/PyProcessSession.cpp
index 78b92dd57..d9047e0e0 100644
--- a/extensions/script/python/PyProcessSession.cpp
+++ b/extensions/script/python/PyProcessSession.cpp
@@ -23,11 +23,7 @@
#include "PyProcessSession.h"
-namespace org {
-namespace apache {
-namespace nifi {
-namespace minifi {
-namespace python {
+namespace org::apache::nifi::minifi::python {
namespace py = pybind11;
namespace core = org::apache::nifi::minifi::core;
@@ -53,8 +49,8 @@ std::shared_ptr<script::ScriptFlowFile> PyProcessSession::get() {
return result;
}
-void PyProcessSession::transfer(std::shared_ptr<script::ScriptFlowFile> script_flow_file,
- core::Relationship relationship) {
+void PyProcessSession::transfer(const std::shared_ptr<script::ScriptFlowFile>& script_flow_file,
+ const core::Relationship& relationship) {
if (!session_) {
throw std::runtime_error("Access of ProcessSession after it has been released");
}
@@ -68,7 +64,7 @@ void PyProcessSession::transfer(std::shared_ptr<script::ScriptFlowFile> script_f
session_->transfer(flow_file, relationship);
}
-void PyProcessSession::read(std::shared_ptr<script::ScriptFlowFile> script_flow_file,
+void PyProcessSession::read(const std::shared_ptr<script::ScriptFlowFile>& script_flow_file,
py::object input_stream_callback) {
if (!session_) {
throw std::runtime_error("Access of ProcessSession after it has been released");
@@ -85,7 +81,7 @@ void PyProcessSession::read(std::shared_ptr<script::ScriptFlowFile> script_flow_
});
}
-void PyProcessSession::write(std::shared_ptr<script::ScriptFlowFile> script_flow_file,
+void PyProcessSession::write(const std::shared_ptr<script::ScriptFlowFile>& script_flow_file,
py::object output_stream_callback) {
if (!session_) {
throw std::runtime_error("Access of ProcessSession after it has been released");
@@ -112,7 +108,7 @@ std::shared_ptr<script::ScriptFlowFile> PyProcessSession::create() {
return result;
}
-std::shared_ptr<script::ScriptFlowFile> PyProcessSession::create(std::shared_ptr<script::ScriptFlowFile> flow_file) {
+std::shared_ptr<script::ScriptFlowFile> PyProcessSession::create(const std::shared_ptr<script::ScriptFlowFile>& flow_file) {
if (!session_) {
throw std::runtime_error("Access of ProcessSession after it has been released");
}
@@ -139,8 +135,4 @@ void PyProcessSession::releaseCoreResources() {
session_.reset();
}
-} /* namespace python */
-} /* namespace minifi */
-} /* namespace nifi */
-} /* namespace apache */
-} /* namespace org */
+} // namespace org::apache::nifi::minifi::python
diff --git a/extensions/script/python/PyProcessSession.h b/extensions/script/python/PyProcessSession.h
index 2cdee8a3e..5637159b9 100644
--- a/extensions/script/python/PyProcessSession.h
+++ b/extensions/script/python/PyProcessSession.h
@@ -29,11 +29,7 @@
#pragma GCC visibility push(hidden)
-namespace org {
-namespace apache {
-namespace nifi {
-namespace minifi {
-namespace python {
+namespace org::apache::nifi::minifi::python {
namespace py = pybind11;
@@ -43,10 +39,10 @@ class PyProcessSession {
std::shared_ptr<script::ScriptFlowFile> get();
std::shared_ptr<script::ScriptFlowFile> create();
- std::shared_ptr<script::ScriptFlowFile> create(std::shared_ptr<script::ScriptFlowFile> flow_file);
- void transfer(std::shared_ptr<script::ScriptFlowFile> flow_file, core::Relationship relationship);
- void read(std::shared_ptr<script::ScriptFlowFile> flow_file, py::object input_stream_callback);
- void write(std::shared_ptr<script::ScriptFlowFile> flow_file, py::object output_stream_callback);
+ std::shared_ptr<script::ScriptFlowFile> create(const std::shared_ptr<script::ScriptFlowFile>& flow_file);
+ void transfer(const std::shared_ptr<script::ScriptFlowFile>& flow_file, const core::Relationship& relationship);
+ void read(const std::shared_ptr<script::ScriptFlowFile>& flow_file, py::object input_stream_callback);
+ void write(const std::shared_ptr<script::ScriptFlowFile>& flow_file, py::object output_stream_callback);
/**
* Sometimes we want to release shared pointers to core resources when
@@ -63,10 +59,6 @@ class PyProcessSession {
std::shared_ptr<core::ProcessSession> session_;
};
-} /* namespace python */
-} /* namespace minifi */
-} /* namespace nifi */
-} /* namespace apache */
-} /* namespace org */
+} // namespace org::apache::nifi::minifi::python
#pragma GCC visibility pop
diff --git a/extensions/script/python/PythonBindings.h b/extensions/script/python/PythonBindings.h
index 443e74f77..15f68b3c7 100644
--- a/extensions/script/python/PythonBindings.h
+++ b/extensions/script/python/PythonBindings.h
@@ -50,7 +50,7 @@ PYBIND11_EMBEDDED_MODULE(minifi_native, m) { // NOLINT
.def("create",
static_cast<std::shared_ptr<script::ScriptFlowFile> (python::PyProcessSession::*)()>(&python::PyProcessSession::create))
.def("create",
- static_cast<std::shared_ptr<script::ScriptFlowFile> (python::PyProcessSession::*)(std::shared_ptr<script::ScriptFlowFile>)>(&python::PyProcessSession::create))
+ static_cast<std::shared_ptr<script::ScriptFlowFile> (python::PyProcessSession::*)(const std::shared_ptr<script::ScriptFlowFile>&)>(&python::PyProcessSession::create))
.def("read", &python::PyProcessSession::read)
.def("write", &python::PyProcessSession::write)
.def("transfer", &python::PyProcessSession::transfer);
diff --git a/extensions/sftp/client/SFTPClient.cpp b/extensions/sftp/client/SFTPClient.cpp
index 39872d309..7afcc98df 100644
--- a/extensions/sftp/client/SFTPClient.cpp
+++ b/extensions/sftp/client/SFTPClient.cpp
@@ -724,7 +724,7 @@ bool SFTPClient::listDirectory(const std::string& path, bool follow_symlinks,
attrs = orig_attrs;
}
}
- children_result.emplace_back(std::string(filename.data()), std::string(longentry.data()), std::move(attrs));
+ children_result.emplace_back(std::string(filename.data()), std::string(longentry.data()), attrs);
} while (true);
return true;
}
diff --git a/extensions/standard-processors/processors/DefragmentText.cpp b/extensions/standard-processors/processors/DefragmentText.cpp
index 18d83f65e..623534889 100644
--- a/extensions/standard-processors/processors/DefragmentText.cpp
+++ b/extensions/standard-processors/processors/DefragmentText.cpp
@@ -94,7 +94,7 @@ void DefragmentText::onTrigger(core::ProcessContext*, core::ProcessSession* sess
auto flowFiles = flow_file_store_.getNewFlowFiles();
for (auto& file : flowFiles) {
if (file)
- processNextFragment(session, gsl::not_null(std::move(file)));
+ processNextFragment(session, gsl::not_null(file));
}
{
std::shared_ptr<core::FlowFile> original_flow_file = session->get();
diff --git a/extensions/standard-processors/tests/integration/TestExecuteProcess.cpp b/extensions/standard-processors/tests/integration/TestExecuteProcess.cpp
index c16c59ddf..7b932244e 100644
--- a/extensions/standard-processors/tests/integration/TestExecuteProcess.cpp
+++ b/extensions/standard-processors/tests/integration/TestExecuteProcess.cpp
@@ -87,17 +87,15 @@ int main(int /*argc*/, char ** /*argv*/) {
core::ProcessSessionFactory factory(contextset);
processor->onSchedule(contextset.get(), &factory);
- for (int i = 0; i < 1; i++) {
- processor_workers.push_back(std::thread([processor, test_repo, &is_ready]() {
- auto node = std::make_shared<core::ProcessorNode>(processor.get());
- auto context = std::make_shared<core::ProcessContext>(node, nullptr, test_repo, test_repo);
- context->setProperty(org::apache::nifi::minifi::processors::ExecuteProcess::Command, "sleep 0.5");
- auto session = std::make_shared<core::ProcessSession>(context);
- while (!is_ready.load(std::memory_order_relaxed)) {
- }
- processor->onTrigger(context.get(), session.get());
- }));
- }
+ processor_workers.push_back(std::thread([processor, test_repo, &is_ready]() {
+ auto node = std::make_shared<core::ProcessorNode>(processor.get());
+ auto context = std::make_shared<core::ProcessContext>(node, nullptr, test_repo, test_repo);
+ context->setProperty(org::apache::nifi::minifi::processors::ExecuteProcess::Command, "sleep 0.5");
+ auto session = std::make_shared<core::ProcessSession>(context);
+ while (!is_ready.load(std::memory_order_relaxed)) {
+ }
+ processor->onTrigger(context.get(), session.get());
+ }));
is_ready.store(true, std::memory_order_relaxed);
diff --git a/extensions/standard-processors/tests/unit/ProcessorTests.cpp b/extensions/standard-processors/tests/unit/ProcessorTests.cpp
index fe00f439c..d563baf59 100644
--- a/extensions/standard-processors/tests/unit/ProcessorTests.cpp
+++ b/extensions/standard-processors/tests/unit/ProcessorTests.cpp
@@ -225,7 +225,7 @@ TEST_CASE("Test GetFile Ignore", "[getfileCreate3]") {
records = reporter->getEvents();
- for (auto provEventRecord : records) {
+ for (const auto& provEventRecord : records) {
REQUIRE(provEventRecord->getComponentType() == processor->getName());
}
session->commit();
@@ -445,7 +445,7 @@ TEST_CASE("Test Find file", "[getfileCreate3]") {
records = plan->getProvenanceRecords();
record = plan->getCurrentFlowFile();
- for (auto provEventRecord : records) {
+ for (const auto& provEventRecord : records) {
REQUIRE(provEventRecord->getComponentType() == processor->getName());
}
std::shared_ptr<core::FlowFile> ffr = plan->getCurrentFlowFile();
@@ -459,7 +459,7 @@ TEST_CASE("Test Find file", "[getfileCreate3]") {
newRecord.DeSerialize(gsl::make_span(entry.second).as_span<const std::byte>());
bool found = false;
- for (auto provRec : records) {
+ for (const auto& provRec : records) {
if (provRec->getEventId() == newRecord.getEventId()) {
REQUIRE(provRec->getEventId() == newRecord.getEventId());
REQUIRE(provRec->getComponentId() == newRecord.getComponentId());
diff --git a/extensions/standard-processors/tests/unit/TailFileTests.cpp b/extensions/standard-processors/tests/unit/TailFileTests.cpp
index 1bd4319a5..387f579a2 100644
--- a/extensions/standard-processors/tests/unit/TailFileTests.cpp
+++ b/extensions/standard-processors/tests/unit/TailFileTests.cpp
@@ -43,6 +43,7 @@
#include "TailFile.h"
#include "LogAttribute.h"
#include "utils/TestUtils.h"
+#include "utils/StringUtils.h"
using namespace std::literals::chrono_literals;
@@ -1349,7 +1350,7 @@ TEST_CASE("TailFile handles the Delimiter setting correctly", "[delimiter]") {
auto temp_directory = testController.createTempDirectory();
std::string delimiter = test_case.second;
- std::string full_file_name = createTempFile(temp_directory, "test.log", "one" + delimiter + "two" + delimiter);
+ std::string full_file_name = createTempFile(temp_directory, "test.log", utils::StringUtils::join_pack("one", delimiter, "two", delimiter));
auto plan = testController.createPlan();
diff --git a/libminifi/include/RemoteProcessorGroupPort.h b/libminifi/include/RemoteProcessorGroupPort.h
index 98663ed69..c78413ccb 100644
--- a/libminifi/include/RemoteProcessorGroupPort.h
+++ b/libminifi/include/RemoteProcessorGroupPort.h
@@ -17,8 +17,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-#ifndef LIBMINIFI_INCLUDE_REMOTEPROCESSORGROUPPORT_H_
-#define LIBMINIFI_INCLUDE_REMOTEPROCESSORGROUPPORT_H_
+#pragma once
#include <string>
#include <utility>
@@ -37,10 +36,7 @@
#include "core/logging/LoggerConfiguration.h"
#include "utils/Export.h"
-namespace org {
-namespace apache {
-namespace nifi {
-namespace minifi {
+namespace org::apache::nifi::minifi {
/**
* Count down latch implementation that's used across
@@ -241,8 +237,4 @@ class RemoteProcessorGroupPort : public core::Processor {
static const char* RPG_SSL_CONTEXT_SERVICE_NAME;
};
-} // namespace minifi
-} // namespace nifi
-} // namespace apache
-} // namespace org
-#endif // LIBMINIFI_INCLUDE_REMOTEPROCESSORGROUPPORT_H_
+} // namespace org::apache::nifi::minifi
diff --git a/libminifi/include/core/controller/StandardControllerServiceNode.h b/libminifi/include/core/controller/StandardControllerServiceNode.h
index 8a710d6e8..90c693a65 100644
--- a/libminifi/include/core/controller/StandardControllerServiceNode.h
+++ b/libminifi/include/core/controller/StandardControllerServiceNode.h
@@ -15,8 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-#ifndef LIBMINIFI_INCLUDE_CORE_CONTROLLER_STANDARDCONTROLLERSERVICENODE_H_
-#define LIBMINIFI_INCLUDE_CORE_CONTROLLER_STANDARDCONTROLLERSERVICENODE_H_
+#pragma once
#include <memory>
#include <string>
@@ -26,12 +25,7 @@
#include "core/logging/LoggerConfiguration.h"
#include "core/ProcessGroup.h"
-namespace org {
-namespace apache {
-namespace nifi {
-namespace minifi {
-namespace core {
-namespace controller {
+namespace org::apache::nifi::minifi::core::controller {
class StandardControllerServiceNode : public ControllerServiceNode {
public:
@@ -90,11 +84,4 @@ class StandardControllerServiceNode : public ControllerServiceNode {
std::shared_ptr<logging::Logger> logger_;
};
-} // namespace controller
-} // namespace core
-} // namespace minifi
-} // namespace nifi
-} // namespace apache
-} // namespace org
-
-#endif // LIBMINIFI_INCLUDE_CORE_CONTROLLER_STANDARDCONTROLLERSERVICENODE_H_
+} // namespace org::apache::nifi::minifi::core::controller
diff --git a/libminifi/include/core/reporting/SiteToSiteProvenanceReportingTask.h b/libminifi/include/core/reporting/SiteToSiteProvenanceReportingTask.h
index 6ac010a2f..e9a1083da 100644
--- a/libminifi/include/core/reporting/SiteToSiteProvenanceReportingTask.h
+++ b/libminifi/include/core/reporting/SiteToSiteProvenanceReportingTask.h
@@ -17,8 +17,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-#ifndef LIBMINIFI_INCLUDE_CORE_REPORTING_SITETOSITEPROVENANCEREPORTINGTASK_H_
-#define LIBMINIFI_INCLUDE_CORE_REPORTING_SITETOSITEPROVENANCEREPORTINGTASK_H_
+#pragma once
#include <mutex>
#include <memory>
@@ -30,12 +29,7 @@
#include "io/StreamFactory.h"
#include "core/logging/LoggerConfiguration.h"
-namespace org {
-namespace apache {
-namespace nifi {
-namespace minifi {
-namespace core {
-namespace reporting {
+namespace org::apache::nifi::minifi::core::reporting {
//! SiteToSiteProvenanceReportingTask Class
class SiteToSiteProvenanceReportingTask : public minifi::RemoteProcessorGroupPort {
@@ -93,11 +87,4 @@ class SiteToSiteProvenanceReportingTask : public minifi::RemoteProcessorGroupPor
// SiteToSiteProvenanceReportingTask
-} // namespace reporting
-} // namespace core
-} // namespace minifi
-} // namespace nifi
-} // namespace apache
-} // namespace org
-
-#endif // LIBMINIFI_INCLUDE_CORE_REPORTING_SITETOSITEPROVENANCEREPORTINGTASK_H_
+} // namespace org::apache::nifi::minifi::core::reporting
diff --git a/libminifi/include/core/yaml/YamlConnectionParser.h b/libminifi/include/core/yaml/YamlConnectionParser.h
index fe6f58ed8..6c906eabe 100644
--- a/libminifi/include/core/yaml/YamlConnectionParser.h
+++ b/libminifi/include/core/yaml/YamlConnectionParser.h
@@ -27,12 +27,7 @@
#include "yaml-cpp/yaml.h"
#include "utils/gsl.h"
-namespace org {
-namespace apache {
-namespace nifi {
-namespace minifi {
-namespace core {
-namespace yaml {
+namespace org::apache::nifi::minifi::core::yaml {
class YamlConnectionParser {
public:
@@ -63,9 +58,4 @@ class YamlConnectionParser {
const std::shared_ptr<logging::Logger> logger_;
};
-} // namespace yaml
-} // namespace core
-} // namespace minifi
-} // namespace nifi
-} // namespace apache
-} // namespace org
+} // namespace org::apache::nifi::minifi::core::yaml
diff --git a/libminifi/include/io/tls/TLSServerSocket.h b/libminifi/include/io/tls/TLSServerSocket.h
index 35160c200..af465f99c 100644
--- a/libminifi/include/io/tls/TLSServerSocket.h
+++ b/libminifi/include/io/tls/TLSServerSocket.h
@@ -15,8 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-#ifndef LIBMINIFI_INCLUDE_IO_TLS_TLSSERVERSOCKET_H_
-#define LIBMINIFI_INCLUDE_IO_TLS_TLSSERVERSOCKET_H_
+#pragma once
#include <memory>
#include <string>
@@ -25,11 +24,7 @@
#include "TLSSocket.h"
#include "../ServerSocket.h"
-namespace org {
-namespace apache {
-namespace nifi {
-namespace minifi {
-namespace io {
+namespace org::apache::nifi::minifi::io {
/**
* Purpose: Server socket abstraction that makes focusing the accept/block paradigm
@@ -74,9 +69,4 @@ class TLSServerSocket : public BaseServerSocket, public TLSSocket {
std::shared_ptr<core::logging::Logger> logger_ = core::logging::LoggerFactory<TLSServerSocket>::getLogger();
};
-} // namespace io
-} // namespace minifi
-} // namespace nifi
-} // namespace apache
-} // namespace org
-#endif // LIBMINIFI_INCLUDE_IO_TLS_TLSSERVERSOCKET_H_
+} // namespace org::apache::nifi::minifi::io
diff --git a/libminifi/include/io/tls/TLSSocket.h b/libminifi/include/io/tls/TLSSocket.h
index 5296fb14c..2ed107b7f 100644
--- a/libminifi/include/io/tls/TLSSocket.h
+++ b/libminifi/include/io/tls/TLSSocket.h
@@ -15,8 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-#ifndef LIBMINIFI_INCLUDE_IO_TLS_TLSSOCKET_H_
-#define LIBMINIFI_INCLUDE_IO_TLS_TLSSOCKET_H_
+#pragma once
#include <openssl/err.h>
#include <openssl/ssl.h>
@@ -33,11 +32,7 @@
#include "io/ClientSocket.h"
#include "properties/Configure.h"
-namespace org {
-namespace apache {
-namespace nifi {
-namespace minifi {
-namespace io {
+namespace org::apache::nifi::minifi::io {
#define TLS_GOOD 0
#define TLS_ERROR_CONTEXT 1
@@ -110,9 +105,9 @@ class TLSSocket : public Socket {
/**
* Move constructor.
*/
- TLSSocket(TLSSocket &&);
+ TLSSocket(TLSSocket &&) noexcept;
- TLSSocket& operator=(TLSSocket&&);
+ TLSSocket& operator=(TLSSocket&&) noexcept;
~TLSSocket() override;
@@ -176,10 +171,4 @@ class TLSSocket : public Socket {
std::map<int, SSL*> ssl_map_;
};
-} // namespace io
-} // namespace minifi
-} // namespace nifi
-} // namespace apache
-} // namespace org
-
-#endif // LIBMINIFI_INCLUDE_IO_TLS_TLSSOCKET_H_
+} // namespace org::apache::nifi::minifi::io
diff --git a/libminifi/include/utils/ClassUtils.h b/libminifi/include/utils/ClassUtils.h
index 3d82c4322..c24d87757 100644
--- a/libminifi/include/utils/ClassUtils.h
+++ b/libminifi/include/utils/ClassUtils.h
@@ -14,17 +14,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-#ifndef LIBMINIFI_INCLUDE_UTILS_CLASSUTILS_H_
-#define LIBMINIFI_INCLUDE_UTILS_CLASSUTILS_H_
+#pragma once
#include <string>
-namespace org {
-namespace apache {
-namespace nifi {
-namespace minifi {
-namespace utils {
-namespace ClassUtils {
+namespace org::apache::nifi::minifi::utils::ClassUtils {
/**
* Shortens class names via the canonical representation ( package with name )
@@ -34,11 +28,4 @@ namespace ClassUtils {
*/
bool shortenClassName(const std::string &class_name, std::string &out);
-} /* namespace ClassUtils */
-} /* namespace utils */
-} /* namespace minifi */
-} /* namespace nifi */
-} /* namespace apache */
-} /* namespace org */
-
-#endif /* LIBMINIFI_INCLUDE_UTILS_CLASSUTILS_H_ */
+} // namespace org::apache::nifi::minifi::utils::ClassUtils
diff --git a/libminifi/include/utils/RegexUtils.h b/libminifi/include/utils/RegexUtils.h
index 410af0670..d09c4e793 100644
--- a/libminifi/include/utils/RegexUtils.h
+++ b/libminifi/include/utils/RegexUtils.h
@@ -129,8 +129,8 @@ class Regex {
Regex(std::string value, const std::vector<Mode> &mode);
Regex(const Regex &);
Regex& operator=(const Regex &);
- Regex(Regex&& other);
- Regex& operator=(Regex&& other);
+ Regex(Regex&& other) noexcept;
+ Regex& operator=(Regex&& other) noexcept;
#ifndef NO_MORE_REGFREEE
~Regex();
#endif
diff --git a/libminifi/include/utils/StringUtils.h b/libminifi/include/utils/StringUtils.h
index b48e8eef6..942f0e13a 100644
--- a/libminifi/include/utils/StringUtils.h
+++ b/libminifi/include/utils/StringUtils.h
@@ -161,7 +161,7 @@ class StringUtils {
* @param output output float
* @param cp failure policy
*/
- static bool StringToFloat(std::string input, float &output, FailurePolicy cp = RETURN);
+ static bool StringToFloat(const std::string& input, float &output, FailurePolicy cp = RETURN);
static std::string replaceEnvironmentVariables(std::string source_string);
diff --git a/libminifi/src/RemoteProcessorGroupPort.cpp b/libminifi/src/RemoteProcessorGroupPort.cpp
index 7160f7b21..55df0cae9 100644
--- a/libminifi/src/RemoteProcessorGroupPort.cpp
+++ b/libminifi/src/RemoteProcessorGroupPort.cpp
@@ -258,7 +258,7 @@ std::pair<std::string, int> RemoteProcessorGroupPort::refreshRemoteSite2SiteInfo
if (nifi_instances_.empty())
return std::make_pair("", -1);
- for (auto nifi : nifi_instances_) {
+ for (const auto& nifi : nifi_instances_) {
std::string host = nifi.host_;
#ifdef WIN32
if ("localhost" == host) {
diff --git a/libminifi/src/c2/C2Agent.cpp b/libminifi/src/c2/C2Agent.cpp
index b22dce01d..5785be368 100644
--- a/libminifi/src/c2/C2Agent.cpp
+++ b/libminifi/src/c2/C2Agent.cpp
@@ -122,7 +122,7 @@ void C2Agent::checkTriggers() {
// handle the response the same way. This means that
// acknowledgements will be sent to the c2 server for every trigger action.
// this is expected
- extractPayload(std::move(triggerAction));
+ extractPayload(triggerAction);
// call reset if the trigger supports this activity
trigger->reset();
} else {
@@ -346,7 +346,7 @@ void C2Agent::handle_c2_server_response(const C2ContentResponse &resp) {
case Operation::RESTART: {
update_sink_->stop();
C2Payload response(Operation::ACKNOWLEDGE, resp.ident, true);
- protocol_.load()->consumePayload(std::move(response));
+ protocol_.load()->consumePayload(response);
restart_needed_ = true;
}
break;
@@ -409,7 +409,7 @@ C2Payload C2Agent::prepareConfigurationOptions(const C2ContentResponse &resp) co
auto unsanitized_keys = configuration_->getConfiguredKeys();
std::vector<std::string> keys;
std::copy_if(unsanitized_keys.begin(), unsanitized_keys.end(), std::back_inserter(keys),
- [](std::string key) {return key.find("pass") == std::string::npos;});
+ [](const std::string& key) {return key.find("pass") == std::string::npos;});
C2Payload response(Operation::ACKNOWLEDGE, resp.ident, true);
C2Payload options(Operation::ACKNOWLEDGE);
@@ -625,13 +625,13 @@ void C2Agent::handlePropertyUpdate(const C2ContentResponse &resp) {
}
};
- for (auto entry : resp.operation_arguments) {
+ for (const auto& [name, value] : resp.operation_arguments) {
bool persist = (
- entry.second.getAnnotation("persist")
+ value.getAnnotation("persist")
| utils::map(&AnnotatedValue::to_string)
| utils::flatMap(utils::StringUtils::toBool)).value_or(true);
PropertyChangeLifetime lifetime = persist ? PropertyChangeLifetime::PERSISTENT : PropertyChangeLifetime::TRANSIENT;
- changeUpdateState(update_property(entry.first, entry.second.to_string(), lifetime));
+ changeUpdateState(update_property(name, value.to_string(), lifetime));
}
// apply changes and persist properties requested to be persisted
const bool propertyWasUpdated = result == state::UpdateState::FULLY_APPLIED || result == state::UpdateState::PARTIALLY_APPLIED;
@@ -739,7 +739,7 @@ utils::TaskRescheduleInfo C2Agent::produce() {
std::make_move_iterator(payload_batch.end()),
[&] (C2Payload&& payload) {
try {
- C2Payload && response = protocol_.load()->consumePayload(std::move(payload));
+ C2Payload response = protocol_.load()->consumePayload(payload);
enqueue_c2_server_response(std::move(response));
}
catch(const std::exception &e) {
@@ -775,7 +775,7 @@ utils::TaskRescheduleInfo C2Agent::produce() {
utils::TaskRescheduleInfo C2Agent::consume() {
if (!responses.empty()) {
const auto consume_success = responses.consume([this] (C2Payload&& payload) {
- extractPayload(std::move(payload));
+ extractPayload(payload);
});
if (!consume_success) {
extractPayload(C2Payload{ Operation::HEARTBEAT });
diff --git a/libminifi/src/c2/C2Client.cpp b/libminifi/src/c2/C2Client.cpp
index c7adee079..139661909 100644
--- a/libminifi/src/c2/C2Client.cpp
+++ b/libminifi/src/c2/C2Client.cpp
@@ -33,6 +33,7 @@
#include "utils/file/FileSystem.h"
#include "utils/file/FileUtils.h"
#include "utils/gsl.h"
+#include "utils/StringUtils.h"
namespace org::apache::nifi::minifi::c2 {
@@ -123,7 +124,7 @@ void C2Client::loadC2ResponseConfiguration(const std::string &prefix) {
for (const std::string& metricsClass : classes) {
try {
- std::string option = prefix + "." + metricsClass;
+ std::string option = utils::StringUtils::join_pack(prefix, ".", metricsClass);
std::string classOption = option + ".classes";
std::string nameOption = option + ".name";
@@ -135,7 +136,7 @@ void C2Client::loadC2ResponseConfiguration(const std::string &prefix) {
if (configuration_->get(classOption, class_definitions)) {
loadNodeClasses(class_definitions, new_node);
} else {
- std::string optionName = option + "." + name;
+ std::string optionName = utils::StringUtils::join_pack(option, ".", name);
loadC2ResponseConfiguration(optionName, new_node);
}
@@ -156,7 +157,7 @@ std::shared_ptr<state::response::ResponseNode> C2Client::loadC2ResponseConfigura
for (const std::string& metricsClass : classes) {
try {
- std::string option = prefix + "." + metricsClass;
+ std::string option = utils::StringUtils::join_pack(prefix, ".", metricsClass);
std::string classOption = option + ".classes";
std::string nameOption = option + ".name";
@@ -180,7 +181,7 @@ std::shared_ptr<state::response::ResponseNode> C2Client::loadC2ResponseConfigura
std::static_pointer_cast<state::response::ObjectNode>(prev_node)->add_node(new_node);
}
} else {
- std::string optionName = option + "." + name;
+ std::string optionName = utils::StringUtils::join_pack(option, ".", name);
auto sub_node = loadC2ResponseConfiguration(optionName, new_node);
std::static_pointer_cast<state::response::ObjectNode>(prev_node)->add_node(sub_node);
}
diff --git a/libminifi/src/core/controller/StandardControllerServiceNode.cpp b/libminifi/src/core/controller/StandardControllerServiceNode.cpp
index 955524512..fd6fa325b 100644
--- a/libminifi/src/core/controller/StandardControllerServiceNode.cpp
+++ b/libminifi/src/core/controller/StandardControllerServiceNode.cpp
@@ -20,12 +20,7 @@
#include <memory>
#include <mutex>
-namespace org {
-namespace apache {
-namespace nifi {
-namespace minifi {
-namespace core {
-namespace controller {
+namespace org::apache::nifi::minifi::core::controller {
bool StandardControllerServiceNode::enable() {
Property property("Linked Services", "Referenced Controller Services");
@@ -33,7 +28,7 @@ bool StandardControllerServiceNode::enable() {
logger_->log_trace("Enabling CSN %s", getName());
if (getProperty(property.getName(), property)) {
active = true;
- for (auto linked_service : property.getValues()) {
+ for (const auto& linked_service : property.getValues()) {
std::shared_ptr<ControllerServiceNode> csNode = provider->getControllerServiceNode(linked_service);
if (nullptr != csNode) {
std::lock_guard<std::mutex> lock(mutex_);
@@ -46,7 +41,7 @@ bool StandardControllerServiceNode::enable() {
if (nullptr != impl) {
std::lock_guard<std::mutex> lock(mutex_);
std::vector<std::shared_ptr<ControllerService> > services;
- for (auto service : linked_controller_services_) {
+ for (const auto& service : linked_controller_services_) {
services.push_back(service->getControllerServiceImplementation());
}
impl->setLinkedControllerServices(services);
@@ -55,9 +50,4 @@ bool StandardControllerServiceNode::enable() {
return true;
}
-} /* namespace controller */
-} /* namespace core */
-} /* namespace minifi */
-} /* namespace nifi */
-} /* namespace apache */
-} /* namespace org */
+} // namespace org::apache::nifi::minifi::core::controller
diff --git a/libminifi/src/core/reporting/SiteToSiteProvenanceReportingTask.cpp b/libminifi/src/core/reporting/SiteToSiteProvenanceReportingTask.cpp
index 85d8af0d7..950977728 100644
--- a/libminifi/src/core/reporting/SiteToSiteProvenanceReportingTask.cpp
+++ b/libminifi/src/core/reporting/SiteToSiteProvenanceReportingTask.cpp
@@ -46,12 +46,7 @@
#include "FlowController.h"
#include "utils/gsl.h"
-namespace org {
-namespace apache {
-namespace nifi {
-namespace minifi {
-namespace core {
-namespace reporting {
+namespace org::apache::nifi::minifi::core::reporting {
const char *SiteToSiteProvenanceReportingTask::ProvenanceAppStr = "MiNiFi Flow";
@@ -102,7 +97,7 @@ void SiteToSiteProvenanceReportingTask::getJsonReport(const std::shared_ptr<core
rapidjson::Document array(rapidjson::kArrayType);
rapidjson::Document::AllocatorType &alloc = array.GetAllocator();
- for (auto sercomp : records) {
+ for (const auto& sercomp : records) {
std::shared_ptr<provenance::ProvenanceEventRecord> record = std::dynamic_pointer_cast<provenance::ProvenanceEventRecord>(sercomp);
if (nullptr == record) {
break;
@@ -131,7 +126,7 @@ void SiteToSiteProvenanceReportingTask::getJsonReport(const std::shared_ptr<core
recordJson.AddMember("remoteIdentifier", getStringValue(record->getSourceSystemFlowFileIdentifier(), alloc), alloc);
recordJson.AddMember("alternateIdentifier", getStringValue(record->getAlternateIdentifierUri(), alloc), alloc);
- for (auto attr : record->getAttributes()) {
+ for (const auto& attr : record->getAttributes()) {
setJsonStr(attr.first, attr.second, updatedAttributesJson, alloc);
}
recordJson.AddMember("updatedAttributes", updatedAttributesJson, alloc);
@@ -202,9 +197,4 @@ void SiteToSiteProvenanceReportingTask::onTrigger(const std::shared_ptr<core::Pr
returnProtocol(std::move(protocol_));
}
-} /* namespace reporting */
-} /* namespace core */
-} /* namespace minifi */
-} /* namespace nifi */
-} /* namespace apache */
-} /* namespace org */
+} // namespace org::apache::nifi::minifi::core::reporting
diff --git a/libminifi/src/core/yaml/YamlConnectionParser.cpp b/libminifi/src/core/yaml/YamlConnectionParser.cpp
index b19e90eb0..5e246d8bc 100644
--- a/libminifi/src/core/yaml/YamlConnectionParser.cpp
+++ b/libminifi/src/core/yaml/YamlConnectionParser.cpp
@@ -24,7 +24,7 @@ namespace org::apache::nifi::minifi::core::yaml {
void YamlConnectionParser::addNewRelationshipToConnection(const std::string& relationship_name, minifi::Connection& connection) const {
core::Relationship relationship(relationship_name, "");
logger_->log_debug("parseConnection: relationship => [%s]", relationship_name);
- connection.addRelationship(std::move(relationship));
+ connection.addRelationship(relationship);
}
void YamlConnectionParser::addFunnelRelationshipToConnection(minifi::Connection& connection) const {
diff --git a/libminifi/src/io/tls/TLSServerSocket.cpp b/libminifi/src/io/tls/TLSServerSocket.cpp
index 3a67c4b2f..b3a473a6b 100644
--- a/libminifi/src/io/tls/TLSServerSocket.cpp
+++ b/libminifi/src/io/tls/TLSServerSocket.cpp
@@ -48,11 +48,7 @@
#include "utils/GeneralUtils.h"
#include "utils/gsl.h"
-namespace org {
-namespace apache {
-namespace nifi {
-namespace minifi {
-namespace io {
+namespace org::apache::nifi::minifi::io {
TLSServerSocket::TLSServerSocket(const std::shared_ptr<TLSContext> &context, const std::string &hostname, const uint16_t port, const uint16_t listeners = -1)
: TLSSocket(context, hostname, port, listeners),
@@ -70,7 +66,7 @@ TLSServerSocket::~TLSServerSocket() {
* @return result of the creation operation.
*/
void TLSServerSocket::registerCallback(std::function<bool()> accept_function, std::function<void(io::BaseStream *)> handler) {
- auto fx = [this](std::function<bool()> /*accept_function*/, std::function<void(io::BaseStream *)> handler) {
+ auto fx = [this](const std::function<bool()>& /*accept_function*/, const std::function<void(io::BaseStream *)>& handler) {
while (running_) {
int fd = select_descriptor(1000);
if (fd >= 0) {
@@ -145,8 +141,4 @@ void TLSServerSocket::close_fd(int fd) {
close_ssl(fd);
}
-} // namespace io
-} // namespace minifi
-} // namespace nifi
-} // namespace apache
-} // namespace org
+} // namespace org::apache::nifi::minifi::io
diff --git a/libminifi/src/io/tls/TLSSocket.cpp b/libminifi/src/io/tls/TLSSocket.cpp
index fa4585197..98de6c8d8 100644
--- a/libminifi/src/io/tls/TLSSocket.cpp
+++ b/libminifi/src/io/tls/TLSSocket.cpp
@@ -35,11 +35,7 @@
#include "utils/gsl.h"
#include "utils/tls/TLSUtils.h"
-namespace org {
-namespace apache {
-namespace nifi {
-namespace minifi {
-namespace io {
+namespace org::apache::nifi::minifi::io {
TLSContext::TLSContext(const std::shared_ptr<Configure> &configure, std::shared_ptr<minifi::controllers::SSLContextService> ssl_service)
: SocketContext(configure),
@@ -130,7 +126,7 @@ int16_t TLSContext::initialize(bool server_method) {
}
// load CA certificates
if (ssl_service_ != nullptr || configure_->get(Configure::nifi_security_client_ca_certificate, caCertificate)) {
- retp = SSL_CTX_load_verify_locations(local_context.get(), caCertificate.c_str(), 0);
+ retp = SSL_CTX_load_verify_locations(local_context.get(), caCertificate.c_str(), nullptr);
if (retp == 0) {
logger_->log_error("Can not load CA certificate, Exiting, error : %s", std::strerror(errno));
error_value = TLS_ERROR_CERT_ERROR;
@@ -150,7 +146,7 @@ TLSSocket::~TLSSocket() {
}
void TLSSocket::close() {
- if (ssl_ != 0) {
+ if (ssl_ != nullptr) {
SSL_free(ssl_);
ssl_ = nullptr;
}
@@ -176,7 +172,7 @@ TLSSocket::TLSSocket(const std::shared_ptr<TLSContext> &context, const std::stri
context_ = context;
}
-TLSSocket::TLSSocket(TLSSocket &&other)
+TLSSocket::TLSSocket(TLSSocket &&other) noexcept
: Socket(std::move(other)),
context_{ std::exchange(other.context_, nullptr) } {
std::lock_guard<std::mutex> lg{ other.ssl_mutex_ }; // NOLINT(bugprone-use-after-move)
@@ -187,7 +183,7 @@ TLSSocket::TLSSocket(TLSSocket &&other)
ssl_map_ = std::exchange(other.ssl_map_, {}); // NOLINT(bugprone-use-after-move)
}
-TLSSocket& TLSSocket::operator=(TLSSocket&& other) {
+TLSSocket& TLSSocket::operator=(TLSSocket&& other) noexcept {
if (&other == this) return *this;
this->Socket::operator=(static_cast<Socket&&>(other));
std::lock_guard<std::mutex> lg{ other.ssl_mutex_ };
@@ -276,9 +272,9 @@ int16_t TLSSocket::select_descriptor(const uint16_t msec) {
std::lock_guard<std::recursive_mutex> guard(selection_mutex_);
if (msec > 0)
- select(socket_max_ + 1, &read_fds_, NULL, NULL, &tv);
+ select(socket_max_ + 1, &read_fds_, nullptr, nullptr, &tv);
else
- select(socket_max_ + 1, &read_fds_, NULL, NULL, NULL);
+ select(socket_max_ + 1, &read_fds_, nullptr, nullptr, nullptr);
for (int i = 0; i <= socket_max_; i++) {
if (!FD_ISSET(i, &read_fds_)) continue;
@@ -451,8 +447,4 @@ size_t TLSSocket::read(gsl::span<std::byte> buffer) {
return total_read;
}
-} /* namespace io */
-} /* namespace minifi */
-} /* namespace nifi */
-} /* namespace apache */
-} /* namespace org */
+} // namespace org::apache::nifi::minifi::io
diff --git a/libminifi/src/utils/ClassUtils.cpp b/libminifi/src/utils/ClassUtils.cpp
index 52914fc49..ecd557054 100644
--- a/libminifi/src/utils/ClassUtils.cpp
+++ b/libminifi/src/utils/ClassUtils.cpp
@@ -23,18 +23,14 @@
#include "utils/StringUtils.h"
-namespace org {
-namespace apache {
-namespace nifi {
-namespace minifi {
-namespace utils {
+namespace org::apache::nifi::minifi::utils {
bool ClassUtils::shortenClassName(const std::string &class_name, std::string &out) {
std::string class_delim = "::";
auto class_split = utils::StringUtils::split(class_name, class_delim);
// support . and ::
if (class_split.size() <= 1) {
- if (class_name.find(".") != std::string::npos) {
+ if (class_name.find('.') != std::string::npos) {
class_delim = ".";
class_split = utils::StringUtils::split(class_name, class_delim);
} else {
@@ -53,9 +49,4 @@ bool ClassUtils::shortenClassName(const std::string &class_name, std::string &ou
return true;
}
-} // namespace utils
-} // namespace minifi
-} // namespace nifi
-} // namespace apache
-} // namespace org
-
+} // namespace org::apache::nifi::minifi::utils
diff --git a/libminifi/src/utils/RegexUtils.cpp b/libminifi/src/utils/RegexUtils.cpp
index d87f11c11..8374fd230 100644
--- a/libminifi/src/utils/RegexUtils.cpp
+++ b/libminifi/src/utils/RegexUtils.cpp
@@ -145,7 +145,7 @@ Regex& Regex::operator=(const Regex& other) {
return *this;
}
-Regex::Regex(Regex&& other)
+Regex::Regex(Regex&& other) noexcept
#ifndef NO_MORE_REGFREEE
: valid_(false),
regex_mode_(REG_EXTENDED)
@@ -154,7 +154,7 @@ Regex::Regex(Regex&& other)
*this = std::move(other);
}
-Regex& Regex::operator=(Regex&& other) {
+Regex& Regex::operator=(Regex&& other) noexcept {
if (this == &other) {
return *this;
}
diff --git a/libminifi/src/utils/StringUtils.cpp b/libminifi/src/utils/StringUtils.cpp
index c7d55b11b..f3a59a081 100644
--- a/libminifi/src/utils/StringUtils.cpp
+++ b/libminifi/src/utils/StringUtils.cpp
@@ -116,7 +116,7 @@ std::vector<std::string> StringUtils::splitAndTrimRemovingEmpty(const std::strin
return result;
}
-bool StringUtils::StringToFloat(std::string input, float &output, FailurePolicy cp /*= RETURN*/) {
+bool StringUtils::StringToFloat(const std::string& input, float &output, FailurePolicy cp /*= RETURN*/) {
try {
output = std::stof(input);
} catch (const std::invalid_argument &ie) {
@@ -156,7 +156,7 @@ std::string StringUtils::replaceEnvironmentVariables(std::string source_string)
beg_seq += 2;
continue;
}
- end_seq = source_string.find("}", beg_seq + 2);
+ end_seq = source_string.find('}', beg_seq + 2);
if (end_seq == std::string::npos) {
break;
}
@@ -215,7 +215,7 @@ std::string StringUtils::replaceMap(std::string source_string, const std::map<st
}
}
- std::sort(replacements.begin(), replacements.end(), [](const std::pair<size_t, std::pair<size_t, std::string>> a,
+ std::sort(replacements.begin(), replacements.end(), [](const std::pair<size_t, std::pair<size_t, std::string>> &a,
const std::pair<size_t, std::pair<size_t, std::string>> &b) {
return a.first > b.first;
});
diff --git a/libminifi/test/archive-tests/ManipulateArchiveTests.cpp b/libminifi/test/archive-tests/ManipulateArchiveTests.cpp
index 65965d16d..b3e2f5d59 100644
--- a/libminifi/test/archive-tests/ManipulateArchiveTests.cpp
+++ b/libminifi/test/archive-tests/ManipulateArchiveTests.cpp
@@ -45,7 +45,7 @@ const char* MODIFY_DEST = "modified";
using PROP_MAP_T = std::map<std::string, std::string>;
-bool run_archive_test(OrderedTestArchive input_archive, OrderedTestArchive output_archive, PROP_MAP_T properties, bool check_attributes = true) {
+bool run_archive_test(OrderedTestArchive& input_archive, const OrderedTestArchive& output_archive, const PROP_MAP_T& properties, bool check_attributes = true) {
TestController testController;
LogTestController::getInstance().setTrace<org::apache::nifi::minifi::processors::FocusArchiveEntry>();
LogTestController::getInstance().setTrace<org::apache::nifi::minifi::processors::UnfocusArchiveEntry>();
@@ -74,8 +74,8 @@ bool run_archive_test(OrderedTestArchive input_archive, OrderedTestArchive outpu
std::shared_ptr<core::Processor> maprocessor = plan->addProcessor("ManipulateArchive", "testManipulateArchive", core::Relationship("success", "description"), true);
- for (auto kv : properties) {
- plan->setProperty(maprocessor, kv.first, kv.second);
+ for (const auto& [name, value] : properties) {
+ plan->setProperty(maprocessor, name, value);
}
std::shared_ptr<core::Processor> putfile2 = plan->addProcessor("PutFile", "PutFile2", core::Relationship("success", "description"), true);
@@ -96,14 +96,14 @@ bool run_archive_test(OrderedTestArchive input_archive, OrderedTestArchive outpu
return check_archive_contents(output_path, output_archive, check_attributes);
}
-bool run_archive_test(TAE_MAP_T input_map, TAE_MAP_T output_map, PROP_MAP_T properties, bool check_attributes = true) {
+bool run_archive_test(TAE_MAP_T input_map, TAE_MAP_T output_map, const PROP_MAP_T& properties, bool check_attributes = true) {
OrderedTestArchive input_archive;
OrderedTestArchive output_archive;
// An empty vector is treated as "ignore order"
input_archive.order = output_archive.order = FN_VEC_T();
- input_archive.map = input_map;
- output_archive.map = output_map;
+ input_archive.map = std::move(input_map);
+ output_archive.map = std::move(output_map);
return run_archive_test(input_archive, output_archive, properties, check_attributes);
}
diff --git a/libminifi/test/archive-tests/MergeFileTests.cpp b/libminifi/test/archive-tests/MergeFileTests.cpp
index 3318085e5..dc5e3ba64 100644
--- a/libminifi/test/archive-tests/MergeFileTests.cpp
+++ b/libminifi/test/archive-tests/MergeFileTests.cpp
@@ -70,7 +70,7 @@ class FixedBuffer {
explicit FixedBuffer(std::size_t capacity) : capacity_(capacity) {
buf_.reset(new uint8_t[capacity_]);
}
- FixedBuffer(FixedBuffer&& other) : buf_(std::move(other.buf_)), size_(other.size_), capacity_(other.capacity_) {
+ FixedBuffer(FixedBuffer&& other) noexcept : buf_(std::move(other.buf_)), size_(other.size_), capacity_(other.capacity_) {
other.size_ = 0;
other.capacity_ = 0;
}
diff --git a/libminifi/test/archive-tests/util/ArchiveTests.cpp b/libminifi/test/archive-tests/util/ArchiveTests.cpp
index fac83b0b1..ab678dd87 100644
--- a/libminifi/test/archive-tests/util/ArchiveTests.cpp
+++ b/libminifi/test/archive-tests/util/ArchiveTests.cpp
@@ -68,7 +68,7 @@ OrderedTestArchive build_ordered_test_archive(int NUM_FILES, const char** FILE_N
return ret;
}
-void build_test_archive(std::string path, TAE_MAP_T entries, FN_VEC_T order) {
+void build_test_archive(const std::string& path, const TAE_MAP_T& entries, FN_VEC_T order) {
std::cout << "Creating " << path << std::endl;
archive * test_archive = archive_write_new();
@@ -81,7 +81,7 @@ void build_test_archive(std::string path, TAE_MAP_T entries, FN_VEC_T order) {
order.push_back(kvp.first);
}
- for (std::string name : order) {
+ for (const std::string& name : order) {
TestArchiveEntry test_entry = entries.at(name);
std::cout << "Adding entry: " << name << std::endl;
@@ -104,11 +104,11 @@ void build_test_archive(std::string path, TAE_MAP_T entries, FN_VEC_T order) {
archive_write_close(test_archive);
}
-void build_test_archive(std::string path, OrderedTestArchive ordered_archive) {
+void build_test_archive(const std::string& path, OrderedTestArchive& ordered_archive) {
build_test_archive(path, ordered_archive.map, ordered_archive.order);
}
-bool check_archive_contents(std::string path, TAE_MAP_T entries, bool check_attributes, FN_VEC_T order) {
+bool check_archive_contents(const std::string& path, const TAE_MAP_T& entries, bool check_attributes, const FN_VEC_T& order) {
FN_VEC_T read_names;
FN_VEC_T extra_names;
bool ok = true;
@@ -133,7 +133,7 @@ bool check_archive_contents(std::string path, TAE_MAP_T entries, bool check_attr
} else {
read_names.push_back(name);
TestArchiveEntry test_entry = it->second;
- size_t size = gsl::narrow<size_t>(archive_entry_size(entry));
+ auto size = gsl::narrow<size_t>(archive_entry_size(entry));
std::cout << "Checking archive entry: " << name << std::endl;
@@ -179,7 +179,7 @@ bool check_archive_contents(std::string path, TAE_MAP_T entries, bool check_attr
if (!extra_names.empty()) {
ok = false;
std::cout << "Extra files found: ";
- for (std::string filename : extra_names)
+ for (const std::string& filename : extra_names)
std::cout << filename << " ";
std::cout << std::endl;
}
@@ -195,7 +195,7 @@ bool check_archive_contents(std::string path, TAE_MAP_T entries, bool check_attr
} else {
std::set<std::string> read_names_set(read_names.begin(), read_names.end());
std::set<std::string> test_file_entries_set;
- std::transform(entries.begin(), entries.end(), std::inserter(test_file_entries_set, test_file_entries_set.end()), [](std::pair<std::string, TestArchiveEntry> p) {return p.first;});
+ std::transform(entries.begin(), entries.end(), std::inserter(test_file_entries_set, test_file_entries_set.end()), [](const std::pair<std::string, TestArchiveEntry>& p) {return p.first;});
REQUIRE(read_names_set == test_file_entries_set);
}
@@ -203,6 +203,6 @@ bool check_archive_contents(std::string path, TAE_MAP_T entries, bool check_attr
return ok;
}
-bool check_archive_contents(std::string path, OrderedTestArchive archive, bool check_attributes) {
+bool check_archive_contents(const std::string& path, const OrderedTestArchive& archive, bool check_attributes) {
return check_archive_contents(path, archive.map, check_attributes, archive.order);
}
diff --git a/libminifi/test/rocksdb-tests/RocksDBTests.cpp b/libminifi/test/rocksdb-tests/RocksDBTests.cpp
index 6420a04de..6e6e93514 100644
--- a/libminifi/test/rocksdb-tests/RocksDBTests.cpp
+++ b/libminifi/test/rocksdb-tests/RocksDBTests.cpp
@@ -48,6 +48,7 @@ struct RocksDBTest : TestController {
rocksdb::DB* db_ptr = nullptr;
std::vector<rocksdb::ColumnFamilyHandle*> cf_handle_ptrs;
std::vector<rocksdb::ColumnFamilyDescriptor> cf_descs;
+ cf_descs.reserve(cf_names.size());
for (auto& cf_name : cf_names) {
cf_descs.emplace_back(cf_name, rocksdb::ColumnFamilyOptions{});
}
diff --git a/libminifi/test/sql-tests/mocks/MockConnectors.cpp b/libminifi/test/sql-tests/mocks/MockConnectors.cpp
index 1e4d7570c..6854a97f3 100644
--- a/libminifi/test/sql-tests/mocks/MockConnectors.cpp
+++ b/libminifi/test/sql-tests/mocks/MockConnectors.cpp
@@ -146,6 +146,7 @@ std::unique_ptr<MockRowset> MockRowset::select(const std::vector<std::string>& c
rowset = std::make_unique<MockRowset>(column_names_, column_types_);
} else {
std::vector<DataType> col_types;
+ col_types.reserve(cols.size());
for (const auto& col : cols) {
col_types.push_back(column_types_.at(getColumnIndex(col)));
}
@@ -156,6 +157,7 @@ std::unique_ptr<MockRowset> MockRowset::select(const std::vector<std::string>& c
for (const auto& row : rows_) {
if (condition(row)) {
std::vector<std::string> values;
+ values.reserve(used_cols.size());
for (const auto& col : used_cols) {
values.push_back(row.getValue(col));
}
diff --git a/libminifi/test/unit/BackTraceTests.cpp b/libminifi/test/unit/BackTraceTests.cpp
index 27fbc5401..4366f7a38 100644
--- a/libminifi/test/unit/BackTraceTests.cpp
+++ b/libminifi/test/unit/BackTraceTests.cpp
@@ -35,9 +35,9 @@ class WorkerNumberExecutions : public utils::AfterExecute<int> {
: tasks(tasks) {
}
- explicit WorkerNumberExecutions(WorkerNumberExecutions && other)
- : runs(std::move(other.runs)),
- tasks(std::move(other.tasks)) {
+ explicit WorkerNumberExecutions(WorkerNumberExecutions && other) noexcept
+ : runs(other.runs),
+ tasks(other.tasks) {
}
bool isFinished(const int &result) override {
diff --git a/libminifi/test/unit/ExpectedTest.cpp b/libminifi/test/unit/ExpectedTest.cpp
index 0bff815e4..26ac4dbe7 100644
--- a/libminifi/test/unit/ExpectedTest.cpp
+++ b/libminifi/test/unit/ExpectedTest.cpp
@@ -52,7 +52,7 @@ TEST_CASE("expected map", "[expected][map]") {
{
const nonstd::expected<int, int> e = 21;
- auto ret = std::move(e) | utils::map(mul2);
+ auto ret = std::move(e) | utils::map(mul2); // NOLINT(performance-move-const-arg)
REQUIRE(ret);
REQUIRE(*ret == 42);
}
@@ -80,7 +80,7 @@ TEST_CASE("expected map", "[expected][map]") {
{
const nonstd::expected<int, int> e(nonstd::unexpect, 21);
- auto ret = std::move(e) | utils::map(mul2);
+ auto ret = std::move(e) | utils::map(mul2); // NOLINT(performance-move-const-arg)
REQUIRE(!ret);
REQUIRE(ret.error() == 21);
}
@@ -111,7 +111,7 @@ TEST_CASE("expected map", "[expected][map]") {
{
const nonstd::expected<int, int> e = 21;
- auto ret = std::move(e) | utils::map(ret_void);
+ auto ret = std::move(e) | utils::map(ret_void); // NOLINT(performance-move-const-arg)
REQUIRE(ret);
STATIC_REQUIRE(
(std::is_same<decltype(ret), nonstd::expected<void, int>>::value));
@@ -143,7 +143,7 @@ TEST_CASE("expected map", "[expected][map]") {
{
const nonstd::expected<int, int> e(nonstd::unexpect, 21);
- auto ret = std::move(e) | utils::map(ret_void);
+ auto ret = std::move(e) | utils::map(ret_void); // NOLINT(performance-move-const-arg)
REQUIRE(!ret);
STATIC_REQUIRE(
(std::is_same<decltype(ret), nonstd::expected<void, int>>::value));
@@ -186,7 +186,7 @@ TEST_CASE("expected flatMap", "[expected][flatMap]") {
{
const nonstd::expected<int, int> e = 21;
- auto ret = std::move(e) | utils::flatMap(succeed);
+ auto ret = std::move(e) | utils::flatMap(succeed); // NOLINT(performance-move-const-arg)
REQUIRE(ret);
REQUIRE(*ret == 42);
}
@@ -214,7 +214,7 @@ TEST_CASE("expected flatMap", "[expected][flatMap]") {
{
const nonstd::expected<int, int> e = 21;
- auto ret = std::move(e) | utils::flatMap(fail);
+ auto ret = std::move(e) | utils::flatMap(fail); // NOLINT(performance-move-const-arg)
REQUIRE(!ret);
REQUIRE(ret.error() == 17);
}
@@ -242,7 +242,7 @@ TEST_CASE("expected flatMap", "[expected][flatMap]") {
{
const nonstd::expected<int, int> e(nonstd::unexpect, 21);
- auto ret = std::move(e) | utils::flatMap(succeed);
+ auto ret = std::move(e) | utils::flatMap(succeed); // NOLINT(performance-move-const-arg)
REQUIRE(!ret);
REQUIRE(ret.error() == 21);
}
@@ -270,7 +270,7 @@ TEST_CASE("expected flatMap", "[expected][flatMap]") {
{
const nonstd::expected<int, int> e(nonstd::unexpect, 21);
- auto ret = std::move(e) | utils::flatMap(fail);
+ auto ret = std::move(e) | utils::flatMap(fail); // NOLINT(performance-move-const-arg)
REQUIRE(!ret);
REQUIRE(ret.error() == 21);
}
@@ -321,7 +321,7 @@ TEST_CASE("expected orElse", "[expected][orElse]") {
{
const nonstd::expected<int, int> e = 21;
- auto ret = std::move(e) | utils::orElse(succeed);
+ auto ret = std::move(e) | utils::orElse(succeed); // NOLINT(performance-move-const-arg)
REQUIRE(ret);
REQUIRE(*ret == 21);
}
@@ -359,7 +359,7 @@ TEST_CASE("expected orElse", "[expected][orElse]") {
{
const nonstd::expected<int, int> e = 21;
- auto ret = std::move(e) | utils::orElse(fail);
+ auto ret = std::move(e) | utils::orElse(fail); // NOLINT(performance-move-const-arg)
REQUIRE(ret);
REQUIRE(*ret == 21);
}
@@ -396,7 +396,7 @@ TEST_CASE("expected orElse", "[expected][orElse]") {
{
const nonstd::expected<int, int> e(nonstd::unexpect, 21);
- auto ret = std::move(e) | utils::orElse(succeed);
+ auto ret = std::move(e) | utils::orElse(succeed); // NOLINT(performance-move-const-arg)
REQUIRE(ret);
REQUIRE(*ret == 42);
}
@@ -461,14 +461,14 @@ TEST_CASE("expected orElse", "[expected][orElse]") {
{
const nonstd::expected<int, int> e(nonstd::unexpect, 21);
- auto ret = std::move(e) | utils::orElse(fail);
+ auto ret = std::move(e) | utils::orElse(fail); // NOLINT(performance-move-const-arg)
REQUIRE(!ret);
REQUIRE(ret.error() == 17);
}
{
const nonstd::expected<int, int> e(nonstd::unexpect, 21);
- auto ret = std::move(e) | utils::orElse(failvoid);
+ auto ret = std::move(e) | utils::orElse(failvoid); // NOLINT(performance-move-const-arg)
REQUIRE(!ret);
REQUIRE(ret.error() == 21);
}
@@ -480,7 +480,7 @@ TEST_CASE("expected valueOrElse", "[expected][valueOrElse]") {
REQUIRE(42 == (ex | utils::valueOrElse([] { return 42; })));
REQUIRE_THROWS_AS(ex | utils::valueOrElse([]{ throw std::exception(); }), std::exception);
REQUIRE(gsl::narrow<int>("hello"sv.size()) == (ex | utils::valueOrElse([](const std::string& err) { return gsl::narrow<int>(err.size()); })));
- REQUIRE_THROWS_AS(ex | utils::valueOrElse([](std::string){ throw std::exception(); }), std::exception);
+ REQUIRE_THROWS_AS(ex | utils::valueOrElse([](std::string){ throw std::exception(); }), std::exception); // NOLINT(performance-unnecessary-value-param)
REQUIRE_THROWS_AS(ex | utils::valueOrElse([](const std::string&) -> int { throw std::exception(); }), std::exception);
REQUIRE_THROWS_AS(std::move(ex) | utils::valueOrElse([](std::string&&) -> int { throw std::exception(); }), std::exception);
}
diff --git a/libminifi/test/unit/IdTests.cpp b/libminifi/test/unit/IdTests.cpp
index a24b8661d..b993f7bcd 100644
--- a/libminifi/test/unit/IdTests.cpp
+++ b/libminifi/test/unit/IdTests.cpp
@@ -77,7 +77,7 @@ TEST_CASE("Test Generate Move", "[id]") {
auto generated = generator->generate();
auto str = generated.to_string();
- utils::Identifier moved = std::move(generated);
+ utils::Identifier moved = std::move(generated); // NOLINT(performance-move-const-arg)
auto str2 = moved.to_string();
REQUIRE(str == str2);
}
diff --git a/libminifi/test/unit/ThreadPoolTests.cpp b/libminifi/test/unit/ThreadPoolTests.cpp
index eec7c9ced..ae05cd56d 100644
--- a/libminifi/test/unit/ThreadPoolTests.cpp
+++ b/libminifi/test/unit/ThreadPoolTests.cpp
@@ -33,9 +33,9 @@ class WorkerNumberExecutions : public utils::AfterExecute<int> {
: tasks(tasks) {
}
- explicit WorkerNumberExecutions(WorkerNumberExecutions && other)
- : runs(std::move(other.runs)),
- tasks(std::move(other.tasks)) {
+ explicit WorkerNumberExecutions(WorkerNumberExecutions && other) noexcept
+ : runs(other.runs),
+ tasks(other.tasks) {
}
bool isFinished(const int &result) override {
[nifi-minifi-cpp] 02/03: MINIFICPP-1888 Move all extension builds to CentOS job
Posted by sz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
szaszm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit a8a4278363bb9b5536f58688a8484732561b158b
Author: Gabor Gyimesi <ga...@gmail.com>
AuthorDate: Thu Aug 18 18:28:05 2022 +0200
MINIFICPP-1888 Move all extension builds to CentOS job
With more extensions added to be turned on by default, the centos build
targeting the default extensions takes longer to finish. To make sure we
can build on both ubuntu/gcc and CentOS and to minimize the CI resources
used, all extension builds that are available in the CentOS docker build
are moved to the CentOS job and tests are run on CentOS and in the clang
Ubuntu build.
Closes #1374
Signed-off-by: Marton Szasz <sz...@apache.org>
---
.github/workflows/ci.yml | 38 +++++++++++++++-----------
cmake/DockerConfig.cmake | 4 +++
cmake/MiNiFiOptions.cmake | 3 ++-
docker/DockerBuild.sh | 36 ++++++++++++++++++-------
docker/Dockerfile | 3 ++-
docker/centos/Dockerfile | 40 +++++++++++++++++++---------
extensions/sftp/tests/ListSFTPTests.cpp | 4 +--
libminifi/include/core/extension/Utils.h | 7 +++--
libminifi/test/unit/CollectionUtilsTests.cpp | 2 +-
9 files changed, 92 insertions(+), 45 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 188cb7eda..50ca97b76 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -4,7 +4,7 @@ jobs:
macos_xcode:
name: "macos-xcode"
runs-on: macos-11
- timeout-minutes: 90
+ timeout-minutes: 180
env:
CCACHE_BASEDIR: ${{ GITHUB.WORKSPACE }}
CCACHE_DIR: ${{ GITHUB.WORKSPACE }}/.ccache
@@ -107,7 +107,7 @@ jobs:
- id: install_deps
run: |
sudo apt update
- sudo apt install -y ccache libfl-dev libpcap-dev libboost-all-dev openjdk-8-jdk maven libusb-1.0-0-dev libpng-dev libgps-dev libsqliteodbc flake8 lua5.3 liblua5.3-dev
+ sudo apt install -y ccache libfl-dev libboost-all-dev libpcap-dev
echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
echo -e "127.0.0.1\t$HOSTNAME" | sudo tee -a /etc/hosts > /dev/null
- name: build
@@ -116,19 +116,11 @@ jobs:
cd build
export CC=gcc-11
export CXX=g++-11
- cmake -DUSE_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCI_BUILD=ON -DSTRICT_GSL_CHECKS=AUDIT -DFAIL_ON_WARNINGS=ON -DENABLE_AWS=ON -DENABLE_AZURE=ON -DENABLE_BUSTACHE=ON -DENABLE_COAP=ON \
- -DENABLE_ENCRYPT_CONFIG=ON -DENABLE_GPS=ON -DENABLE_JNI=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_LINTER=ON -DENABLE_MQTT=ON -DENABLE_NANOFI=ON -DENABLE_OPC=ON -DENABLE_OPENCV=ON \
- -DENABLE_OPENWSMAN=ON -DENABLE_OPS=ON -DENABLE_PCAP=ON -DENABLE_PYTHON=ON -DENABLE_SENSORS=ON -DENABLE_SFTP=ON -DENABLE_SQL=ON -DENABLE_SYSTEMD=ON -DENABLE_TENSORFLOW=OFF \
- -DENABLE_USB_CAMERA=ON -DENABLE_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_GCP=ON -DENABLE_PROCFS=ON -DENABLE_PROMETHEUS=ON -DENABLE_ELASTICSEARCH=ON ..
+ cmake -DUSE_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCI_BUILD=ON -DSTRICT_GSL_CHECKS=AUDIT -DFAIL_ON_WARNINGS=ON -DENABLE_SQL=OFF -DENABLE_LIBRDKAFKA=OFF -DENABLE_AWS=OFF \
+ -DENABLE_AZURE=OFF -DENABLE_SPLUNK=OFF -DENABLE_GCP=OFF -DENABLE_PROCFS=OFF -DENABLE_BUSTACHE=ON -DENABLE_PCAP=ON ..
make -j$(nproc) VERBOSE=1
- name: test
run: cd build && make test ARGS="--timeout 300 -j2 --output-on-failure"
- - name: linter
- run: cd build && make -j$(nproc) linter
- - name: shellcheck
- run: cd build && make shellcheck
- - id: flake8_check
- run: cd build && make flake8
ubuntu_20_04_clang:
name: "ubuntu-20.04-clang"
runs-on: ubuntu-20.04
@@ -152,7 +144,7 @@ jobs:
echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main" | sudo tee -a /etc/apt/sources.list
echo "deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main" | sudo tee -a /etc/apt/sources.list
sudo apt update
- sudo apt install -y ccache libfl-dev libpcap-dev libboost-all-dev openjdk-8-jdk maven libusb-1.0-0-dev libpng-dev libgps-dev clang-14 clang-tidy-14 libc++-14-dev libc++abi-14-dev libsqliteodbc lua5.3 liblua5.3-dev
+ sudo apt install -y ccache libfl-dev libpcap-dev libboost-all-dev openjdk-8-jdk maven libusb-1.0-0-dev libpng-dev libgps-dev clang-14 clang-tidy-14 libc++-14-dev libc++abi-14-dev libsqliteodbc lua5.3 liblua5.3-dev flake8
echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
echo -e "127.0.0.1\t$HOSTNAME" | sudo tee -a /etc/hosts > /dev/null
- name: build
@@ -171,6 +163,12 @@ jobs:
cmake --build . --parallel $(nproc)
- name: test
run: cd build && make test ARGS="--timeout 300 -j8 --output-on-failure"
+ - name: linter
+ run: cd build && make -j$(nproc) linter
+ - name: shellcheck
+ run: cd build && make shellcheck
+ - id: flake8_check
+ run: cd build && make flake8
- id: files
uses: Ana06/get-changed-files@v2.1.0
- name: clang-tidy
@@ -183,7 +181,7 @@ jobs:
centos:
name: "centos"
runs-on: ubuntu-20.04
- timeout-minutes: 60
+ timeout-minutes: 180
steps:
- id: checkout
uses: actions/checkout@v2
@@ -201,11 +199,19 @@ jobs:
sudo apt install -y ccache
echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
- id: build
- run: mkdir build && cd build && cmake -DSTRICT_GSL_CHECKS=AUDIT -DENABLE_KUBERNETES=ON .. && make centos
+ run: |
+ if [ -d ~/.ccache ]; then mv ~/.ccache .; fi
+ mkdir build && cd build && cmake -DUSE_SHARED_LIBS=ON -DCI_BUILD=ON -DSTRICT_GSL_CHECKS=AUDIT -DFAIL_ON_WARNINGS=ON -DENABLE_AWS=ON -DENABLE_AZURE=ON -DENABLE_COAP=ON \
+ -DENABLE_ENCRYPT_CONFIG=ON -DENABLE_GPS=ON -DENABLE_JNI=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_LINTER=ON -DENABLE_MQTT=ON -DENABLE_NANOFI=ON -DENABLE_OPC=ON -DENABLE_OPENCV=ON \
+ -DENABLE_OPENWSMAN=ON -DENABLE_OPS=ON -DENABLE_PYTHON=ON -DENABLE_SENSORS=ON -DENABLE_SFTP=ON -DENABLE_SQL=ON -DENABLE_SYSTEMD=ON -DENABLE_TENSORFLOW=OFF \
+ -DENABLE_USB_CAMERA=ON -DENABLE_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_GCP=ON -DENABLE_PROCFS=ON -DENABLE_PROMETHEUS=ON -DENABLE_ELASTICSEARCH=ON \
+ -DDOCKER_SKIP_TESTS=OFF -DDOCKER_BUILD_ONLY=ON -DDOCKER_CCACHE_DUMP_LOCATION=$HOME/.ccache .. && make centos
+ - id: test
+ run: docker run --rm apacheminificpp:$(docker images | grep apacheminificpp | grep centos | awk '{print $2}') bash -c 'cd /opt/minifi/build && make test ARGS="--timeout 300 -j8 --output-on-failure"'
docker_integration_tests:
name: "Docker integration tests"
runs-on: ubuntu-20.04
- timeout-minutes: 120
+ timeout-minutes: 180
steps:
- id: checkout
uses: actions/checkout@v2
diff --git a/cmake/DockerConfig.cmake b/cmake/DockerConfig.cmake
index b6c21d85c..04b9bb92d 100644
--- a/cmake/DockerConfig.cmake
+++ b/cmake/DockerConfig.cmake
@@ -25,6 +25,7 @@ add_custom_target(
-o ${MINIFI_DOCKER_OPTIONS_STR}
-c DOCKER_BASE_IMAGE=${DOCKER_BASE_IMAGE}
-c DOCKER_CCACHE_DUMP_LOCATION=${DOCKER_CCACHE_DUMP_LOCATION}
+ -c DOCKER_SKIP_TESTS=${DOCKER_SKIP_TESTS}
-c BUILD_NUMBER=${BUILD_NUMBER}
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/)
@@ -45,6 +46,7 @@ add_custom_target(
-DDISABLE_PYTHON_SCRIPTING=ON
-DENABLE_ENCRYPT_CONFIG=OFF \"
-c DOCKER_BASE_IMAGE=${DOCKER_BASE_IMAGE}
+ -c DOCKER_SKIP_TESTS=${DOCKER_SKIP_TESTS}
-c BUILD_NUMBER=${BUILD_NUMBER}
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/)
@@ -58,6 +60,8 @@ add_custom_target(
-l ${CMAKE_BINARY_DIR}
-d centos
-c BUILD_NUMBER=${BUILD_NUMBER}
+ -c DOCKER_CCACHE_DUMP_LOCATION=${DOCKER_CCACHE_DUMP_LOCATION}
+ -c DOCKER_SKIP_TESTS=${DOCKER_SKIP_TESTS}
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/)
add_custom_target(
diff --git a/cmake/MiNiFiOptions.cmake b/cmake/MiNiFiOptions.cmake
index 9ef856893..a4429e499 100644
--- a/cmake/MiNiFiOptions.cmake
+++ b/cmake/MiNiFiOptions.cmake
@@ -32,6 +32,7 @@ add_minifi_option(CI_BUILD "Build is used for CI." OFF)
add_minifi_option(ENABLE_LINTER "Create linter components" ON)
add_minifi_option(SKIP_TESTS "Skips building all tests." OFF)
add_minifi_option(DOCKER_BUILD_ONLY "Disables all targets except docker build scripts. Ideal for systems without an up-to-date compiler." OFF)
+add_minifi_option(DOCKER_SKIP_TESTS "Skip building tests in docker image targets." ON)
add_minifi_option(PORTABLE "Instructs the compiler to remove architecture specific optimizations" ON)
add_minifi_option(USE_SHARED_LIBS "Builds using shared libraries" ON)
@@ -127,7 +128,7 @@ endif()
function(get_minifi_docker_options RET_VALUE)
set(MINIFI_DOCKER_OPTIONS_STR ${MINIFI_EXTERNAL_DOCKER_OPTIONS_STR})
foreach(MINIFI_OPTION ${MINIFI_OPTIONS})
- if (MINIFI_OPTION STREQUAL "CI_BUILD" OR MINIFI_OPTION STREQUAL "ENABLE_LINTER" OR MINIFI_OPTION STREQUAL "SKIP_TESTS" OR MINIFI_OPTION STREQUAL "DOCKER_BUILD_ONLY")
+ if (MINIFI_OPTION STREQUAL "CI_BUILD" OR MINIFI_OPTION STREQUAL "ENABLE_LINTER" OR MINIFI_OPTION STREQUAL "SKIP_TESTS" OR MINIFI_OPTION STREQUAL "DOCKER_BUILD_ONLY" OR MINIFI_OPTION STREQUAL "DOCKER_SKIP_TESTS")
continue()
endif()
set(MINIFI_DOCKER_OPTIONS_STR "${MINIFI_DOCKER_OPTIONS_STR} -D${MINIFI_OPTION}=${${MINIFI_OPTION}}")
diff --git a/docker/DockerBuild.sh b/docker/DockerBuild.sh
index 82b093917..3f6ed54b3 100755
--- a/docker/DockerBuild.sh
+++ b/docker/DockerBuild.sh
@@ -29,6 +29,7 @@ DUMP_LOCATION=
DISTRO_NAME=
BUILD_NUMBER=
DOCKER_CCACHE_DUMP_LOCATION=
+DOCKER_SKIP_TESTS=ON
function usage {
echo "Usage: ./DockerBuild.sh -v <MINIFI_VERSION> [additional options]"
@@ -45,6 +46,15 @@ function usage {
exit 1
}
+function dump_ccache() {
+ ccache_source_image=$1
+ docker_ccache_dump_location=$2
+ container_id=$(docker run --rm -d "${ccache_source_image}" sh -c "while true; do sleep 1; done")
+ mkdir -p "${docker_ccache_dump_location}"
+ docker cp "${container_id}:/home/minificpp/.ccache/." "${docker_ccache_dump_location}"
+ docker rm -f "${container_id}"
+}
+
BUILD_ARGS=()
while [[ $# -gt 0 ]]; do
key="$1"
@@ -88,6 +98,8 @@ while [[ $# -gt 0 ]]; do
BUILD_ARGS+=("--build-arg" "BASE_ALPINE_IMAGE=${ARR[1]}")
elif [ "${ARR[0]}" == "DOCKER_CCACHE_DUMP_LOCATION" ]; then
DOCKER_CCACHE_DUMP_LOCATION="${ARR[1]}"
+ elif [ "${ARR[0]}" == "DOCKER_SKIP_TESTS" ]; then
+ DOCKER_SKIP_TESTS="${ARR[1]}"
else
BUILD_ARGS+=("--build-arg" "${ARR[0]}=${ARR[1]}")
fi
@@ -146,19 +158,25 @@ BUILD_ARGS+=("--build-arg" "UID=${UID_ARG}"
"--build-arg" "GID=${GID_ARG}"
"--build-arg" "MINIFI_VERSION=${MINIFI_VERSION}"
"--build-arg" "DUMP_LOCATION=${DUMP_LOCATION}"
- "--build-arg" "DISTRO_NAME=${DISTRO_NAME}")
+ "--build-arg" "DISTRO_NAME=${DISTRO_NAME}"
+ "--build-arg" "DOCKER_SKIP_TESTS=${DOCKER_SKIP_TESTS}")
-if [ -n "${DOCKER_CCACHE_DUMP_LOCATION}" ]; then
- DOCKER_BUILDKIT=1 docker build "${BUILD_ARGS[@]}" -f ${DOCKERFILE} --target build -t minifi_build ..
+if [ -n "${DISTRO_NAME}" ]; then
+ echo DOCKER_BUILDKIT=0 docker build "${BUILD_ARGS[@]}" -f "${DOCKERFILE}" -t apacheminificpp:"${TAG}" ..
+ DOCKER_BUILDKIT=0 docker build "${BUILD_ARGS[@]}" -f "${DOCKERFILE}" -t apacheminificpp:"${TAG}" ..
- container_id=$(docker run --rm -d minifi_build sh -c "while true; do sleep 1; done")
- mkdir -p "${DOCKER_CCACHE_DUMP_LOCATION}"
- docker cp "${container_id}:/home/minificpp/.ccache/." "${DOCKER_CCACHE_DUMP_LOCATION}"
- docker rm -f "${container_id}"
+ if [ -n "${DOCKER_CCACHE_DUMP_LOCATION}" ]; then
+ dump_ccache "apacheminificpp:${TAG}" "${DOCKER_CCACHE_DUMP_LOCATION}"
+ fi
+else
+ if [ -n "${DOCKER_CCACHE_DUMP_LOCATION}" ]; then
+ DOCKER_BUILDKIT=1 docker build "${BUILD_ARGS[@]}" -f "${DOCKERFILE}" --target build -t minifi_build ..
+ dump_ccache "minifi_build" "${DOCKER_CCACHE_DUMP_LOCATION}"
+ fi
+ echo DOCKER_BUILDKIT=1 docker build "${BUILD_ARGS[@]}" -f "${DOCKERFILE}" -t apacheminificpp:"${TAG}" ..
+ DOCKER_BUILDKIT=1 docker build "${BUILD_ARGS[@]}" -f "${DOCKERFILE}" -t apacheminificpp:"${TAG}" ..
fi
-DOCKER_BUILDKIT=0 docker build "${BUILD_ARGS[@]}" -f ${DOCKERFILE} -t apacheminificpp:"${TAG}" ..
-
if [ -n "${DUMP_LOCATION}" ]; then
docker run --rm --entrypoint cat "apacheminificpp:${TAG}" "/opt/minifi/build/nifi-minifi-cpp-${MINIFI_VERSION}.tar.gz" >"${DUMP_LOCATION}/nifi-minifi-cpp-${MINIFI_VERSION}-${TARGZ_TAG}.tar.gz"
fi
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 305e6ef9b..1eb88f757 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -30,6 +30,7 @@ ARG GID=1000
# use it to define cmake options (e.g. -DENABLE_AWS=ON -DENABLE_AZURE=ON)
ARG MINIFI_OPTIONS=""
ARG CMAKE_BUILD_TYPE=Release
+ARG DOCKER_SKIP_TESTS=ON
# Install the system dependencies needed for a build
RUN apk --no-cache add gcc \
@@ -77,7 +78,7 @@ USER ${USER}
ENV PATH /usr/lib/ccache/bin:${PATH}
RUN mkdir ${MINIFI_BASE_DIR}/build
WORKDIR ${MINIFI_BASE_DIR}/build
-RUN cmake -DSTATIC_BUILD= -DSKIP_TESTS=true ${MINIFI_OPTIONS} -DAWS_ENABLE_UNITY_BUILD=OFF -DEXCLUDE_BOOST=ON -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" .. && \
+RUN cmake -DSTATIC_BUILD= -DSKIP_TESTS=${DOCKER_SKIP_TESTS} ${MINIFI_OPTIONS} -DAWS_ENABLE_UNITY_BUILD=OFF -DEXCLUDE_BOOST=ON -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" .. && \
make -j "$(nproc)" package && \
tar -xzvf "${MINIFI_BASE_DIR}/build/nifi-minifi-cpp-${MINIFI_VERSION}.tar.gz" -C "${MINIFI_BASE_DIR}"
diff --git a/docker/centos/Dockerfile b/docker/centos/Dockerfile
index 285b567e6..830857323 100644
--- a/docker/centos/Dockerfile
+++ b/docker/centos/Dockerfile
@@ -17,33 +17,47 @@
#
# First stage: the build environment
-FROM centos:7 AS build_deps
+FROM centos:7
LABEL maintainer="Apache NiFi <de...@nifi.apache.org>"
ARG MINIFI_VERSION
+ARG UID=1000
+ARG GID=1000
+
+# MINIFI_OPTIONS will be passed directly to cmake
+# use it to define cmake options (e.g. -DENABLE_AWS=ON -DENABLE_AZURE=ON)
+ARG MINIFI_OPTIONS=""
+ARG CMAKE_BUILD_TYPE=Release
+ARG DOCKER_SKIP_TESTS=ON
# Install the system dependencies needed for a build
ENV MINIFI_BASE_DIR /opt/minifi
ENV MINIFI_HOME $MINIFI_BASE_DIR/nifi-minifi-cpp-$MINIFI_VERSION
+ENV USER minificpp
# gpsd-devel is in EPEL
RUN yum -y install epel-release && yum -y install java-1.8.0-openjdk java-1.8.0-openjdk-devel python36-devel sudo git which maven make libarchive boost-devel lua-devel libusbx-devel libpng-devel \
- gpsd-devel libpcap-devel
+ gpsd-devel libpcap-devel ccache
RUN mkdir -p $MINIFI_BASE_DIR
COPY . ${MINIFI_BASE_DIR}
-FROM build_deps AS release
+RUN cd $MINIFI_BASE_DIR && \
+ ./bootstrap.sh -t && \
+ ln -s /usr/bin/ccache /usr/lib64/ccache/c++
-# MINIFI_OPTIONS will be passed directly to cmake
-# use it to define cmake options (e.g. -DENABLE_AWS=ON -DENABLE_AZURE=ON)
-ARG MINIFI_OPTIONS=""
-ARG CMAKE_BUILD_TYPE=Release
-# Perform the build
-RUN cd $MINIFI_BASE_DIR \
- && ./bootstrap.sh -t \
- && cd build \
- && scl enable devtoolset-10 -- cmake3 -DSTATIC_BUILD= -DSKIP_TESTS=true ${MINIFI_OPTIONS} -DAWS_ENABLE_UNITY_BUILD=OFF -DEXCLUDE_BOOST=ON -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" .. \
- && scl enable devtoolset-10 -- make -j "$(nproc)" package
+# Setup minificpp user
+RUN groupadd -g ${GID} ${USER} && useradd -g ${GID} ${USER} && \
+ chown -R ${USER}:${USER} ${MINIFI_BASE_DIR} && \
+ if [ -d "${MINIFI_BASE_DIR}/.ccache" ]; then mv ${MINIFI_BASE_DIR}/.ccache /home/${USER}/.ccache; fi
+
+USER ${USER}
+# Perform the build
+RUN cd $MINIFI_BASE_DIR && \
+ cd build && \
+ source /opt/rh/devtoolset-10/enable && \
+ export PATH=/usr/lib64/ccache${PATH:+:${PATH}} && \
+ cmake3 -DSTATIC_BUILD= -DSKIP_TESTS=${DOCKER_SKIP_TESTS} ${MINIFI_OPTIONS} -DAWS_ENABLE_UNITY_BUILD=OFF -DEXCLUDE_BOOST=ON -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" .. && \
+ make -j "$(nproc)" package
diff --git a/extensions/sftp/tests/ListSFTPTests.cpp b/extensions/sftp/tests/ListSFTPTests.cpp
index 5baa407cf..f98a51d40 100644
--- a/extensions/sftp/tests/ListSFTPTests.cpp
+++ b/extensions/sftp/tests/ListSFTPTests.cpp
@@ -235,8 +235,8 @@ TEST_CASE_METHOD(ListSFTPTestsFixture, "ListSFTP list one file writes attributes
auto file = src_dir + "/vfs/nifi_test/tstFile.ext";
auto mtime_str = utils::timeutils::getDateTimeStr(std::chrono::time_point_cast<std::chrono::seconds>(utils::file::to_sys(utils::file::last_write_time(file).value())));
- uint64_t uid;
- uint64_t gid;
+ uint64_t uid = 0;
+ uint64_t gid = 0;
REQUIRE(true == utils::file::FileUtils::get_uid_gid(file, uid, gid));
uint32_t permissions;
REQUIRE(true == utils::file::FileUtils::get_permissions(file, permissions));
diff --git a/libminifi/include/core/extension/Utils.h b/libminifi/include/core/extension/Utils.h
index 693844ec3..53d915538 100644
--- a/libminifi/include/core/extension/Utils.h
+++ b/libminifi/include/core/extension/Utils.h
@@ -29,6 +29,7 @@
#include "utils/gsl.h"
#include "utils/StringUtils.h"
#include "utils/file/FileUtils.h"
+#include "utils/file/PathUtils.h"
namespace org::apache::nifi::minifi::core::extension::internal {
@@ -84,13 +85,15 @@ std::optional<LibraryDescriptor> asDynamicLibrary(const std::filesystem::path& p
#else
static const std::string_view prefix = "lib";
#endif
- const std::string filename = path.filename().string();
+ std::string filepath;
+ std::string filename;
+ utils::file::getFileNameAndPath(path.string(), filepath, filename);
if (!utils::StringUtils::startsWith(filename, prefix) || !utils::StringUtils::endsWith(filename, extension)) {
return {};
}
return LibraryDescriptor{
filename.substr(prefix.length(), filename.length() - extension.length() - prefix.length()),
- path.parent_path(),
+ filepath,
filename
};
}
diff --git a/libminifi/test/unit/CollectionUtilsTests.cpp b/libminifi/test/unit/CollectionUtilsTests.cpp
index 870f06209..cf487cf60 100644
--- a/libminifi/test/unit/CollectionUtilsTests.cpp
+++ b/libminifi/test/unit/CollectionUtilsTests.cpp
@@ -46,7 +46,7 @@ TEST_CASE("TestHaveCommonItem", "[haveCommonItem]") {
template<typename T>
struct MockSet : std::set<T> {
using std::set<T>::set;
- [[nodiscard]] auto find(const T& item) const -> decltype(this->std::set<T>::find(item)) {
+ auto find(const T& item) const -> decltype(this->std::set<T>::find(item)) { // NOLINT(modernize-use-nodiscard)
on_find_();
return std::set<T>::find(item);
}