You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by fg...@apache.org on 2026/02/26 15:25:14 UTC
(nifi-minifi-cpp) branch main updated (f856bb60f -> 2f26cb92f)
This is an automated email from the ASF dual-hosted git repository.
fgerlits pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
from f856bb60f MINIFICPP-2730 Check if config.yml exists when creating the debug bundle
new 30fa5d153 MINIFICPP-2726 Fix HTTP S2S no data and error handling
new 162ccfb55 MINIFICPP-2732 Clear class descriptions when extensions are cleared
new 4096b8637 MINIFICPP-2733 Fix compilation on Windows in debug mode with Ninja
new 24933300e MINIFICPP-2734 Remove BinFiles from processor list
new 2f26cb92f MINIFICPP-2723 Add log and properties.d dir property files to debug bundle
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CMakeLists.txt | 2 +-
PROCESSORS.md | 29 ----------
README.md | 2 +-
.../minifi_test_framework/steps/checking_steps.py | 10 ++--
core-framework/src/http/HTTPClient.cpp | 5 +-
docker/test/integration/features/s2s.feature | 4 ++
extensions/libarchive/BinFiles.cpp | 2 -
.../include/core/extension/ExtensionManager.h | 6 ++
libminifi/include/properties/Configure.h | 3 +
libminifi/include/properties/Properties.h | 3 +-
libminifi/src/Configure.cpp | 14 +++++
libminifi/src/FlowController.cpp | 17 ++++++
libminifi/src/agent/agent_docs.cpp | 3 +
libminifi/src/core/extension/ExtensionManager.cpp | 8 +++
libminifi/src/sitetosite/HttpSiteToSiteClient.cpp | 65 +++++++++++++++-------
libminifi/test/integration/C2DebugBundleTest.cpp | 22 ++++++--
.../test/libtest/integration/IntegrationBase.cpp | 3 +-
.../test/libtest/integration/IntegrationBase.h | 9 ++-
minifi-api/include/minifi-cpp/agent/agent_docs.h | 1 +
.../include/minifi-cpp/properties/Configure.h | 3 +
.../include/minifi-cpp/properties/Properties.h | 1 +
21 files changed, 145 insertions(+), 67 deletions(-)
(nifi-minifi-cpp) 03/05: MINIFICPP-2733 Fix compilation on Windows in debug mode with Ninja
Posted by fg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
fgerlits pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 4096b863734d1d8000fd64357561044e1fc2d013
Author: Gabor Gyimesi <ga...@gmail.com>
AuthorDate: Thu Feb 26 14:22:06 2026 +0100
MINIFICPP-2733 Fix compilation on Windows in debug mode with Ninja
Signed-off-by: Ferenc Gerlits <fg...@gmail.com>
Closes #2122
---
CMakeLists.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 595e570de..4bc5ce888 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -105,7 +105,7 @@ if (WIN32)
add_compile_options(/W3 /utf-8 /bigobj /MP /diagnostics:caret)
if (CMAKE_GENERATOR STREQUAL "Ninja")
# these flags are default on msbuild, but not on ninja
- add_compile_options(/MD /GS /fp:precise /Zc:wchar_t /Zc:forScope /Zc:inline)
+ add_compile_options($<$<CONFIG:Debug>:/MDd> $<$<NOT:$<CONFIG:Debug>>:/MD> /GS /fp:precise /Zc:wchar_t /Zc:forScope /Zc:inline)
endif()
endif()
(nifi-minifi-cpp) 05/05: MINIFICPP-2723 Add log and properties.d dir property files to debug bundle
Posted by fg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
fgerlits pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 2f26cb92f5331a8facbfd7354d5dfca148895b74
Author: Gabor Gyimesi <ga...@gmail.com>
AuthorDate: Thu Feb 26 15:53:20 2026 +0100
MINIFICPP-2723 Add log and properties.d dir property files to debug bundle
Signed-off-by: Ferenc Gerlits <fg...@gmail.com>
Closes #2124
---
libminifi/include/properties/Configure.h | 3 +++
libminifi/include/properties/Properties.h | 3 ++-
libminifi/src/Configure.cpp | 14 ++++++++++++++
libminifi/src/FlowController.cpp | 17 +++++++++++++++++
libminifi/test/integration/C2DebugBundleTest.cpp | 22 ++++++++++++++++++----
.../test/libtest/integration/IntegrationBase.cpp | 3 ++-
.../test/libtest/integration/IntegrationBase.h | 9 ++++++++-
.../include/minifi-cpp/properties/Configure.h | 3 +++
.../include/minifi-cpp/properties/Properties.h | 1 +
9 files changed, 68 insertions(+), 7 deletions(-)
diff --git a/libminifi/include/properties/Configure.h b/libminifi/include/properties/Configure.h
index d869ad7da..7f35c8324 100644
--- a/libminifi/include/properties/Configure.h
+++ b/libminifi/include/properties/Configure.h
@@ -49,6 +49,9 @@ class ConfigureImpl : public ConfigurationImpl, public virtual core::AgentIdenti
std::string getAgentIdentifier() const override;
void setFallbackAgentIdentifier(const std::string& id) override;
+ std::optional<std::filesystem::path> logPropertiesFilePath() const override;
+ std::optional<std::filesystem::path> extraLogPropertiesFilesDirName() const override;
+
using Configuration::set;
void set(const std::string& key, const std::string& value, PropertyChangeLifetime lifetime) override;
bool commitChanges() override;
diff --git a/libminifi/include/properties/Properties.h b/libminifi/include/properties/Properties.h
index 4f94f6401..f4880a557 100644
--- a/libminifi/include/properties/Properties.h
+++ b/libminifi/include/properties/Properties.h
@@ -124,8 +124,9 @@ class PropertiesImpl : public virtual Properties {
std::map<std::string, std::string> getProperties() const override;
+ std::filesystem::path extraPropertiesFilesDirName() const override;
+
private:
- std::filesystem::path extraPropertiesFilesDirName() const;
void setPropertiesFromFile(const std::filesystem::path& properties_file, std::string_view prefix);
std::map<std::string, PropertyValue> properties_;
diff --git a/libminifi/src/Configure.cpp b/libminifi/src/Configure.cpp
index a0ad4f91d..35280d56c 100644
--- a/libminifi/src/Configure.cpp
+++ b/libminifi/src/Configure.cpp
@@ -110,6 +110,20 @@ void ConfigureImpl::setFallbackAgentIdentifier(const std::string& id) {
fallback_identifier_ = id;
}
+std::optional<std::filesystem::path> ConfigureImpl::logPropertiesFilePath() const {
+ if (logger_properties_) {
+ return logger_properties_->getFilePath();
+ }
+ return std::nullopt;
+}
+
+std::optional<std::filesystem::path> ConfigureImpl::extraLogPropertiesFilesDirName() const {
+ if (logger_properties_) {
+ return logger_properties_->extraPropertiesFilesDirName();
+ }
+ return std::nullopt;
+}
+
void ConfigureImpl::set(const std::string& key, const std::string& value, PropertyChangeLifetime lifetime) {
const std::string_view log_prefix = "nifi.log.";
if (utils::string::startsWith(key, log_prefix)) {
diff --git a/libminifi/src/FlowController.cpp b/libminifi/src/FlowController.cpp
index 16a43ec13..a0ab0f90c 100644
--- a/libminifi/src/FlowController.cpp
+++ b/libminifi/src/FlowController.cpp
@@ -482,12 +482,29 @@ std::map<std::string, std::unique_ptr<io::InputStream>> FlowController::getDebug
std::string index_str = i == logs.size() - 1 ? "" : "." + std::to_string(logs.size() - 1 - i);
debug_info["minifi.log" + index_str + ".gz"] = std::move(logs[i]);
}
+
if (auto opt_flow_path = flow_configuration_->getConfigurationPath()) {
if (utils::file::exists(opt_flow_path.value())) {
debug_info["config.yml"] = std::make_unique<io::FileStream>(opt_flow_path.value(), 0, false);
}
}
+
+ const auto addDirectoryContents = [this, &debug_info](const std::filesystem::path& dir) {
+ if (!utils::file::exists(dir)) { return; }
+ for (const auto& [parent, file] : utils::file::list_dir_all(dir, logger_, false)) {
+ debug_info[parent.filename().string() + "/" + file.string()] = std::make_unique<io::FileStream>(dir / file, 0, false);
+ }
+ };
+
debug_info["minifi.properties"] = std::make_unique<io::FileStream>(configuration_->getFilePath(), 0, false);
+ addDirectoryContents(configuration_->extraPropertiesFilesDirName());
+
+ if (auto log_properties_path = configuration_->logPropertiesFilePath()) {
+ debug_info["minifi-log.properties"] = std::make_unique<io::FileStream>(*log_properties_path, 0, false);
+ }
+ if (auto log_properties_dir = configuration_->extraLogPropertiesFilesDirName()) {
+ addDirectoryContents(*log_properties_dir);
+ }
return debug_info;
}
diff --git a/libminifi/test/integration/C2DebugBundleTest.cpp b/libminifi/test/integration/C2DebugBundleTest.cpp
index 6a5ab86b3..f68f8d7a3 100644
--- a/libminifi/test/integration/C2DebugBundleTest.cpp
+++ b/libminifi/test/integration/C2DebugBundleTest.cpp
@@ -138,6 +138,9 @@ class C2HeartbeatHandler : public ServerAwareHandler {
};
static std::string properties_file = "some.dummy.content = here\n";
+static std::string c2_properties_file = "c2.dummy.content = content\n";
+static std::string log_properties_file = "log.dummy.content = log\n";
+static std::string extra_log_properties_file = "extra.log.dummy.content = extra\n";
static std::string flow_config_file = empty_flow;
TEST_CASE("C2DebugBundleTest", "[c2test]") {
@@ -149,7 +152,12 @@ TEST_CASE("C2DebugBundleTest", "[c2test]") {
std::filesystem::path home_dir = controller.createTempDirectory();
minifi::utils::file::PathUtils::create_dir(home_dir / "conf");
+ minifi::utils::file::PathUtils::create_dir(home_dir / "conf" / "minifi.properties.d");
+ minifi::utils::file::PathUtils::create_dir(home_dir / "conf" / "minifi-log.properties.d");
std::ofstream{home_dir / "conf" / "minifi.properties", std::ios::binary} << properties_file;
+ std::ofstream{home_dir / "conf" / "minifi.properties.d" / "90_c2.properties", std::ios::binary} << c2_properties_file;
+ std::ofstream{home_dir / "conf" / "minifi-log.properties", std::ios::binary} << log_properties_file;
+ std::ofstream{home_dir / "conf" / "minifi-log.properties.d" / "99_c2.properties", std::ios::binary} << extra_log_properties_file;
std::ofstream{home_dir / "conf" / "config.yml", std::ios::binary} << flow_config_file;
VerifyDebugInfo harness(home_dir / "conf" / "config.yml", [&]() -> bool {
@@ -174,8 +182,11 @@ TEST_CASE("C2DebugBundleTest", "[c2test]") {
file_content.length());
archive_content[info->filename] = std::move(file_content);
}
- REQUIRE(archive_content["minifi.properties"] == properties_file);
- REQUIRE(archive_content["config.yml"] == flow_config_file);
+ CHECK(archive_content["minifi.properties"] == properties_file);
+ CHECK(archive_content["minifi.properties.d/90_c2.properties"] == c2_properties_file);
+ CHECK(archive_content["minifi-log.properties"] == log_properties_file);
+ CHECK(archive_content["minifi-log.properties.d/99_c2.properties"] == extra_log_properties_file);
+ CHECK(archive_content["config.yml"] == flow_config_file);
auto log_gz = archive_content["minifi.log.gz"];
auto log_stream = std::make_shared<minifi::io::BufferStream>();
{
@@ -185,11 +196,12 @@ TEST_CASE("C2DebugBundleTest", "[c2test]") {
std::string log_text;
log_text.resize(log_stream->size());
log_stream->read(as_writable_bytes(std::span(log_text)));
- REQUIRE(log_text.find("Tis but a scratch") != std::string::npos);
- REQUIRE(archive_content["manifest.json"].find("minifi-archive-extensions") != std::string::npos);
+ CHECK(log_text.find("Tis but a scratch") != std::string::npos);
+ CHECK(archive_content["manifest.json"].find("minifi-archive-extensions") != std::string::npos);
return true;
});
+ harness.getLoggerProperties()->loadConfigureFile(home_dir / "conf" / "minifi-log.properties");
harness.getConfiguration()->loadConfigureFile(home_dir / "conf" / "minifi.properties");
harness.setUrl("http://localhost:0/heartbeat", &heartbeat_handler);
harness.setUrl("http://localhost:0/acknowledge", &ack_handler);
@@ -213,6 +225,7 @@ TEST_CASE("Test that the debug bundle operation works when config.yml does not e
std::filesystem::path home_dir = controller.createTempDirectory();
minifi::utils::file::PathUtils::create_dir(home_dir / "conf");
std::ofstream{home_dir / "conf" / "minifi.properties", std::ios::binary} << properties_file;
+ std::ofstream{home_dir / "conf" / "minifi-log.properties", std::ios::binary} << log_properties_file;
VerifyDebugInfo harness(home_dir / "conf" / "config.yml", [&]() -> bool {
if (!ack_handler.isAcknowledged("79")) {
@@ -224,6 +237,7 @@ TEST_CASE("Test that the debug bundle operation works when config.yml does not e
});
harness.getConfiguration()->loadConfigureFile(home_dir / "conf" / "minifi.properties");
+ harness.getLoggerProperties()->loadConfigureFile(home_dir / "conf" / "minifi-log.properties");
harness.setUrl("http://localhost:0/heartbeat", &heartbeat_handler);
harness.setUrl("http://localhost:0/acknowledge", &ack_handler);
harness.setUrl("http://localhost:0/debug_bundle", &bundle_handler);
diff --git a/libminifi/test/libtest/integration/IntegrationBase.cpp b/libminifi/test/libtest/integration/IntegrationBase.cpp
index cb29d5a02..8a767546a 100644
--- a/libminifi/test/libtest/integration/IntegrationBase.cpp
+++ b/libminifi/test/libtest/integration/IntegrationBase.cpp
@@ -31,7 +31,8 @@
namespace org::apache::nifi::minifi::test {
IntegrationBase::IntegrationBase(const std::optional<std::filesystem::path>& test_file_location, const std::optional<std::filesystem::path>& home_path, std::chrono::milliseconds waitTime)
- : configuration(std::make_shared<minifi::ConfigureImpl>()),
+ : logger_properties_(std::make_shared<core::logging::LoggerProperties>(std::filesystem::temp_directory_path())),
+ configuration(std::make_shared<minifi::ConfigureImpl>(std::nullopt, logger_properties_)),
wait_time_(waitTime),
home_path_(home_path) {
flow_config_path_.config_path = test_file_location;
diff --git a/libminifi/test/libtest/integration/IntegrationBase.h b/libminifi/test/libtest/integration/IntegrationBase.h
index df7a29688..9f406d57a 100644
--- a/libminifi/test/libtest/integration/IntegrationBase.h
+++ b/libminifi/test/libtest/integration/IntegrationBase.h
@@ -50,7 +50,8 @@ class IntegrationBase {
std::chrono::milliseconds waitTime = std::chrono::milliseconds(DEFAULT_WAITTIME_MSECS));
IntegrationBase(const IntegrationBase&) = delete;
IntegrationBase(IntegrationBase&& other) noexcept
- : configuration{std::move(other.configuration)},
+ : logger_properties_{std::move(other.logger_properties_)},
+ configuration{std::move(other.configuration)},
flowController_{std::move(other.flowController_)},
wait_time_{other.wait_time_},
port{std::move(other.port)},
@@ -64,6 +65,7 @@ class IntegrationBase {
IntegrationBase& operator=(const IntegrationBase&) = delete;
IntegrationBase& operator=(IntegrationBase&& other) noexcept {
if (&other == this) return *this;
+ logger_properties_ = std::move(other.logger_properties_);
configuration = std::move(other.configuration);
flowController_ = std::move(other.flowController_);
wait_time_ = other.wait_time_;
@@ -94,6 +96,10 @@ class IntegrationBase {
return configuration;
}
+ const std::shared_ptr<core::logging::LoggerProperties>& getLoggerProperties() const {
+ return logger_properties_;
+ }
+
void setConfiguration(std::shared_ptr<minifi::Configure> configuration) {
this->configuration = std::move(configuration);
}
@@ -124,6 +130,7 @@ class IntegrationBase {
}
void configureSecurity();
+ std::shared_ptr<core::logging::LoggerProperties> logger_properties_;
std::shared_ptr<minifi::Configure> configuration;
std::unique_ptr<minifi::utils::file::AssetManager> asset_manager_;
std::unique_ptr<core::BulletinStore> bulletin_store_;
diff --git a/minifi-api/include/minifi-cpp/properties/Configure.h b/minifi-api/include/minifi-cpp/properties/Configure.h
index 12427cf8e..5402e5a18 100644
--- a/minifi-api/include/minifi-cpp/properties/Configure.h
+++ b/minifi-api/include/minifi-cpp/properties/Configure.h
@@ -41,6 +41,9 @@ class Configure : public virtual Configuration, public virtual core::AgentIdenti
virtual void setFallbackAgentIdentifier(const std::string& id) = 0;
+ virtual std::optional<std::filesystem::path> logPropertiesFilePath() const = 0;
+ virtual std::optional<std::filesystem::path> extraLogPropertiesFilesDirName() const = 0;
+
using Configuration::set;
void set(const std::string& key, const std::string& value, PropertyChangeLifetime lifetime) override = 0;
bool commitChanges() override = 0;
diff --git a/minifi-api/include/minifi-cpp/properties/Properties.h b/minifi-api/include/minifi-cpp/properties/Properties.h
index a9573a09a..a40d540fd 100644
--- a/minifi-api/include/minifi-cpp/properties/Properties.h
+++ b/minifi-api/include/minifi-cpp/properties/Properties.h
@@ -60,6 +60,7 @@ class Properties {
virtual utils::ChecksumCalculator& getChecksumCalculator() = 0;
virtual std::filesystem::path getFilePath() const = 0;
virtual std::map<std::string, std::string> getProperties() const = 0;
+ virtual std::filesystem::path extraPropertiesFilesDirName() const = 0;
static std::shared_ptr<Properties> create();
};
(nifi-minifi-cpp) 01/05: MINIFICPP-2726 Fix HTTP S2S no data and error handling
Posted by fg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
fgerlits pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 30fa5d1530fb07ce3fe5e026d04add890ae6d072
Author: Gabor Gyimesi <ga...@gmail.com>
AuthorDate: Thu Feb 26 14:20:12 2026 +0100
MINIFICPP-2726 Fix HTTP S2S no data and error handling
- In case no data is sent we do not send cancel message to the remote
peer, this is in line with NiFi-NiFi site to site communication
- Handle errors in case HTTP client submit fails
- Fix handling of 4xx and 5xx HTTP response codes: fail in case client
side error code is received, only log error in case server side error
is received
- Log additional data for HTTP client communication
Signed-off-by: Ferenc Gerlits <fg...@gmail.com>
Closes #2116
---
.../minifi_test_framework/steps/checking_steps.py | 10 ++--
core-framework/src/http/HTTPClient.cpp | 5 +-
docker/test/integration/features/s2s.feature | 4 ++
libminifi/src/sitetosite/HttpSiteToSiteClient.cpp | 65 +++++++++++++++-------
4 files changed, 57 insertions(+), 27 deletions(-)
diff --git a/behave_framework/src/minifi_test_framework/steps/checking_steps.py b/behave_framework/src/minifi_test_framework/steps/checking_steps.py
index c6593c1bf..2538953d7 100644
--- a/behave_framework/src/minifi_test_framework/steps/checking_steps.py
+++ b/behave_framework/src/minifi_test_framework/steps/checking_steps.py
@@ -23,7 +23,7 @@ import humanfriendly
from behave import then, step
from minifi_test_framework.containers.http_proxy_container import HttpProxy
-from minifi_test_framework.core.helpers import wait_for_condition, check_condition_after_wait
+from minifi_test_framework.core.helpers import wait_for_condition, check_condition_after_wait, log_due_to_failure
from minifi_test_framework.core.minifi_test_context import DEFAULT_MINIFI_CONTAINER_NAME, MinifiTestContext
@@ -67,17 +67,17 @@ def step_impl(context: MinifiTestContext, content: str, directory: str, duration
def step_impl(context: MinifiTestContext, message: str, duration: str):
duration_seconds = humanfriendly.parse_timespan(duration)
time.sleep(duration_seconds)
- assert message not in context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_logs()
+ assert message not in context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_logs() or log_due_to_failure(context)
@then("the Minifi logs do not contain errors")
def step_impl(context: MinifiTestContext):
- assert "[error]" not in context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_logs() or context.containers[DEFAULT_MINIFI_CONTAINER_NAME].log_app_output()
+ assert "[error]" not in context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_logs() or log_due_to_failure(context)
@then("the Minifi logs do not contain warnings")
def step_impl(context: MinifiTestContext):
- assert "[warning]" not in context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_logs() or context.containers[DEFAULT_MINIFI_CONTAINER_NAME].log_app_output()
+ assert "[warning]" not in context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_logs() or log_due_to_failure(context)
@then("the Minifi logs contain the following message: '{message}' in less than {duration}")
@@ -93,7 +93,7 @@ def step_impl(context: MinifiTestContext, message: str, duration: str):
def step_impl(context, log_message, count, duration):
duration_seconds = humanfriendly.parse_timespan(duration)
time.sleep(duration_seconds)
- assert context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_logs().count(log_message) == count or context.containers[DEFAULT_MINIFI_CONTAINER_NAME].log_app_output()
+ assert context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_logs().count(log_message) == count or log_due_to_failure(context)
@then("the Minifi logs match the following regex: \"{regex}\" in less than {duration}")
diff --git a/core-framework/src/http/HTTPClient.cpp b/core-framework/src/http/HTTPClient.cpp
index 76e5d1116..fae2a78c9 100644
--- a/core-framework/src/http/HTTPClient.cpp
+++ b/core-framework/src/http/HTTPClient.cpp
@@ -357,7 +357,7 @@ bool HTTPClient::submit() {
}
curl_easy_setopt(http_session_.get(), CURLOPT_URL, url_.c_str());
- logger_->log_debug("Submitting to {}", url_);
+ logger_->log_debug("Submitting to {} {}", method_ ? magic_enum::enum_name(*method_) : "NONE", url_);
if (read_callback_ == nullptr) {
curl_easy_setopt(http_session_.get(), CURLOPT_WRITEFUNCTION, &HTTPRequestResponse::receiveWrite);
curl_easy_setopt(http_session_.get(), CURLOPT_WRITEDATA, static_cast<void*>(&content_));
@@ -381,12 +381,11 @@ bool HTTPClient::submit() {
logger_->log_error("HTTP operation timed out, with absolute timeout {}\n", absolute_timeout);
}
if (res_ != CURLE_OK) {
- logger_->log_info("{}", request_headers_.size());
logger_->log_error("curl_easy_perform() failed {} on {}, error code {}\n", curl_easy_strerror(res_), url_, magic_enum::enum_underlying(res_));
return false;
}
- logger_->log_debug("Finished with {}", url_);
+ logger_->log_debug("Finished with {} {}", method_ ? magic_enum::enum_name(*method_) : "NONE", url_);
return true;
}
diff --git a/docker/test/integration/features/s2s.feature b/docker/test/integration/features/s2s.feature
index 9b3321d4f..711a520e3 100644
--- a/docker/test/integration/features/s2s.feature
+++ b/docker/test/integration/features/s2s.feature
@@ -209,6 +209,7 @@ Feature: Sending data from MiNiFi-C++ to NiFi using S2S protocol
Then a flowfile with the content "test" is placed in the monitored directory in less than 90 seconds
And the Minifi logs do not contain the following message: "ProcessSession rollback" after 1 seconds
+ And the Minifi logs do not contain the following message: "response code 500" after 1 seconds
Scenario: A NiFi instance produces and transfers data to a MiNiFi instance via s2s
Given a file with the content "test" is present in "/tmp/input"
@@ -245,6 +246,7 @@ Feature: Sending data from MiNiFi-C++ to NiFi using S2S protocol
Then a flowfile with the content "test" is placed in the monitored directory in less than 90 seconds
And the Minifi logs do not contain the following message: "ProcessSession rollback" after 1 seconds
+ And the Minifi logs do not contain the following message: "response code 500" after 1 seconds
Scenario: A NiFi instance produces and transfers data to a MiNiFi instance via s2s with SSL config defined in minifi.properties
Given a file with the content "test" is present in "/tmp/input"
@@ -285,6 +287,7 @@ Feature: Sending data from MiNiFi-C++ to NiFi using S2S protocol
Then a flowfile with the content "test" is placed in the monitored directory in less than 90 seconds
And the Minifi logs do not contain the following message: "ProcessSession rollback" after 1 seconds
+ And the Minifi logs do not contain the following message: "response code 500" after 1 seconds
Scenario: A MiNiFi instance produces and transfers data to a NiFi instance via s2s using compression
Given a GetFile processor with the "Input Directory" property set to "/tmp/input"
@@ -370,3 +373,4 @@ Feature: Sending data from MiNiFi-C++ to NiFi using S2S protocol
Then a flowfile with the content "test" is placed in the monitored directory in less than 90 seconds
And the Minifi logs do not contain the following message: "ProcessSession rollback" after 1 seconds
+ And the Minifi logs do not contain the following message: "response code 500" after 1 seconds
diff --git a/libminifi/src/sitetosite/HttpSiteToSiteClient.cpp b/libminifi/src/sitetosite/HttpSiteToSiteClient.cpp
index aacc70609..e2862afd9 100644
--- a/libminifi/src/sitetosite/HttpSiteToSiteClient.cpp
+++ b/libminifi/src/sitetosite/HttpSiteToSiteClient.cpp
@@ -102,7 +102,10 @@ std::shared_ptr<Transaction> HttpSiteToSiteClient::createTransaction(TransferDir
client->setRequestHeader("Accept", "application/json");
client->setRequestHeader("Transfer-Encoding", "chunked");
client->setPostFields("");
- client->submit();
+ if (!client->submit()) {
+ logger_->log_warn("Failed to submit create transaction request for transaction {}", uri.str());
+ return nullptr;
+ }
if (auto http_stream = dynamic_cast<http::HttpStream*>(peer_->getStream())) {
logger_->log_debug("Closing {}", http_stream->getClientRef()->getURL());
@@ -110,7 +113,7 @@ std::shared_ptr<Transaction> HttpSiteToSiteClient::createTransaction(TransferDir
if (client->getResponseCode() != 201) {
peer_->setStream(nullptr);
- logger_->log_debug("Could not create transaction, received {}", client->getResponseCode());
+ logger_->log_debug("Could not create transaction, received response code {}", client->getResponseCode());
return nullptr;
}
// parse the headers
@@ -242,7 +245,10 @@ std::optional<std::vector<PeerStatus>> HttpSiteToSiteClient::getPeerList() {
setSiteToSiteHeaders(*client);
- client->submit();
+ if (!client->submit()) {
+ logger_->log_warn("Failed to submit get peer list request {}", uri.str());
+ return std::nullopt;
+ }
if (client->getResponseCode() == 200) {
return parsePeerStatuses(logger_, std::string(client->getResponseBody().data(), client->getResponseBody().size()), port_id_);
@@ -314,6 +320,11 @@ void HttpSiteToSiteClient::closeTransaction(const utils::Identifier &transaction
return;
}
+ const auto guard = gsl::finally([&transaction]() {
+ transaction->close();
+ transaction->decrementCurrentTransfers();
+ });
+
logger_->log_trace("Site to Site closing transaction {}", transaction->getUUIDStr());
bool data_received = transaction->getDirection() == TransferDirection::RECEIVE && (current_code_ == ResponseCode::CONFIRM_TRANSACTION || current_code_ == ResponseCode::TRANSACTION_FINISHED);
@@ -326,7 +337,9 @@ void HttpSiteToSiteClient::closeTransaction(const utils::Identifier &transaction
if (transaction->getState() == TransactionState::TRANSACTION_CONFIRMED || data_received) {
code = ResponseCode::CONFIRM_TRANSACTION;
} else if (transaction->getCurrentTransfers() == 0 && !transaction->isDataAvailable()) {
- code = ResponseCode::CANCEL_TRANSACTION;
+ // If there was no data to send, the transaction is removed on server side, no need to send delete request.
+ logger_->log_debug("Transaction {} canceled with no transfers, skipping DELETE to server", transaction->getUUIDStr());
+ return;
} else {
std::string directon = transaction->getDirection() == TransferDirection::RECEIVE ? "Receive" : "Send";
logger_->log_error("Transaction {} to be closed is in unexpected state. Direction: {}, transfers: {}, bytes: {}, state: {}",
@@ -347,19 +360,21 @@ void HttpSiteToSiteClient::closeTransaction(const utils::Identifier &transaction
setSiteToSiteHeaders(*client);
client->setConnectionTimeout(std::chrono::milliseconds(5000));
client->setRequestHeader("Accept", "application/json");
- client->submit();
-
- logger_->log_debug("Received {} response code from delete", client->getResponseCode());
-
- if (client->getResponseCode() >= 400) {
- std::string error(client->getResponseBody().data(), client->getResponseBody().size());
- logger_->log_warn("{} received: {}", client->getResponseCode(), error);
- throw Exception(SITE2SITE_EXCEPTION, fmt::format("Received {} from {}", client->getResponseCode(), uri.str()));
+ if (!client->submit()) {
+ logger_->log_warn("Failed to submit delete transaction request for transaction {}", transaction_id.to_string());
+ } else {
+ if (client->getResponseCode() >= 400) {
+ const std::string error(client->getResponseBody().data(), client->getResponseBody().size());
+ const auto message = fmt::format("Received response code {} while deleting transaction {}: {}", client->getResponseCode(), transaction_id.to_string(), error);
+ if (client->getResponseCode() < 500) {
+ logger_->log_error(fmt::runtime(message));
+ throw Exception(SITE2SITE_EXCEPTION, message);
+ } else {
+ logger_->log_warn(fmt::runtime(message));
+ }
+ }
}
-
- transaction->close();
- transaction->decrementCurrentTransfers();
}
void HttpSiteToSiteClient::deleteTransaction(const utils::Identifier& transaction_id) {
@@ -395,6 +410,8 @@ std::pair<uint64_t, uint64_t> HttpSiteToSiteClient::readFlowFiles(const std::sha
try {
read_result = SiteToSiteClient::readFlowFiles(transaction, session);
} catch (const Exception&) {
+ // Wait for the HTTP response to fully complete before checking the response code
+ http_stream->getClient();
auto response_code = http_stream->getClientRef()->getResponseCode();
// 200 tells us that there is no content to read, so we should not treat it as an error.
@@ -407,11 +424,21 @@ std::pair<uint64_t, uint64_t> HttpSiteToSiteClient::readFlowFiles(const std::sha
current_code_ = ResponseCode::CANCEL_TRANSACTION;
return {0, 0};
}
- throw;
- }
- if (auto response_code = http_stream->getClientRef()->getResponseCode(); response_code >= 400) {
- throw Exception(SITE2SITE_EXCEPTION, fmt::format("HTTP error code received while reading flow files: {}", response_code));
+ if (response_code >= 400) {
+ const std::string error = std::string(http_stream->getClientRef()->getResponseBody().data(), http_stream->getClientRef()->getResponseBody().size());
+ const auto message = fmt::format("Received response code {} while reading flow files for transaction {}: {}", response_code, transaction->getUUIDStr(), error);
+ if (response_code < 500) {
+ logger_->log_error(fmt::runtime(message));
+ throw Exception(SITE2SITE_EXCEPTION, message);
+ } else {
+ logger_->log_warn(fmt::runtime(message));
+ transaction->setState(TransactionState::TRANSACTION_ERROR);
+ current_code_ = ResponseCode::CANCEL_TRANSACTION;
+ return {0, 0};
+ }
+ }
+ throw;
}
return read_result;
}
(nifi-minifi-cpp) 04/05: MINIFICPP-2734 Remove BinFiles from processor list
Posted by fg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
fgerlits pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 24933300e2d35ae8c517e02557e9c966daa2ca85
Author: Gabor Gyimesi <ga...@gmail.com>
AuthorDate: Thu Feb 26 14:22:35 2026 +0100
MINIFICPP-2734 Remove BinFiles from processor list
Signed-off-by: Ferenc Gerlits <fg...@gmail.com>
Closes #2123
---
PROCESSORS.md | 29 -----------------------------
README.md | 2 +-
extensions/libarchive/BinFiles.cpp | 2 --
3 files changed, 1 insertion(+), 32 deletions(-)
diff --git a/PROCESSORS.md b/PROCESSORS.md
index 7851a06af..4a3d19507 100644
--- a/PROCESSORS.md
+++ b/PROCESSORS.md
@@ -19,7 +19,6 @@ limitations under the License.
- [ApplyTemplate](#ApplyTemplate)
- [AttributeRollingWindow](#AttributeRollingWindow)
- [AttributesToJSON](#AttributesToJSON)
-- [BinFiles](#BinFiles)
- [CaptureRTSPFrame](#CaptureRTSPFrame)
- [CollectKubernetesPodMetrics](#CollectKubernetesPodMetrics)
- [CompressContent](#CompressContent)
@@ -217,34 +216,6 @@ In the list below, the names of required properties appear in bold. Any other pr
| success | All FlowFiles received are routed to success |
-## BinFiles
-
-### Description
-
-Bins flow files into buckets based on the number of entries or size of entries
-
-### Properties
-
-In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language.
-
-| Name | Default Value | Allowable Values | Description |
-|---------------------------|---------------|------------------|------------------------------------------------------------------------------------------------------------|
-| Minimum Group Size | 0 | | The minimum size of for the bundle |
-| Maximum Group Size | | | The maximum size for the bundle. If not specified, there is no maximum. |
-| Minimum Number of Entries | 1 | | The minimum number of files to include in a bundle |
-| Maximum Number of Entries | | | The maximum number of files to include in a bundle. If not specified, there is no maximum. |
-| Maximum number of Bins | 100 | | Specifies the maximum number of bins that can be held in memory at any one time |
-| Max Bin Age | | | The maximum age of a Bin that will trigger a Bin to be complete. Expected format is <duration> <time unit> |
-| Batch Size | 1 | | Maximum number of FlowFiles processed in a single session |
-
-### Relationships
-
-| Name | Description |
-|----------|------------------------------------------------------------------------------------------------------------------------------|
-| failure | If the bundle cannot be created, all FlowFiles that would have been used to create the bundle will be transferred to failure |
-| original | The FlowFiles that were used to create the bundle |
-
-
## CaptureRTSPFrame
### Description
diff --git a/README.md b/README.md
index 1d0798b69..05388a282 100644
--- a/README.md
+++ b/README.md
@@ -72,7 +72,7 @@ The next table outlines CMAKE flags that correspond with MiNiFi extensions. Exte
| Extension Set | Processors and Controller Services [...]
|----------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- [...]
-| Archive Extensions | [ApplyTemplate](PROCESSORS.md#applytemplate)<br/>[BinFiles](PROCESSORS.md#binfiles)<br/>[CompressContent](PROCESSORS.md#compresscontent)<br/>[ManipulateArchive](PROCESSORS.md#manipulatearchive)<br/>[MergeContent](PROCESSORS.md#mergecontent)<br/>[FocusArchiveEntry](PROCESSORS.md#focusarchiveentry)<br/>[UnfocusArchiveEntry](PROCESSORS.md#unfocusarchiveentry) [...]
+| Archive Extensions | [ApplyTemplate](PROCESSORS.md#applytemplate)<br/>[CompressContent](PROCESSORS.md#compresscontent)<br/>[ManipulateArchive](PROCESSORS.md#manipulatearchive)<br/>[MergeContent](PROCESSORS.md#mergecontent)<br/>[FocusArchiveEntry](PROCESSORS.md#focusarchiveentry)<br/>[UnfocusArchiveEntry](PROCESSORS.md#unfocusarchiveentry) [...]
| AWS | [AWSCredentialsService](CONTROLLERS.md#awscredentialsservice)<br/>[PutS3Object](PROCESSORS.md#puts3object)<br/>[DeleteS3Object](PROCESSORS.md#deletes3object)<br/>[FetchS3Object](PROCESSORS.md#fetchs3object)<br/>[ListS3](PROCESSORS.md#lists3)<br/>[PutKinesisStream](PROCESSORS.md#putkinesisstream) [...]
| Azure | [AzureStorageCredentialsService](CONTROLLERS.md#azurestoragecredentialsservice)<br/>[PutAzureBlobStorage](PROCESSORS.md#putazureblobstorage)<br/>[DeleteAzureBlobStorage](PROCESSORS.md#deleteazureblobstorage)<br/>[FetchAzureBlobStorage](PROCESSORS.md#fetchazureblobstorage)<br/>[ListAzureBlobStorage](PROCESSORS.md#listazureblobstorage)<br/>[PutAzureDataLakeStorage](PROCESSORS.md#putazuredatalakestorage)<br/>[DeleteAzureDataLakeStorage](PROCESSORS.md#del [...]
| CivetWeb | [ListenHTTP](PROCESSORS.md#listenhttp) [...]
diff --git a/extensions/libarchive/BinFiles.cpp b/extensions/libarchive/BinFiles.cpp
index 5d3322291..b4e0c8df4 100644
--- a/extensions/libarchive/BinFiles.cpp
+++ b/extensions/libarchive/BinFiles.cpp
@@ -317,6 +317,4 @@ void BinFiles::restore(const std::shared_ptr<core::FlowFile>& flowFile) {
file_store_.put(flowFile);
}
-REGISTER_RESOURCE(BinFiles, Processor);
-
} // namespace org::apache::nifi::minifi::processors
(nifi-minifi-cpp) 02/05: MINIFICPP-2732 Clear class descriptions when extensions are cleared
Posted by fg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
fgerlits pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 162ccfb558abc3c3a97326daa5da010d28977da2
Author: Gabor Gyimesi <ga...@gmail.com>
AuthorDate: Thu Feb 26 14:21:36 2026 +0100
MINIFICPP-2732 Clear class descriptions when extensions are cleared
Signed-off-by: Ferenc Gerlits <fg...@gmail.com>
Closes #2121
---
libminifi/include/core/extension/ExtensionManager.h | 6 ++++++
libminifi/src/agent/agent_docs.cpp | 3 +++
libminifi/src/core/extension/ExtensionManager.cpp | 8 ++++++++
minifi-api/include/minifi-cpp/agent/agent_docs.h | 1 +
4 files changed, 18 insertions(+)
diff --git a/libminifi/include/core/extension/ExtensionManager.h b/libminifi/include/core/extension/ExtensionManager.h
index ded6b4173..60af1229b 100644
--- a/libminifi/include/core/extension/ExtensionManager.h
+++ b/libminifi/include/core/extension/ExtensionManager.h
@@ -30,6 +30,12 @@ namespace org::apache::nifi::minifi::core::extension {
class ExtensionManager {
public:
explicit ExtensionManager(const std::shared_ptr<Configure>& config);
+ ~ExtensionManager();
+
+ ExtensionManager(const ExtensionManager&) = delete;
+ ExtensionManager& operator=(const ExtensionManager&) = delete;
+ ExtensionManager(ExtensionManager&&) = delete;
+ ExtensionManager& operator=(ExtensionManager&&) = delete;
private:
std::vector<std::unique_ptr<Extension>> extensions_;
diff --git a/libminifi/src/agent/agent_docs.cpp b/libminifi/src/agent/agent_docs.cpp
index 0aeb58487..057e9188b 100644
--- a/libminifi/src/agent/agent_docs.cpp
+++ b/libminifi/src/agent/agent_docs.cpp
@@ -34,5 +34,8 @@ std::map<BundleIdentifier, Components>& ClassDescriptionRegistry::getMutableClas
return getAgentDocsClassMappings();
}
+void ClassDescriptionRegistry::clearClassDescriptions() {
+ getAgentDocsClassMappings().clear();
+}
} // namespace org::apache::nifi::minifi
diff --git a/libminifi/src/core/extension/ExtensionManager.cpp b/libminifi/src/core/extension/ExtensionManager.cpp
index 0d60e7c6b..a8ef38caa 100644
--- a/libminifi/src/core/extension/ExtensionManager.cpp
+++ b/libminifi/src/core/extension/ExtensionManager.cpp
@@ -21,6 +21,7 @@
#include "core/logging/LoggerConfiguration.h"
#include "core/extension/Extension.h"
+#include "minifi-cpp/agent/agent_docs.h"
#include "utils/file/FilePattern.h"
#include "minifi-cpp/agent/agent_version.h"
#include "core/extension/Utils.h"
@@ -86,4 +87,11 @@ ExtensionManager::ExtensionManager(const std::shared_ptr<Configure>& config): lo
}
}
+ExtensionManager::~ExtensionManager() {
+ // Clear the class description registry to avoid dangling pointers
+ // to validator objects that live in extension DLLs to be unloaded.
+ minifi::ClassDescriptionRegistry::clearClassDescriptions();
+ extensions_.clear();
+}
+
} // namespace org::apache::nifi::minifi::core::extension
diff --git a/minifi-api/include/minifi-cpp/agent/agent_docs.h b/minifi-api/include/minifi-cpp/agent/agent_docs.h
index 912377d39..82e84f876 100644
--- a/minifi-api/include/minifi-cpp/agent/agent_docs.h
+++ b/minifi-api/include/minifi-cpp/agent/agent_docs.h
@@ -73,6 +73,7 @@ class ClassDescriptionRegistry {
public:
static const std::map<minifi::BundleIdentifier, Components>& getClassDescriptions();
static std::map<minifi::BundleIdentifier, Components>& getMutableClassDescriptions();
+ static void clearClassDescriptions();
template<typename Class, ResourceType Type>
static void createClassDescription(std::string bundle_name, std::string class_name, std::string version);