You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ma...@apache.org on 2024/06/17 13:29:07 UTC

(nifi-minifi-cpp) branch main updated (685bcee0f -> 29bdc8ea2)

This is an automated email from the ASF dual-hosted git repository.

martinzink pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git


    from 685bcee0f MINIFICPP-2392 Remove duplicate controller properties from encrypt-config
     new 2b9ec5e87 MINIFICPP-2403 - Refactor c2 server url
     new 1608b190a MINIFICPP-2397 Add option for non-sync writes in RocksDB content repo
     new 669e030b8 MINIFICPP-2375 Do not throw in lua ExecuteScript if onTrigger is missing
     new 29bdc8ea2 MINIFICPP-2351 Fix flakiness in MultipartUploadStateStorageTest

The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 C2.md                                              |  2 +-
 CONFIGURE.md                                       |  7 +++
 conf/minifi.properties                             |  3 ++
 encrypt-config/tests/resources/minifi.properties   |  6 +--
 ...th-additional-sensitive-props.minifi.properties |  6 +--
 extensions/aws/s3/MultipartUploadStateStorage.cpp  |  9 ++--
 extensions/aws/s3/MultipartUploadStateStorage.h    |  4 +-
 .../aws/tests/MultipartUploadStateStorageTest.cpp  | 63 +++++-----------------
 extensions/lua/LuaScriptEngine.h                   |  5 +-
 .../TestExecuteScriptProcessorWithLuaScript.cpp    | 10 ++++
 .../TestExecuteScriptProcessorWithPythonScript.cpp | 10 ++++
 .../rocksdb-repos/DatabaseContentRepository.cpp    | 10 ++--
 .../rocksdb-repos/DatabaseContentRepository.h      |  5 +-
 extensions/rocksdb-repos/RocksDbStream.cpp         |  7 +--
 extensions/rocksdb-repos/RocksDbStream.h           |  4 +-
 libminifi/include/properties/Configuration.h       |  1 +
 libminifi/src/Configuration.cpp                    |  1 +
 .../test/resources/encrypted.minifi.properties     |  6 +--
 18 files changed, 80 insertions(+), 79 deletions(-)


(nifi-minifi-cpp) 04/04: MINIFICPP-2351 Fix flakiness in MultipartUploadStateStorageTest

Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

martinzink pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git

commit 29bdc8ea2ce584c3179ab0ee07ed4424559eb66e
Author: Ferenc Gerlits <fg...@gmail.com>
AuthorDate: Mon Jun 17 12:03:27 2024 +0200

    MINIFICPP-2351 Fix flakiness in MultipartUploadStateStorageTest
    
    Closes #1798
    
    Signed-off-by: Martin Zink <ma...@apache.org>
---
 extensions/aws/s3/MultipartUploadStateStorage.cpp  |  9 ++--
 extensions/aws/s3/MultipartUploadStateStorage.h    |  4 +-
 .../aws/tests/MultipartUploadStateStorageTest.cpp  | 63 +++++-----------------
 3 files changed, 17 insertions(+), 59 deletions(-)

diff --git a/extensions/aws/s3/MultipartUploadStateStorage.cpp b/extensions/aws/s3/MultipartUploadStateStorage.cpp
index 699876a4e..aa9c49e4e 100644
--- a/extensions/aws/s3/MultipartUploadStateStorage.cpp
+++ b/extensions/aws/s3/MultipartUploadStateStorage.cpp
@@ -26,7 +26,7 @@ void MultipartUploadStateStorage::storeState(const std::string& bucket, const st
   state_manager_->get(stored_state);
   std::string state_key = bucket + "/" + key;
   stored_state[state_key + ".upload_id"] = state.upload_id;
-  stored_state[state_key + ".upload_time"] = std::to_string(state.upload_time.Millis());
+  stored_state[state_key + ".upload_time"] = std::to_string(state.upload_time_ms_since_epoch);
   stored_state[state_key + ".uploaded_parts"] = std::to_string(state.uploaded_parts);
   stored_state[state_key + ".uploaded_size"] = std::to_string(state.uploaded_size);
   stored_state[state_key + ".part_size"] = std::to_string(state.part_size);
@@ -56,10 +56,7 @@ std::optional<MultipartUploadState> MultipartUploadStateStorage::getState(const
   MultipartUploadState state;
   state.upload_id = state_map[state_key + ".upload_id"];
 
-  int64_t stored_upload_time = 0;
-  core::Property::StringToInt(state_map[state_key + ".upload_time"], stored_upload_time);
-  state.upload_time = Aws::Utils::DateTime(stored_upload_time);
-
+  core::Property::StringToInt(state_map[state_key + ".upload_time"], state.upload_time_ms_since_epoch);
   core::Property::StringToInt(state_map[state_key + ".uploaded_parts"], state.uploaded_parts);
   core::Property::StringToInt(state_map[state_key + ".uploaded_size"], state.uploaded_size);
   core::Property::StringToInt(state_map[state_key + ".part_size"], state.part_size);
@@ -108,7 +105,7 @@ void MultipartUploadStateStorage::removeAgedStates(std::chrono::milliseconds mul
 
   std::vector<std::string> keys_to_remove;
   for (const auto& [property_key, value] : state_map) {
-    std::string upload_time_suffix = ".upload_time";
+    static constexpr std::string_view upload_time_suffix = ".upload_time";
     if (!minifi::utils::string::endsWith(property_key, upload_time_suffix)) {
       continue;
     }
diff --git a/extensions/aws/s3/MultipartUploadStateStorage.h b/extensions/aws/s3/MultipartUploadStateStorage.h
index 91a9d2640..2da6df063 100644
--- a/extensions/aws/s3/MultipartUploadStateStorage.h
+++ b/extensions/aws/s3/MultipartUploadStateStorage.h
@@ -41,13 +41,13 @@ struct MultipartUploadState {
     : upload_id(upload_id),
       part_size(part_size),
       full_size(full_size),
-      upload_time(upload_time) {}
+      upload_time_ms_since_epoch(upload_time.Millis()) {}
   std::string upload_id;
   size_t uploaded_parts{};
   uint64_t uploaded_size{};
   uint64_t part_size{};
   uint64_t full_size{};
-  Aws::Utils::DateTime upload_time;
+  int64_t upload_time_ms_since_epoch;
   std::vector<std::string> uploaded_etags;
 
   bool operator==(const MultipartUploadState&) const = default;
diff --git a/extensions/aws/tests/MultipartUploadStateStorageTest.cpp b/extensions/aws/tests/MultipartUploadStateStorageTest.cpp
index 3f2d01049..29458b230 100644
--- a/extensions/aws/tests/MultipartUploadStateStorageTest.cpp
+++ b/extensions/aws/tests/MultipartUploadStateStorageTest.cpp
@@ -44,53 +44,33 @@ class MultipartUploadStateStorageTestFixture {
 
 TEST_CASE_METHOD(MultipartUploadStateStorageTestFixture, "Store and get current key state", "[s3StateStorage]") {
   REQUIRE(upload_storage_->getState("test_bucket", "key") == std::nullopt);
-  minifi::aws::s3::MultipartUploadState state;
-  state.upload_id = "id1";
+  minifi::aws::s3::MultipartUploadState state("id1", 50_MiB, 200_MiB, Aws::Utils::DateTime::Now());
   state.uploaded_parts = 2;
   state.uploaded_size = 100_MiB;
-  state.part_size = 50_MiB;
-  state.full_size = 200_MiB;
-  state.upload_time = Aws::Utils::DateTime::CurrentTimeMillis();
   state.uploaded_etags = {"etag1", "etag2"};
   upload_storage_->storeState("test_bucket", "key", state);
   REQUIRE(*upload_storage_->getState("test_bucket", "key") == state);
 }
 
 TEST_CASE_METHOD(MultipartUploadStateStorageTestFixture, "Get key upload state from multiple keys and buckets", "[s3StateStorage]") {
-  minifi::aws::s3::MultipartUploadState state1;
-  state1.upload_id = "id1";
+  minifi::aws::s3::MultipartUploadState state1("id1", 50_MiB, 200_MiB, Aws::Utils::DateTime::Now());
   state1.uploaded_parts = 3;
   state1.uploaded_size = 150_MiB;
-  state1.part_size = 50_MiB;
-  state1.full_size = 200_MiB;
-  state1.upload_time = Aws::Utils::DateTime::CurrentTimeMillis();
   state1.uploaded_etags = {"etag1", "etag2", "etag3"};
   upload_storage_->storeState("old_bucket", "key1", state1);
-  minifi::aws::s3::MultipartUploadState state2;
-  state2.upload_id = "id2";
+  minifi::aws::s3::MultipartUploadState state2("id2", 50_MiB, 200_MiB, Aws::Utils::DateTime::Now());
   state2.uploaded_parts = 2;
   state2.uploaded_size = 100_MiB;
-  state2.part_size = 50_MiB;
-  state2.full_size = 200_MiB;
-  state2.upload_time = Aws::Utils::DateTime::CurrentTimeMillis();
   state2.uploaded_etags = {"etag4", "etag5"};
   upload_storage_->storeState("test_bucket", "key1", state2);
-  minifi::aws::s3::MultipartUploadState state3;
-  state3.upload_id = "id3";
+  minifi::aws::s3::MultipartUploadState state3("id3", 50_MiB, 400_MiB, Aws::Utils::DateTime::Now());
   state3.uploaded_parts = 4;
   state3.uploaded_size = 200_MiB;
-  state3.part_size = 50_MiB;
-  state3.full_size = 400_MiB;
-  state3.upload_time = Aws::Utils::DateTime::CurrentTimeMillis();
   state3.uploaded_etags = {"etag6", "etag7", "etag9", "etag8"};
   upload_storage_->storeState("test_bucket", "key2", state3);
-  minifi::aws::s3::MultipartUploadState state4;
-  state2.upload_id = "id2";
+  minifi::aws::s3::MultipartUploadState state4("id2", 50_MiB, 200_MiB, Aws::Utils::DateTime::Now());
   state2.uploaded_parts = 3;
   state2.uploaded_size = 150_MiB;
-  state2.part_size = 50_MiB;
-  state2.full_size = 200_MiB;
-  state2.upload_time = Aws::Utils::DateTime::CurrentTimeMillis();
   state2.uploaded_etags = {"etag4", "etag5", "etag10"};
   upload_storage_->storeState("test_bucket", "key1", state4);
   REQUIRE(*upload_storage_->getState("test_bucket", "key1") == state4);
@@ -99,31 +79,19 @@ TEST_CASE_METHOD(MultipartUploadStateStorageTestFixture, "Get key upload state f
 }
 
 TEST_CASE_METHOD(MultipartUploadStateStorageTestFixture, "Remove state", "[s3StateStorage]") {
-  minifi::aws::s3::MultipartUploadState state1;
-  state1.upload_id = "id1";
+  minifi::aws::s3::MultipartUploadState state1("id1", 50_MiB, 200_MiB, Aws::Utils::DateTime::Now());
   state1.uploaded_parts = 3;
   state1.uploaded_size = 150_MiB;
-  state1.part_size = 50_MiB;
-  state1.full_size = 200_MiB;
-  state1.upload_time = Aws::Utils::DateTime::CurrentTimeMillis();
   state1.uploaded_etags = {"etag1", "etag2", "etag3"};
   upload_storage_->storeState("old_bucket", "key1", state1);
-  minifi::aws::s3::MultipartUploadState state2;
-  state2.upload_id = "id2";
+  minifi::aws::s3::MultipartUploadState state2("id2", 50_MiB, 200_MiB, Aws::Utils::DateTime::Now());
   state2.uploaded_parts = 2;
   state2.uploaded_size = 100_MiB;
-  state2.part_size = 50_MiB;
-  state2.full_size = 200_MiB;
-  state2.upload_time = Aws::Utils::DateTime::CurrentTimeMillis();
   state2.uploaded_etags = {"etag4", "etag5"};
   upload_storage_->storeState("test_bucket", "key1", state2);
-  minifi::aws::s3::MultipartUploadState state3;
-  state3.upload_id = "id3";
+  minifi::aws::s3::MultipartUploadState state3("id3", 50_MiB, 400_MiB, Aws::Utils::DateTime::Now());
   state3.uploaded_parts = 4;
   state3.uploaded_size = 200_MiB;
-  state3.part_size = 50_MiB;
-  state3.full_size = 400_MiB;
-  state3.upload_time = Aws::Utils::DateTime::CurrentTimeMillis();
   state3.uploaded_etags = {"etag6", "etag7", "etag9", "etag8"};
   upload_storage_->storeState("test_bucket", "key2", state3);
   REQUIRE(*upload_storage_->getState("old_bucket", "key1") == state1);
@@ -136,25 +104,18 @@ TEST_CASE_METHOD(MultipartUploadStateStorageTestFixture, "Remove state", "[s3Sta
 }
 
 TEST_CASE_METHOD(MultipartUploadStateStorageTestFixture, "Remove aged off state", "[s3StateStorage]") {
-  minifi::aws::s3::MultipartUploadState state1;
-  state1.upload_id = "id1";
+  using namespace std::literals::chrono_literals;
+  minifi::aws::s3::MultipartUploadState state1("id1", 50_MiB, 200_MiB, Aws::Utils::DateTime::Now() - 10min);
   state1.uploaded_parts = 3;
   state1.uploaded_size = 150_MiB;
-  state1.part_size = 50_MiB;
-  state1.full_size = 200_MiB;
-  state1.upload_time = Aws::Utils::DateTime(Aws::Utils::DateTime::CurrentTimeMillis()) - std::chrono::minutes(10);
   state1.uploaded_etags = {"etag1", "etag2", "etag3"};
   upload_storage_->storeState("test_bucket", "key1", state1);
-  minifi::aws::s3::MultipartUploadState state2;
-  state2.upload_id = "id2";
+  minifi::aws::s3::MultipartUploadState state2("id2", 50_MiB, 200_MiB, Aws::Utils::DateTime::Now());
   state2.uploaded_parts = 2;
   state2.uploaded_size = 100_MiB;
-  state2.part_size = 50_MiB;
-  state2.full_size = 200_MiB;
-  state2.upload_time = Aws::Utils::DateTime::CurrentTimeMillis();
   state2.uploaded_etags = {"etag4", "etag5"};
   upload_storage_->storeState("test_bucket", "key2", state2);
-  upload_storage_->removeAgedStates(std::chrono::milliseconds(10));
+  upload_storage_->removeAgedStates(10min);
   REQUIRE(upload_storage_->getState("test_bucket", "key1") == std::nullopt);
   REQUIRE(upload_storage_->getState("test_bucket", "key2") == state2);
 }


(nifi-minifi-cpp) 03/04: MINIFICPP-2375 Do not throw in lua ExecuteScript if onTrigger is missing

Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

martinzink pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git

commit 669e030b8a116bd7ce4d8999247d05d003e3821a
Author: Ferenc Gerlits <fg...@gmail.com>
AuthorDate: Mon Jun 17 12:03:19 2024 +0200

    MINIFICPP-2375 Do not throw in lua ExecuteScript if onTrigger is missing
    
    Closes #1808
    
    Signed-off-by: Martin Zink <ma...@apache.org>
---
 extensions/lua/LuaScriptEngine.h                               |  5 +++--
 .../lua/tests/TestExecuteScriptProcessorWithLuaScript.cpp      | 10 ++++++++++
 .../tests/TestExecuteScriptProcessorWithPythonScript.cpp       | 10 ++++++++++
 3 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/extensions/lua/LuaScriptEngine.h b/extensions/lua/LuaScriptEngine.h
index c0f099b8c..4dcc6d2cd 100644
--- a/extensions/lua/LuaScriptEngine.h
+++ b/extensions/lua/LuaScriptEngine.h
@@ -52,8 +52,9 @@ class LuaScriptEngine {
   void call(const std::string& fn_name, Args&& ...args) {
     sol::protected_function_result function_result{};
     try {
-      sol::protected_function fn = lua_[fn_name.c_str()];
-      function_result = fn(convert(args)...);
+      if (sol::protected_function fn = lua_[fn_name.c_str()]) {
+        function_result = fn(convert(args)...);
+      }
     } catch (const std::exception& e) {
       throw LuaScriptException(e.what());
     }
diff --git a/extensions/lua/tests/TestExecuteScriptProcessorWithLuaScript.cpp b/extensions/lua/tests/TestExecuteScriptProcessorWithLuaScript.cpp
index d680b6442..88619b7af 100644
--- a/extensions/lua/tests/TestExecuteScriptProcessorWithLuaScript.cpp
+++ b/extensions/lua/tests/TestExecuteScriptProcessorWithLuaScript.cpp
@@ -30,6 +30,16 @@
 
 namespace org::apache::nifi::minifi::processors::test {
 
+TEST_CASE("Lua: hello world") {
+  const auto execute_script = std::make_shared<ExecuteScript>("ExecuteScript");
+  minifi::test::SingleProcessorTestController controller{execute_script};
+
+  execute_script->setProperty(ExecuteScript::ScriptEngine, "lua");
+  execute_script->setProperty(ExecuteScript::ScriptBody, R"(print("Hello world!"))");
+
+  CHECK_NOTHROW(controller.trigger());
+}
+
 TEST_CASE("Script engine is not set", "[executescriptMisconfiguration]") {
   TestController test_controller;
   auto plan = test_controller.createPlan();
diff --git a/extensions/python/tests/TestExecuteScriptProcessorWithPythonScript.cpp b/extensions/python/tests/TestExecuteScriptProcessorWithPythonScript.cpp
index e49af68be..8e0c3b456 100644
--- a/extensions/python/tests/TestExecuteScriptProcessorWithPythonScript.cpp
+++ b/extensions/python/tests/TestExecuteScriptProcessorWithPythonScript.cpp
@@ -29,6 +29,16 @@
 
 namespace org::apache::nifi::minifi::processors::test {
 
+TEST_CASE("Python: hello world") {
+  const auto execute_script = std::make_shared<ExecuteScript>("ExecuteScript");
+  minifi::test::SingleProcessorTestController controller{execute_script};
+
+  execute_script->setProperty(ExecuteScript::ScriptEngine, "python");
+  execute_script->setProperty(ExecuteScript::ScriptBody, R"(print("Hello world!"))");
+
+  CHECK_NOTHROW(controller.trigger());
+}
+
 TEST_CASE("Script engine is not set", "[executescriptMisconfiguration]") {
   TestController test_controller;
   auto plan = test_controller.createPlan();


(nifi-minifi-cpp) 02/04: MINIFICPP-2397 Add option for non-sync writes in RocksDB content repo

Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

martinzink pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git

commit 1608b190ab9ea218198c76fe768909964c4d4d6f
Author: Gabor Gyimesi <ga...@gmail.com>
AuthorDate: Mon Jun 17 12:03:11 2024 +0200

    MINIFICPP-2397 Add option for non-sync writes in RocksDB content repo
    
    Closes #1811
    
    Signed-off-by: Martin Zink <ma...@apache.org>
---
 CONFIGURE.md                                           |  7 +++++++
 conf/minifi.properties                                 |  3 +++
 extensions/rocksdb-repos/DatabaseContentRepository.cpp | 10 +++++++---
 extensions/rocksdb-repos/DatabaseContentRepository.h   |  5 ++++-
 extensions/rocksdb-repos/RocksDbStream.cpp             |  7 ++++---
 extensions/rocksdb-repos/RocksDbStream.h               |  4 +++-
 libminifi/include/properties/Configuration.h           |  1 +
 libminifi/src/Configuration.cpp                        |  1 +
 8 files changed, 30 insertions(+), 8 deletions(-)

diff --git a/CONFIGURE.md b/CONFIGURE.md
index d057bc760..dcded1698 100644
--- a/CONFIGURE.md
+++ b/CONFIGURE.md
@@ -429,6 +429,13 @@ Rocksdb has an option to run compaction at specific intervals not just when need
      nifi.flowfile.repository.rocksdb.compaction.period=2 min
      nifi.database.content.repository.rocksdb.compaction.period=2 min
 
+### Configuring synchronous or asynchronous writes for RocksDB content repository
+
+RocksDB has an option to set synchronous writes for its database, ensuring that the write operation does not return until the data being written has been pushed all the way to persistent storage. In MiNiFi C++, this is set to true by default to avoid any data loss, as according to the RocksDB documentation using non-sync writes can result in data loss in the event of a host machine crash. You can read more information about this option on the [RocksDB wiki page](https://github.com/facebo [...]
+
+    # in minifi.properties
+    nifi.content.repository.rocksdb.use.synchronous.writes=true
+
 ### Global RocksDB options
 
 There are a few options for RocksDB that are set for all used RocksDB databases in MiNiFi:
diff --git a/conf/minifi.properties b/conf/minifi.properties
index 67de200a3..e24f3706e 100644
--- a/conf/minifi.properties
+++ b/conf/minifi.properties
@@ -35,6 +35,9 @@ nifi.provenance.repository.class.name=NoOpRepository
 nifi.content.repository.class.name=DatabaseContentRepository
 # nifi.content.repository.rocksdb.compression=auto
 
+# Use synchronous writes for the RocksDB content repository. Disable for better write performance, if data loss is acceptable in case of the host crashing.
+# nifi.content.repository.rocksdb.use.synchronous.writes=true
+
 ## Relates to the internal workings of the rocksdb backend
 # nifi.flowfile.repository.rocksdb.compaction.period=2 min
 # nifi.database.content.repository.rocksdb.compaction.period=2 min
diff --git a/extensions/rocksdb-repos/DatabaseContentRepository.cpp b/extensions/rocksdb-repos/DatabaseContentRepository.cpp
index b1cd9859c..7f22d9e73 100644
--- a/extensions/rocksdb-repos/DatabaseContentRepository.cpp
+++ b/extensions/rocksdb-repos/DatabaseContentRepository.cpp
@@ -80,6 +80,8 @@ bool DatabaseContentRepository::initialize(const std::shared_ptr<minifi::Configu
     logger_->log_error("NiFi Content DB Repository database open {} fail", directory_);
     is_valid_ = false;
   }
+
+  use_synchronous_writes_ = configuration->get(Configure::nifi_content_repository_rocksdb_use_synchronous_writes).value_or("true") != "false";
   return is_valid_;
 }
 
@@ -137,10 +139,12 @@ void DatabaseContentRepository::stop() {
   }
 }
 
-DatabaseContentRepository::Session::Session(std::shared_ptr<ContentRepository> repository) : BufferedContentSession(std::move(repository)) {}
+DatabaseContentRepository::Session::Session(std::shared_ptr<ContentRepository> repository, bool use_synchronous_writes)
+    : BufferedContentSession(std::move(repository)),
+      use_synchronous_writes_(use_synchronous_writes) {}
 
 std::shared_ptr<ContentSession> DatabaseContentRepository::createSession() {
-  return std::make_shared<Session>(sharedFromThis());
+  return std::make_shared<Session>(sharedFromThis(), use_synchronous_writes_);
 }
 
 void DatabaseContentRepository::Session::commit() {
@@ -172,7 +176,7 @@ void DatabaseContentRepository::Session::commit() {
   }
 
   rocksdb::WriteOptions options;
-  options.sync = true;
+  options.sync = use_synchronous_writes_;
   rocksdb::Status status = opendb->Write(options, &batch);
   if (!status.ok()) {
     throw Exception(REPOSITORY_EXCEPTION, "Batch write failed: " + status.ToString());
diff --git a/extensions/rocksdb-repos/DatabaseContentRepository.h b/extensions/rocksdb-repos/DatabaseContentRepository.h
index 46641e7f1..f8f5f48cb 100644
--- a/extensions/rocksdb-repos/DatabaseContentRepository.h
+++ b/extensions/rocksdb-repos/DatabaseContentRepository.h
@@ -36,9 +36,11 @@ namespace org::apache::nifi::minifi::core::repository {
 class DatabaseContentRepository : public core::ContentRepository {
   class Session : public BufferedContentSession {
    public:
-    explicit Session(std::shared_ptr<ContentRepository> repository);
+    explicit Session(std::shared_ptr<ContentRepository> repository, bool use_synchronous_writes);
 
     void commit() override;
+   private:
+    bool use_synchronous_writes_;
   };
 
   static constexpr std::chrono::milliseconds DEFAULT_COMPACTION_PERIOD = std::chrono::minutes{2};
@@ -103,6 +105,7 @@ class DatabaseContentRepository : public core::ContentRepository {
   std::mutex keys_mtx_;
   std::vector<std::string> keys_to_delete_;
   std::unique_ptr<utils::StoppableThread> gc_thread_;
+  bool use_synchronous_writes_ = true;
 };
 
 }  // namespace org::apache::nifi::minifi::core::repository
diff --git a/extensions/rocksdb-repos/RocksDbStream.cpp b/extensions/rocksdb-repos/RocksDbStream.cpp
index 2dfaa0e4f..1ba4cdec9 100644
--- a/extensions/rocksdb-repos/RocksDbStream.cpp
+++ b/extensions/rocksdb-repos/RocksDbStream.cpp
@@ -28,7 +28,7 @@
 
 namespace org::apache::nifi::minifi::io {
 
-RocksDbStream::RocksDbStream(std::string path, gsl::not_null<minifi::internal::RocksDatabase*> db, bool write_enable, minifi::internal::WriteBatch* batch)
+RocksDbStream::RocksDbStream(std::string path, gsl::not_null<minifi::internal::RocksDatabase*> db, bool write_enable, minifi::internal::WriteBatch* batch, bool use_synchronous_writes)
     : BaseStream(),
       path_(std::move(path)),
       write_enable_(write_enable),
@@ -39,7 +39,8 @@ RocksDbStream::RocksDbStream(std::string path, gsl::not_null<minifi::internal::R
       }()),
       offset_(0),
       batch_(batch),
-      size_(value_.size()) {
+      size_(value_.size()),
+      use_synchronous_writes_(use_synchronous_writes) {
 }
 
 void RocksDbStream::close() {
@@ -67,7 +68,7 @@ size_t RocksDbStream::write(const uint8_t *value, size_t size) {
     status = batch_->Merge(path_, slice_value);
   } else {
     rocksdb::WriteOptions opts;
-    opts.sync = true;
+    opts.sync = use_synchronous_writes_;
     status = opendb->Merge(opts, path_, slice_value);
   }
   if (status.ok()) {
diff --git a/extensions/rocksdb-repos/RocksDbStream.h b/extensions/rocksdb-repos/RocksDbStream.h
index 42a61c526..ec903ee4f 100644
--- a/extensions/rocksdb-repos/RocksDbStream.h
+++ b/extensions/rocksdb-repos/RocksDbStream.h
@@ -44,7 +44,8 @@ class RocksDbStream : public io::BaseStream {
    * File Stream constructor that accepts an fstream shared pointer.
    * It must already be initialized for read and write.
    */
-  explicit RocksDbStream(std::string path, gsl::not_null<minifi::internal::RocksDatabase*> db, bool write_enable = false, minifi::internal::WriteBatch* batch = nullptr);
+  explicit RocksDbStream(std::string path, gsl::not_null<minifi::internal::RocksDatabase*> db, bool write_enable = false,
+    minifi::internal::WriteBatch* batch = nullptr, bool use_synchronous_writes = true);
 
   ~RocksDbStream() override {
     close();
@@ -89,6 +90,7 @@ class RocksDbStream : public io::BaseStream {
   size_t offset_;
   minifi::internal::WriteBatch* batch_;
   size_t size_;
+  bool use_synchronous_writes_;
 
  private:
   std::shared_ptr<core::logging::Logger> logger_ = core::logging::LoggerFactory<RocksDbStream>::getLogger();
diff --git a/libminifi/include/properties/Configuration.h b/libminifi/include/properties/Configuration.h
index cec669155..79ef4c53c 100644
--- a/libminifi/include/properties/Configuration.h
+++ b/libminifi/include/properties/Configuration.h
@@ -77,6 +77,7 @@ class Configuration : public Properties {
   static constexpr const char *nifi_flowfile_repository_rocksdb_compaction_period = "nifi.flowfile.repository.rocksdb.compaction.period";
   static constexpr const char *nifi_dbcontent_repository_rocksdb_compaction_period = "nifi.database.content.repository.rocksdb.compaction.period";
   static constexpr const char *nifi_dbcontent_repository_purge_period = "nifi.database.content.repository.purge.period";
+  static constexpr const char *nifi_content_repository_rocksdb_use_synchronous_writes = "nifi.content.repository.rocksdb.use.synchronous.writes";
 
   static constexpr const char *nifi_remote_input_secure = "nifi.remote.input.secure";
   static constexpr const char *nifi_security_need_ClientAuth = "nifi.security.need.ClientAuth";
diff --git a/libminifi/src/Configuration.cpp b/libminifi/src/Configuration.cpp
index 7a4028eb0..3cf4e06c2 100644
--- a/libminifi/src/Configuration.cpp
+++ b/libminifi/src/Configuration.cpp
@@ -53,6 +53,7 @@ const std::unordered_map<std::string_view, gsl::not_null<const core::PropertyVal
   {Configuration::nifi_dbcontent_repository_directory_default, gsl::make_not_null(&core::StandardPropertyTypes::VALID_TYPE)},
   {Configuration::nifi_flowfile_repository_rocksdb_compaction_period, gsl::make_not_null(&core::StandardPropertyTypes::TIME_PERIOD_TYPE)},
   {Configuration::nifi_dbcontent_repository_rocksdb_compaction_period, gsl::make_not_null(&core::StandardPropertyTypes::TIME_PERIOD_TYPE)},
+  {Configuration::nifi_content_repository_rocksdb_use_synchronous_writes, gsl::make_not_null(&core::StandardPropertyTypes::BOOLEAN_TYPE)},
   {Configuration::nifi_dbcontent_repository_purge_period, gsl::make_not_null(&core::StandardPropertyTypes::TIME_PERIOD_TYPE)},
   {Configuration::nifi_remote_input_secure, gsl::make_not_null(&core::StandardPropertyTypes::BOOLEAN_TYPE)},
   {Configuration::nifi_security_need_ClientAuth, gsl::make_not_null(&core::StandardPropertyTypes::BOOLEAN_TYPE)},


(nifi-minifi-cpp) 01/04: MINIFICPP-2403 - Refactor c2 server url

Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

martinzink pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git

commit 2b9ec5e874f880f7f67ec985f09bf4b9d7b620d3
Author: Adam Debreceni <ad...@apache.org>
AuthorDate: Mon Jun 17 12:02:30 2024 +0200

    MINIFICPP-2403 - Refactor c2 server url
    
    Closes #1816
    
    Signed-off-by: Martin Zink <ma...@apache.org>
---
 C2.md                                                               | 2 +-
 encrypt-config/tests/resources/minifi.properties                    | 6 +++---
 .../resources/with-additional-sensitive-props.minifi.properties     | 6 +++---
 libminifi/test/resources/encrypted.minifi.properties                | 6 +++---
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/C2.md b/C2.md
index c0690dd0c..27165a41b 100644
--- a/C2.md
+++ b/C2.md
@@ -113,7 +113,7 @@ Flow id and URL are usually retrieved from the C2 server. These identify the las
 
     # in minifi.properties
     nifi.c2.flow.id=8da5de7f-dcdb-4f6b-aa2f-6f162a7f9dc4
-    nifi.c2.flow.url=http://localhost:10090/efm/api/flows/8da5de7f-dcdb-4f6b-aa2f-6f162a7f9dc4/content?aid=efmtest
+    nifi.c2.flow.url=http://localhost:10090/c2-server/api/flows/8da5de7f-dcdb-4f6b-aa2f-6f162a7f9dc4/content?aid=efmtest
 
 #### Agent Identifier Fallback
 
diff --git a/encrypt-config/tests/resources/minifi.properties b/encrypt-config/tests/resources/minifi.properties
index 824f55a50..7df24ba7f 100644
--- a/encrypt-config/tests/resources/minifi.properties
+++ b/encrypt-config/tests/resources/minifi.properties
@@ -59,9 +59,9 @@ nifi.c2.agent.protocol.class=RESTSender
 #nifi.c2.agent.coap.port=
 ## base URL of the c2 server,
 ## very likely the same base url of rest urls
-nifi.c2.flow.base.url=http://localhost:10080/efm/api
-nifi.c2.rest.url=http://localhost:10080/efm/api/c2-protocol/heartbeat
-nifi.c2.rest.url.ack=http://localhost:10080/efm/api/c2-protocol/acknowledge
+nifi.c2.flow.base.url=http://localhost:10080/c2-server/api
+nifi.c2.rest.url=http://localhost:10080/c2-server/api/c2-protocol/heartbeat
+nifi.c2.rest.url.ack=http://localhost:10080/c2-server/api/c2-protocol/acknowledge
 nifi.c2.root.classes=DeviceInfoNode,AgentInformation,FlowInformation
 ## Minimize heartbeat payload size by excluding agent manifest from the heartbeat
 #nifi.c2.full.heartbeat=false
diff --git a/encrypt-config/tests/resources/with-additional-sensitive-props.minifi.properties b/encrypt-config/tests/resources/with-additional-sensitive-props.minifi.properties
index a9739fe57..60e6ff61c 100644
--- a/encrypt-config/tests/resources/with-additional-sensitive-props.minifi.properties
+++ b/encrypt-config/tests/resources/with-additional-sensitive-props.minifi.properties
@@ -61,9 +61,9 @@ nifi.c2.agent.protocol.class=RESTSender
 #nifi.c2.agent.coap.port=
 ## base URL of the c2 server,
 ## very likely the same base url of rest urls
-nifi.c2.flow.base.url=http://localhost:10080/efm/api
-nifi.c2.rest.url=http://localhost:10080/efm/api/c2-protocol/heartbeat
-nifi.c2.rest.url.ack=http://localhost:10080/efm/api/c2-protocol/acknowledge
+nifi.c2.flow.base.url=http://localhost:10080/c2-server/api
+nifi.c2.rest.url=http://localhost:10080/c2-server/api/c2-protocol/heartbeat
+nifi.c2.rest.url.ack=http://localhost:10080/c2-server/api/c2-protocol/acknowledge
 nifi.c2.root.classes=DeviceInfoNode,AgentInformation,FlowInformation
 ## Minimize heartbeat payload size by excluding agent manifest from the heartbeat
 #nifi.c2.full.heartbeat=false
diff --git a/libminifi/test/resources/encrypted.minifi.properties b/libminifi/test/resources/encrypted.minifi.properties
index 2f79928f2..3a9b84452 100644
--- a/libminifi/test/resources/encrypted.minifi.properties
+++ b/libminifi/test/resources/encrypted.minifi.properties
@@ -61,9 +61,9 @@ nifi.c2.agent.protocol.class=RESTSender
 #nifi.c2.agent.coap.port=
 ## base URL of the c2 server,
 ## very likely the same base url of rest urls
-nifi.c2.flow.base.url=http://localhost:10080/efm/api
-nifi.c2.rest.url=http://localhost:10080/efm/api/c2-protocol/heartbeat
-nifi.c2.rest.url.ack=http://localhost:10080/efm/api/c2-protocol/acknowledge
+nifi.c2.flow.base.url=http://localhost:10080/c2-server/api
+nifi.c2.rest.url=http://localhost:10080/c2-server/api/c2-protocol/heartbeat
+nifi.c2.rest.url.ack=http://localhost:10080/c2-server/api/c2-protocol/acknowledge
 nifi.c2.root.classes=DeviceInfoNode,AgentInformation,FlowInformation
 ## Minimize heartbeat payload size by excluding agent manifest from the heartbeat
 #nifi.c2.full.heartbeat=false