You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ma...@apache.org on 2023/03/06 08:43:50 UTC

[nifi-minifi-cpp] branch main updated (a57e38c75 -> ab396b7aa)

This is an automated email from the ASF dual-hosted git repository.

martinzink pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git


    from a57e38c75 MINIFICPP-2065 Only accept configuration format the agent can handle
     new d12c354ea MINIFICPP-2056 CWEL: support UserData in json Closes #1515
     new ab396b7aa MINIFICPP-2066 Close handle when querying file owner on Windows Closes #1523

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 extensions/http-curl/tests/C2UpdateAssetTest.cpp   |  3 +
 .../tests/ConsumeWindowsEventLogTests.cpp          | 65 ++++++++++++++++++++++
 extensions/windows-event-log/wel/JSONUtils.cpp     | 53 ++++++++++++------
 libminifi/include/utils/file/FileUtils.h           |  2 +
 libminifi/test/TestBase.h                          |  4 +-
 .../keyvalue-tests/PersistentStateStorageTest.cpp  |  5 +-
 .../keyvalue-tests/VolatileMapStateStorageTest.cpp |  2 +
 libminifi/test/unit/FileStreamTests.cpp            |  1 +
 8 files changed, 114 insertions(+), 21 deletions(-)


[nifi-minifi-cpp] 02/02: MINIFICPP-2066 Close handle when querying file owner on Windows Closes #1523

Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

martinzink pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git

commit ab396b7aa67e60355c69017bdce6065c852c418b
Author: Gabor Gyimesi <ga...@gmail.com>
AuthorDate: Mon Mar 6 09:28:12 2023 +0100

    MINIFICPP-2066 Close handle when querying file owner on Windows
    Closes #1523
    
    Signed-off-by: Martin Zink <ma...@apache.org>
---
 extensions/http-curl/tests/C2UpdateAssetTest.cpp              | 3 +++
 libminifi/include/utils/file/FileUtils.h                      | 2 ++
 libminifi/test/TestBase.h                                     | 4 ++--
 libminifi/test/keyvalue-tests/PersistentStateStorageTest.cpp  | 5 +++--
 libminifi/test/keyvalue-tests/VolatileMapStateStorageTest.cpp | 2 ++
 libminifi/test/unit/FileStreamTests.cpp                       | 1 +
 6 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/extensions/http-curl/tests/C2UpdateAssetTest.cpp b/extensions/http-curl/tests/C2UpdateAssetTest.cpp
index 9c606c989..ac545d180 100644
--- a/extensions/http-curl/tests/C2UpdateAssetTest.cpp
+++ b/extensions/http-curl/tests/C2UpdateAssetTest.cpp
@@ -26,6 +26,7 @@
 #include "HTTPHandlers.h"
 #include "utils/IntegrationTestUtils.h"
 #include "utils/Environment.h"
+#include "utils/file/FileUtils.h"
 
 class FileProvider : public ServerAwareHandler {
  public:
@@ -257,4 +258,6 @@ int main() {
       assert(false);
     }
   }
+
+  std::filesystem::current_path(minifi::utils::file::get_executable_dir());
 }
diff --git a/libminifi/include/utils/file/FileUtils.h b/libminifi/include/utils/file/FileUtils.h
index e4fb301d5..db37d861b 100644
--- a/libminifi/include/utils/file/FileUtils.h
+++ b/libminifi/include/utils/file/FileUtils.h
@@ -480,6 +480,8 @@ inline std::optional<std::string> get_file_owner(const std::filesystem::path& fi
     return std::nullopt;
   }
 
+  auto close_file_handle = gsl::finally([&file_handle] { CloseHandle(file_handle); });
+
   // Get the owner SID of the file.
   return_code = GetSecurityInfo(
     file_handle,
diff --git a/libminifi/test/TestBase.h b/libminifi/test/TestBase.h
index e2118144e..57d9bb159 100644
--- a/libminifi/test/TestBase.h
+++ b/libminifi/test/TestBase.h
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
+#undef NDEBUG
 #pragma once
 
 #include <chrono>
@@ -188,7 +188,7 @@ class TempDirectory {
 
   ~TempDirectory() {
     if (is_owner_) {
-      minifi::utils::file::FileUtils::delete_dir(path_, true);
+      assert(minifi::utils::file::FileUtils::delete_dir(path_, true) == 0);
     }
   }
 
diff --git a/libminifi/test/keyvalue-tests/PersistentStateStorageTest.cpp b/libminifi/test/keyvalue-tests/PersistentStateStorageTest.cpp
index 6c945ef3c..1d494708d 100644
--- a/libminifi/test/keyvalue-tests/PersistentStateStorageTest.cpp
+++ b/libminifi/test/keyvalue-tests/PersistentStateStorageTest.cpp
@@ -32,6 +32,7 @@
 #include "controllers/keyvalue/AutoPersistor.h"
 #include "unit/ProvenanceTestHelper.h"
 #include "repository/VolatileContentRepository.h"
+#include "utils/file/FileUtils.h"
 
 static std::string config_yaml; // NOLINT
 
@@ -70,6 +71,7 @@ class PersistentStateStorageTestsFixture {
 
   virtual ~PersistentStateStorageTestsFixture() {
     LogTestController::getInstance().reset();
+    std::filesystem::current_path(minifi::utils::file::get_executable_dir());
   }
 
   void loadYaml() {
@@ -108,6 +110,7 @@ class PersistentStateStorageTestsFixture {
   }
 
  protected:
+  TestController testController;
   std::shared_ptr<minifi::Configure> configuration;
   std::shared_ptr<core::Repository> test_repo;
   std::shared_ptr<core::Repository> test_flow_repo;
@@ -119,8 +122,6 @@ class PersistentStateStorageTestsFixture {
 
   std::shared_ptr<core::controller::ControllerServiceNode> persistable_key_value_store_service_node;
   std::shared_ptr<minifi::controllers::KeyValueStateStorage> controller;
-
-  TestController testController;
 };
 
 
diff --git a/libminifi/test/keyvalue-tests/VolatileMapStateStorageTest.cpp b/libminifi/test/keyvalue-tests/VolatileMapStateStorageTest.cpp
index 6edc57df0..7681bab92 100644
--- a/libminifi/test/keyvalue-tests/VolatileMapStateStorageTest.cpp
+++ b/libminifi/test/keyvalue-tests/VolatileMapStateStorageTest.cpp
@@ -27,6 +27,7 @@
 #include "core/yaml/YamlConfiguration.h"
 #include "unit/ProvenanceTestHelper.h"
 #include "repository/VolatileContentRepository.h"
+#include "utils/file/FileUtils.h"
 
 namespace {
   std::string config_yaml; // NOLINT
@@ -77,6 +78,7 @@ class VolatileMapStateStorageTestFixture {
   }
 
   virtual ~VolatileMapStateStorageTestFixture() {
+    std::filesystem::current_path(minifi::utils::file::get_executable_dir());
     LogTestController::getInstance().reset();
   }
 
diff --git a/libminifi/test/unit/FileStreamTests.cpp b/libminifi/test/unit/FileStreamTests.cpp
index c29bddf7e..7032be269 100644
--- a/libminifi/test/unit/FileStreamTests.cpp
+++ b/libminifi/test/unit/FileStreamTests.cpp
@@ -280,6 +280,7 @@ TEST_CASE("Opening file without permission creates error logs") {
   minifi::io::FileStream stream(path_to_permissionless_file, 0, true);  // write_enabled, because permissionless file only means read-only on Windows
   REQUIRE(test_controller.getLog().getInstance().contains("Error opening file", std::chrono::seconds(0)));
   REQUIRE(test_controller.getLog().getInstance().contains("Permission denied", std::chrono::seconds(0)));
+  utils::file::FileUtils::set_permissions(path_to_permissionless_file, 0777);
 }
 
 TEST_CASE("Readonly filestream write test") {


[nifi-minifi-cpp] 01/02: MINIFICPP-2056 CWEL: support UserData in json Closes #1515

Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

martinzink pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git

commit d12c354ea5c25510aa62f3bd2cd18f7b82b6183c
Author: Marton Szasz <sz...@apache.org>
AuthorDate: Mon Mar 6 09:26:11 2023 +0100

    MINIFICPP-2056 CWEL: support UserData in json
    Closes #1515
    
    Signed-off-by: Martin Zink <ma...@apache.org>
---
 .../tests/ConsumeWindowsEventLogTests.cpp          | 65 ++++++++++++++++++++++
 extensions/windows-event-log/wel/JSONUtils.cpp     | 53 ++++++++++++------
 2 files changed, 101 insertions(+), 17 deletions(-)

diff --git a/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp b/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp
index ee9020161..4a309986d 100644
--- a/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp
+++ b/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp
@@ -27,6 +27,7 @@
 #include "CWELTestUtils.h"
 #include "Utils.h"
 #include "../wel/UniqueEvtHandle.h"
+#include "../wel/JSONUtils.h"
 #include "utils/Deleters.h"
 
 using ConsumeWindowsEventLog = org::apache::nifi::minifi::processors::ConsumeWindowsEventLog;
@@ -469,4 +470,68 @@ TEST_CASE("ConsumeWindowsEventLog batch commit size works", "[onTrigger]") {
   batchCommitSizeTestHelper(5, 0, 5);
 }
 
+TEST_CASE("ConsumeWindowsEventLog Simple JSON works with UserData", "[cwel][json][userdata]") {
+  using org::apache::nifi::minifi::wel::jsonToString;
+  using org::apache::nifi::minifi::wel::toSimpleJSON;
+  using org::apache::nifi::minifi::wel::toFlattenedJSON;
+  const auto event_xml = R"(
+<Event xmlns="http://schemas.microsoft.com/win/2004/08/events/event">
+  <System>
+    <Provider Name="Microsoft-Windows-AppLocker" Guid="CBDA4DBF-8D5D-4F69-9578-BE14AA540D22">
+    </Provider>
+    <EventID>8002</EventID>
+    <Version>0</Version>
+    <Level>4</Level>
+    <Task>0</Task>
+    <Opcode>0</Opcode>
+    <Keywords>0x8000000000000000</Keywords>
+    <TimeCreated SystemTime="2023-02-06T16:58:09.008534Z">
+    </TimeCreated>
+    <EventRecordID>46</EventRecordID>
+    <Correlation>
+    </Correlation>
+    <Execution ProcessID="1234" ThreadID="1235">
+    </Execution>
+    <Channel>Microsoft-Windows-AppLocker/EXE and DLL</Channel>
+    <Computer>example.local</Computer>
+    <Security UserID="S-1-1-0">
+    </Security>
+  </System>
+  <UserData>
+    <RuleAndFileData xmlns="http://schemas.microsoft.com/schemas/event/Microsoft.Windows/1.0.0.0">
+      <PolicyNameLength>3</PolicyNameLength>
+      <PolicyName>EXE</PolicyName>
+      <RuleNameLength>9</RuleNameLength>
+      <RuleName>All files</RuleName>
+      <RuleSddlLength>48</RuleSddlLength>
+      <RuleSddl>D:(XA;;FX;;;S-1-1-0;(APPID://PATH Contains &quot;*&quot;))</RuleSddl>
+      <TargetUser>S-1-1-0</TargetUser>
+      <TargetProcessId>1234</TargetProcessId>
+      <FilePathLength>22</FilePathLength>
+      <FilePath>%SYSTEM32%\CSCRIPT.EXE</FilePath>
+      <FileHashLength>0</FileHashLength>
+      <FileHash></FileHash>
+      <FqbnLength>1</FqbnLength>
+      <Fqbn>-</Fqbn>
+      <Parent foo="bar"><Child/></Parent>
+      <Leaf foo="bar"></Leaf>
+      <AltLeaf foo="bar"/>
+    </RuleAndFileData>
+  </UserData>
+</Event>
+)";
+  pugi::xml_document doc;
+  REQUIRE(doc.load_string(event_xml));
+  SECTION("simple") {
+    const auto simple_json = jsonToString(toSimpleJSON(doc));
+    const auto expected_json = R"json({"System":{"Provider":{"Name":"Microsoft-Windows-AppLocker","Guid":"CBDA4DBF-8D5D-4F69-9578-BE14AA540D22"},"EventID":"8002","Version":"0","Level":"4","Task":"0","Opcode":"0","Keywords":"0x8000000000000000","TimeCreated":{"SystemTime":"2023-02-06T16:58:09.008534Z"},"EventRecordID":"46","Correlation":{},"Execution":{"ProcessID":"1234","ThreadID":"1235"},"Channel":"Microsoft-Windows-AppLocker/EXE and DLL","Computer":"example.local"},"EventData":[],"User [...]
+    CHECK(expected_json == simple_json);
+  }
+  SECTION("flattened") {
+    const auto flattened_json = jsonToString(toFlattenedJSON(doc));
+    const auto expected_json = R"json({"Name":"Microsoft-Windows-AppLocker","Guid":"CBDA4DBF-8D5D-4F69-9578-BE14AA540D22","EventID":"8002","Version":"0","Level":"4","Task":"0","Opcode":"0","Keywords":"0x8000000000000000","SystemTime":"2023-02-06T16:58:09.008534Z","EventRecordID":"46","ProcessID":"1234","ThreadID":"1235","Channel":"Microsoft-Windows-AppLocker/EXE and DLL","Computer":"example.local","EventData":[],"PolicyNameLength":"3","PolicyName":"EXE","RuleNameLength":"9","RuleName":"A [...]
+    CHECK(expected_json == flattened_json);
+  }
+}
+
 }  // namespace org::apache::nifi::minifi::test
diff --git a/extensions/windows-event-log/wel/JSONUtils.cpp b/extensions/windows-event-log/wel/JSONUtils.cpp
index da108d6ba..e6bfc2f7e 100644
--- a/extensions/windows-event-log/wel/JSONUtils.cpp
+++ b/extensions/windows-event-log/wel/JSONUtils.cpp
@@ -17,27 +17,19 @@
 
 #include "JSONUtils.h"
 
-#include <vector>
+#include <algorithm>
 #include <string>
-#include <functional>
 
 #include <pugixml.hpp>
 
 #include "rapidjson/document.h"
 #include "rapidjson/writer.h"
 #include "rapidjson/stringbuffer.h"
-#include "rapidjson/prettywriter.h"
 
 #include "utils/gsl.h"
 
-namespace org {
-namespace apache {
-namespace nifi {
-namespace minifi {
-namespace wel {
-
+namespace org::apache::nifi::minifi::wel {
 namespace {
-
 rapidjson::Value xmlElementToJSON(const pugi::xml_node& node, rapidjson::Document& doc) {
   gsl_Expects(node.type() == pugi::xml_node_type::node_element);
   rapidjson::Value object(rapidjson::kObjectType);
@@ -67,6 +59,29 @@ rapidjson::Value xmlDocumentToJSON(const pugi::xml_node& node, rapidjson::Docume
   return children;
 }
 
+void simplifiedGenericXmlToJson(const pugi::xml_node& source_node, rapidjson::Value& output_value, rapidjson::Document& allocator_source_document, bool flatten = false) {
+  gsl_Expects(source_node.type() == pugi::xml_node_type::node_element);
+  for (const auto& attr : source_node.attributes()) {
+    if (attr.name() == std::string_view{"xmlns"}) {
+      continue;  // skip xmlns attribute, because it's metadata
+    }
+    output_value.AddMember(rapidjson::StringRef(attr.name()), rapidjson::StringRef(attr.value()), allocator_source_document.GetAllocator());
+  }
+  for (const auto& child: source_node.children()) {
+    if (child.type() == pugi::xml_node_type::node_element) {
+      const auto is_pcdata = [](const pugi::xml_node& node) { return node.type() == pugi::xml_node_type::node_pcdata; };
+      if (std::all_of(child.children().begin(), child.children().end(), is_pcdata)) {
+        // all children are pcdata (text): leaf node
+        output_value.AddMember(rapidjson::StringRef(child.name()), rapidjson::StringRef(child.text().get()), allocator_source_document.GetAllocator());
+      } else {
+        // there are non-text children: recurse further
+        auto& child_val = flatten ? output_value : output_value.AddMember(rapidjson::StringRef(child.name()), rapidjson::kObjectType, allocator_source_document.GetAllocator())[child.name()];
+        simplifiedGenericXmlToJson(child, child_val, allocator_source_document, flatten);
+      }
+    }
+  }
+}
+
 rapidjson::Document toJSONImpl(const pugi::xml_node& root, bool flatten) {
   rapidjson::Document doc{rapidjson::kObjectType};
 
@@ -101,7 +116,10 @@ rapidjson::Document toJSONImpl(const pugi::xml_node& root, bool flatten) {
     {
       auto correlation_xml = system_xml.child("Correlation");
       auto& correlation = flatten ? doc : system.AddMember("Correlation", rapidjson::kObjectType, doc.GetAllocator())["Correlation"];
-      correlation.AddMember("ActivityID", rapidjson::StringRef(correlation_xml.attribute("ActivityID").value()), doc.GetAllocator());
+      const auto activity_id = correlation_xml.attribute("ActivityID");
+      if (!activity_id.empty()) {
+        correlation.AddMember("ActivityID", rapidjson::StringRef(activity_id.value()), doc.GetAllocator());
+      }
     }
 
     {
@@ -135,6 +153,12 @@ rapidjson::Document toJSONImpl(const pugi::xml_node& root, bool flatten) {
     }
   }
 
+  const auto userdata_xml = event_xml.child("UserData");
+  if (!userdata_xml.empty()) {
+    auto& userdata = flatten ? doc : doc.AddMember("UserData", rapidjson::kObjectType, doc.GetAllocator())["UserData"];
+    simplifiedGenericXmlToJson(userdata_xml, userdata, doc, flatten);
+  }
+
   return doc;
 }
 
@@ -162,9 +186,4 @@ std::string jsonToString(const rapidjson::Document& doc) {
   doc.Accept(writer);
   return buffer.GetString();
 }
-
-}  // namespace wel
-}  // namespace minifi
-}  // namespace nifi
-}  // namespace apache
-}  // namespace org
+}  // namespace org::apache::nifi::minifi::wel