You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by sz...@apache.org on 2023/04/20 16:34:51 UTC
[nifi-minifi-cpp] 02/03: MINIFICPP-2089 CWEL: prefix EventData in flattened JSON
This is an automated email from the ASF dual-hosted git repository.
szaszm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 242686b5a9385fb82b416043863379a589aeff63
Author: Martin Zink <ma...@protonmail.com>
AuthorDate: Thu Apr 20 18:25:23 2023 +0200
MINIFICPP-2089 CWEL: prefix EventData in flattened JSON
...so it doesnt need to be included twice
Closes #1552
Signed-off-by: Marton Szasz <sz...@apache.org>
---
CMakeLists.txt | 1 +
.../tests/CWELCustomProviderTests.cpp | 7 ++-
.../tests/ConsumeWindowsEventLogTests.cpp | 8 +--
extensions/windows-event-log/wel/JSONUtils.cpp | 68 +++++++++++++++-------
4 files changed, 55 insertions(+), 29 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7eb4edc2a..1bc7f382c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -288,6 +288,7 @@ target_include_directories(concurrentqueue SYSTEM INTERFACE "${CMAKE_CURRENT_SOU
# RapidJSON
add_library(RapidJSON INTERFACE)
target_include_directories(RapidJSON SYSTEM INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/rapidjson-48fbd8cd202ca54031fe799db2ad44ffa8e77c13/include")
+target_compile_definitions(RapidJSON INTERFACE RAPIDJSON_HAS_STDSTRING)
# cxxopts
include(CxxOpts)
diff --git a/extensions/windows-event-log/tests/CWELCustomProviderTests.cpp b/extensions/windows-event-log/tests/CWELCustomProviderTests.cpp
index 1a771ec3a..9a3a61e4c 100644
--- a/extensions/windows-event-log/tests/CWELCustomProviderTests.cpp
+++ b/extensions/windows-event-log/tests/CWELCustomProviderTests.cpp
@@ -172,9 +172,10 @@ TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Flattened correctly cus
{
"Name": ")" + CUSTOM_PROVIDER_NAME + R"(",
"Channel": ")" + CUSTOM_CHANNEL /* Channel is not overwritten by data named "Channel" */ + R"(",
- "EventData": )" + EVENT_DATA_JSON /* EventData is not discarded */ + R"(,
- "param1": "Actual event",
- "param2": "Second"
+ "EventData.param1": "Actual event",
+ "EventData.param2": "Second",
+ "EventData.Channel": "Third",
+ "EventData": "0901"
}
)");
}
diff --git a/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp b/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp
index d91174419..d0e4d39d7 100644
--- a/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp
+++ b/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp
@@ -390,11 +390,7 @@ TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Flattened correctly", "
{
"Name": "Application",
"Channel": "Application",
- "EventData": [{
- "Type": "Data",
- "Content": "Event one",
- "Name": ""
- }]
+ "EventData": "Event one"
}
)json");
}
@@ -526,7 +522,7 @@ TEST_CASE("ConsumeWindowsEventLog Simple JSON works with UserData", "[cwel][json
}
SECTION("flattened") {
const auto flattened_json = jsonToString(toFlattenedJSON(doc));
- const auto expected_json = R"json({"Name":"Microsoft-Windows-AppLocker","Guid":"CBDA4DBF-8D5D-4F69-9578-BE14AA540D22","EventID":"8002","Version":"0","Level":"4","Task":"0","Opcode":"0","Keywords":"0x8000000000000000","SystemTime":"2023-02-06T16:58:09.008534Z","EventRecordID":"46","ProcessID":"1234","ThreadID":"1235","Channel":"Microsoft-Windows-AppLocker/EXE and DLL","Computer":"example.local","EventData":[],"PolicyNameLength":"3","PolicyName":"EXE","RuleNameLength":"9","RuleName":"A [...]
+ const auto expected_json = R"json({"Name":"Microsoft-Windows-AppLocker","Guid":"CBDA4DBF-8D5D-4F69-9578-BE14AA540D22","EventID":"8002","Version":"0","Level":"4","Task":"0","Opcode":"0","Keywords":"0x8000000000000000","SystemTime":"2023-02-06T16:58:09.008534Z","EventRecordID":"46","ProcessID":"1234","ThreadID":"1235","Channel":"Microsoft-Windows-AppLocker/EXE and DLL","Computer":"example.local","UserData.RuleAndFileData.PolicyNameLength":"3","UserData.RuleAndFileData.PolicyName":"EXE" [...]
CHECK(expected_json == flattened_json);
}
}
diff --git a/extensions/windows-event-log/wel/JSONUtils.cpp b/extensions/windows-event-log/wel/JSONUtils.cpp
index e6bfc2f7e..0f2913d62 100644
--- a/extensions/windows-event-log/wel/JSONUtils.cpp
+++ b/extensions/windows-event-log/wel/JSONUtils.cpp
@@ -27,6 +27,7 @@
#include "rapidjson/stringbuffer.h"
#include "utils/gsl.h"
+#include "utils/StringUtils.h"
namespace org::apache::nifi::minifi::wel {
namespace {
@@ -59,29 +60,51 @@ rapidjson::Value xmlDocumentToJSON(const pugi::xml_node& node, rapidjson::Docume
return children;
}
-void simplifiedGenericXmlToJson(const pugi::xml_node& source_node, rapidjson::Value& output_value, rapidjson::Document& allocator_source_document, bool flatten = false) {
+void simplifiedGenericXmlToJson(const pugi::xml_node& source_node,
+ rapidjson::Value& output_value,
+ rapidjson::Document::AllocatorType& allocator,
+ std::optional<std::string> prefix_for_flat_structure) {
gsl_Expects(source_node.type() == pugi::xml_node_type::node_element);
+ const bool is_flattened = prefix_for_flat_structure.has_value();
for (const auto& attr : source_node.attributes()) {
if (attr.name() == std::string_view{"xmlns"}) {
continue; // skip xmlns attribute, because it's metadata
}
- output_value.AddMember(rapidjson::StringRef(attr.name()), rapidjson::StringRef(attr.value()), allocator_source_document.GetAllocator());
+ if (!is_flattened) {
+ output_value.AddMember(rapidjson::StringRef(attr.name()), rapidjson::StringRef(attr.value()), allocator);
+ } else {
+ output_value.AddMember(rapidjson::Value(*prefix_for_flat_structure + attr.name(), allocator).Move(), rapidjson::StringRef(attr.value()), allocator);
+ }
}
for (const auto& child: source_node.children()) {
if (child.type() == pugi::xml_node_type::node_element) {
const auto is_pcdata = [](const pugi::xml_node& node) { return node.type() == pugi::xml_node_type::node_pcdata; };
if (std::all_of(child.children().begin(), child.children().end(), is_pcdata)) {
// all children are pcdata (text): leaf node
- output_value.AddMember(rapidjson::StringRef(child.name()), rapidjson::StringRef(child.text().get()), allocator_source_document.GetAllocator());
+ if (!is_flattened) {
+ output_value.AddMember(rapidjson::StringRef(child.name()), rapidjson::StringRef(child.text().get()), allocator);
+ } else {
+ output_value.AddMember(rapidjson::Value(*prefix_for_flat_structure + child.name(), allocator).Move(), rapidjson::StringRef(child.text().get()), allocator);
+ }
} else {
// there are non-text children: recurse further
- auto& child_val = flatten ? output_value : output_value.AddMember(rapidjson::StringRef(child.name()), rapidjson::kObjectType, allocator_source_document.GetAllocator())[child.name()];
- simplifiedGenericXmlToJson(child, child_val, allocator_source_document, flatten);
+ auto& child_val = is_flattened ? output_value : output_value.AddMember(rapidjson::StringRef(child.name()), rapidjson::kObjectType, allocator)[child.name()];
+ auto new_prefix = is_flattened ? std::optional(*prefix_for_flat_structure + child.name() + ".") : std::nullopt;
+ simplifiedGenericXmlToJson(child, child_val, allocator, new_prefix);
}
}
}
}
+std::string createUniqueKey(const std::string& key, const rapidjson::Value& parent) {
+ auto proposed_key = key;
+ size_t postfix = 1;
+ while (parent.HasMember(proposed_key)) {
+ proposed_key = key + std::to_string(postfix++);
+ }
+ return proposed_key;
+}
+
rapidjson::Document toJSONImpl(const pugi::xml_node& root, bool flatten) {
rapidjson::Document doc{rapidjson::kObjectType};
@@ -135,20 +158,24 @@ rapidjson::Document toJSONImpl(const pugi::xml_node& root, bool flatten) {
{
auto eventData_xml = event_xml.child("EventData");
- // create EventData subarray even if flatten requested
- doc.AddMember("EventData", rapidjson::kArrayType, doc.GetAllocator());
- for (const auto& data : eventData_xml.children()) {
- auto name_attr = data.attribute("Name");
- rapidjson::Value item(rapidjson::kObjectType);
- item.AddMember("Name", rapidjson::StringRef(name_attr.value()), doc.GetAllocator());
- item.AddMember("Content", rapidjson::StringRef(data.text().get()), doc.GetAllocator());
- item.AddMember("Type", rapidjson::StringRef(data.name()), doc.GetAllocator());
- // we need to query EventData because a reference to it wouldn't be stable, as we
- // possibly add members to its parent which could result in reallocation
- doc["EventData"].PushBack(item, doc.GetAllocator());
- // check collision
- if (flatten && !name_attr.empty() && !doc.HasMember(name_attr.value())) {
- doc.AddMember(rapidjson::StringRef(name_attr.value()), rapidjson::StringRef(data.text().get()), doc.GetAllocator());
+ if (flatten) {
+ for (const auto& event_data_child : eventData_xml.children()) {
+ std::string key = "EventData";
+ if (auto name_attr = event_data_child.attribute("Name"); !name_attr.empty()) {
+ key = utils::StringUtils::join_pack(key, ".", name_attr.value());
+ }
+
+ doc.AddMember(rapidjson::Value(createUniqueKey(key, doc), doc.GetAllocator()).Move(), rapidjson::StringRef(event_data_child.text().get()), doc.GetAllocator());
+ }
+ } else {
+ doc.AddMember("EventData", rapidjson::kArrayType, doc.GetAllocator());
+ for (const auto& event_data_child : eventData_xml.children()) {
+ auto name_attr = event_data_child.attribute("Name");
+ rapidjson::Value item(rapidjson::kObjectType);
+ item.AddMember("Name", rapidjson::StringRef(name_attr.value()), doc.GetAllocator());
+ item.AddMember("Content", rapidjson::StringRef(event_data_child.text().get()), doc.GetAllocator());
+ item.AddMember("Type", rapidjson::StringRef(event_data_child.name()), doc.GetAllocator());
+ doc["EventData"].PushBack(item, doc.GetAllocator()); // we need to re-query EventData because a reference to it wouldn't be stable
}
}
}
@@ -156,7 +183,8 @@ rapidjson::Document toJSONImpl(const pugi::xml_node& root, bool flatten) {
const auto userdata_xml = event_xml.child("UserData");
if (!userdata_xml.empty()) {
auto& userdata = flatten ? doc : doc.AddMember("UserData", rapidjson::kObjectType, doc.GetAllocator())["UserData"];
- simplifiedGenericXmlToJson(userdata_xml, userdata, doc, flatten);
+ auto prefix = flatten ? std::optional("UserData.") : std::nullopt;
+ simplifiedGenericXmlToJson(userdata_xml, userdata, doc.GetAllocator(), prefix);
}
return doc;