You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by sz...@apache.org on 2024/03/14 13:27:51 UTC
(nifi-minifi-cpp) 03/04: MINIFICPP-2283 Create tool to encrypt sensitive properties in config.yml
This is an automated email from the ASF dual-hosted git repository.
szaszm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 530e4dc7b7ea70e319493399ce03b0ccba255840
Author: Ferenc Gerlits <fg...@gmail.com>
AuthorDate: Thu Mar 14 14:06:09 2024 +0100
MINIFICPP-2283 Create tool to encrypt sensitive properties in config.yml
Closes 1725
Signed-off-by: Marton Szasz <sz...@apache.org>
---
cmake/ArgParse.cmake | 4 +-
encrypt-config/EncryptConfig.cpp | 116 +++++++++-----
encrypt-config/EncryptConfig.h | 38 ++---
encrypt-config/EncryptConfigMain.cpp | 63 +++++---
encrypt-config/FlowConfigEncryptor.cpp | 174 +++++++++++++++++++++
.../FlowConfigEncryptor.h | 24 +--
encrypt-config/tests/ConfigFileEncryptorTests.cpp | 2 +-
libminifi/include/core/FlowConfiguration.h | 2 +-
libminifi/include/core/ProcessGroup.h | 2 +
.../core/controller/ControllerServiceNode.h | 1 +
.../include/core/flow/AdaptiveConfiguration.h | 2 +
libminifi/include/core/flow/FlowSerializer.h | 5 +-
libminifi/include/core/json/JsonFlowSerializer.h | 6 +-
libminifi/include/core/yaml/YamlFlowSerializer.h | 6 +-
libminifi/src/core/ProcessGroup.cpp | 8 +
.../src/core/controller/ControllerServiceNode.cpp | 18 +--
libminifi/src/core/flow/AdaptiveConfiguration.cpp | 6 +
.../src/core/flow/StructuredConfiguration.cpp | 2 +-
libminifi/src/core/json/JsonFlowSerializer.cpp | 35 +++--
libminifi/src/core/yaml/YamlFlowSerializer.cpp | 35 +++--
libminifi/test/unit/JsonFlowSerializerTests.cpp | 131 ++++++++++++++--
libminifi/test/unit/YamlFlowSerializerTests.cpp | 56 ++++++-
22 files changed, 576 insertions(+), 160 deletions(-)
diff --git a/cmake/ArgParse.cmake b/cmake/ArgParse.cmake
index 883f34986..29251ceb9 100644
--- a/cmake/ArgParse.cmake
+++ b/cmake/ArgParse.cmake
@@ -18,7 +18,7 @@
include(FetchContent)
FetchContent_Declare(
argparse
- URL https://github.com/p-ranav/argparse/archive/refs/tags/v2.9.tar.gz
- URL_HASH SHA256=cd563293580b9dc592254df35b49cf8a19b4870ff5f611c7584cf967d9e6031e
+ URL https://github.com/p-ranav/argparse/archive/refs/tags/v3.0.tar.gz
+ URL_HASH SHA256=ba7b465759bb01069d57302855eaf4d1f7d677f21ad7b0b00b92939645c30f47
)
FetchContent_MakeAvailable(argparse)
diff --git a/encrypt-config/EncryptConfig.cpp b/encrypt-config/EncryptConfig.cpp
index d2876b63a..a1a3c89c9 100644
--- a/encrypt-config/EncryptConfig.cpp
+++ b/encrypt-config/EncryptConfig.cpp
@@ -25,65 +25,78 @@
#include "ConfigFile.h"
#include "ConfigFileEncryptor.h"
+#include "utils/Enum.h"
#include "utils/file/FileUtils.h"
#include "Defaults.h"
+#include "core/extension/ExtensionManager.h"
+#include "FlowConfigEncryptor.h"
namespace {
+constexpr std::string_view ENCRYPTION_KEY_PROPERTY_NAME = "nifi.bootstrap.sensitive.key";
+constexpr std::string_view SENSITIVE_PROPERTIES_KEY_PROPERTY_NAME = "nifi.bootstrap.sensitive.properties.key";
-constexpr const char* OLD_KEY_PROPERTY_NAME = "nifi.bootstrap.sensitive.key.old";
-constexpr const char* ENCRYPTION_KEY_PROPERTY_NAME = "nifi.bootstrap.sensitive.key";
-
+std::string readFile(const std::filesystem::path& file_path) {
+ try {
+ std::ifstream file_stream{file_path, std::ios::binary};
+ file_stream.exceptions(std::ios::failbit | std::ios::badbit);
+ return {std::istreambuf_iterator<char>(file_stream), {}};
+ } catch (...) {
+ throw std::runtime_error("Error while reading file \"" + file_path.string() + "\"");
+ }
+}
} // namespace
namespace org::apache::nifi::minifi::encrypt_config {
EncryptConfig::EncryptConfig(const std::string& minifi_home) : minifi_home_(minifi_home) {
if (sodium_init() < 0) {
+ // encryption/decryption depends on the libsodium library which needs to be initialized
throw std::runtime_error{"Could not initialize the libsodium library!"};
}
- // encryption/decryption depends on the libsodium library which needs to be initialized
- keys_ = getEncryptionKeys();
+
+ std::filesystem::current_path(minifi_home_);
}
-EncryptConfig::EncryptionType EncryptConfig::encryptSensitiveProperties() const {
- encryptSensitiveProperties(keys_);
- if (keys_.old_key) {
- return EncryptionType::RE_ENCRYPT;
- }
- return EncryptionType::ENCRYPT;
+bool EncryptConfig::isReencrypting() const {
+ encrypt_config::ConfigFile bootstrap_file{std::ifstream{bootstrapFilePath()}};
+
+ std::string decryption_key_name = utils::string::join_pack(ENCRYPTION_KEY_PROPERTY_NAME, ".old");
+ std::optional<std::string> decryption_key_hex = bootstrap_file.getValue(decryption_key_name);
+
+ return (decryption_key_hex && !decryption_key_hex->empty());
}
-void EncryptConfig::encryptFlowConfig() const {
+std::filesystem::path EncryptConfig::flowConfigPath() const {
encrypt_config::ConfigFile properties_file{std::ifstream{propertiesFilePath()}};
std::optional<std::filesystem::path> config_path{properties_file.getValue(Configure::nifi_flow_configuration_file)};
if (!config_path) {
config_path = utils::file::PathUtils::resolve(minifi_home_, "conf/config.yml");
- std::cout << "Couldn't find path of configuration file, using default: \"" << *config_path << "\"\n";
+ std::cout << "Couldn't find path of configuration file, using default: " << *config_path << '\n';
} else {
config_path = utils::file::PathUtils::resolve(minifi_home_, *config_path);
- std::cout << "Encrypting flow configuration file: \"" << *config_path << "\"\n";
- }
- std::string config_content;
- try {
- std::ifstream config_file{*config_path, std::ios::binary};
- config_file.exceptions(std::ios::failbit | std::ios::badbit);
- config_content = std::string{std::istreambuf_iterator<char>(config_file), {}};
- } catch (...) {
- throw std::runtime_error("Error while reading flow configuration file \"" + config_path->string() + "\"");
+ std::cout << "Encrypting flow configuration file: " << *config_path << '\n';
}
+ return *config_path;
+}
+
+void EncryptConfig::encryptWholeFlowConfigFile() const {
+ EncryptionKeys keys = getEncryptionKeys(ENCRYPTION_KEY_PROPERTY_NAME);
+
+ std::filesystem::path config_path = flowConfigPath();
+ std::string config_content = readFile(config_path);
try {
- utils::crypto::decrypt(config_content, keys_.encryption_key);
+ utils::crypto::decrypt(config_content, keys.encryption_key);
std::cout << "Flow config file is already properly encrypted.\n";
return;
} catch (const std::exception&) {}
if (utils::crypto::isEncrypted(config_content)) {
- if (!keys_.old_key) {
+ if (!keys.old_key) {
throw std::runtime_error("Config file is encrypted, but no old encryption key is set.");
}
std::cout << "Trying to decrypt flow config file using the old key ...\n";
try {
- config_content = utils::crypto::decrypt(config_content, *keys_.old_key);
+ config_content = utils::crypto::decrypt(config_content, *keys.old_key);
} catch (const std::exception&) {
throw std::runtime_error("Flow config is encrypted, but couldn't be decrypted.");
}
@@ -91,15 +104,15 @@ void EncryptConfig::encryptFlowConfig() const {
std::cout << "Flow config file is not encrypted, using as-is.\n";
}
- std::string encrypted_content = utils::crypto::encrypt(config_content, keys_.encryption_key);
+ std::string encrypted_content = utils::crypto::encrypt(config_content, keys.encryption_key);
try {
- std::ofstream encrypted_file{*config_path, std::ios::binary};
+ std::ofstream encrypted_file{config_path, std::ios::binary};
encrypted_file.exceptions(std::ios::failbit | std::ios::badbit);
encrypted_file << encrypted_content;
} catch (...) {
- throw std::runtime_error("Error while writing encrypted flow configuration file \"" + config_path->string() + "\"");
+ throw std::runtime_error("Error while writing encrypted flow configuration file \"" + config_path.string() + "\"");
}
- std::cout << "Successfully encrypted flow configuration file: \"" << *config_path << "\"\n";
+ std::cout << "Successfully encrypted flow configuration file: " << config_path << '\n';
}
std::filesystem::path EncryptConfig::bootstrapFilePath() const {
@@ -110,27 +123,31 @@ std::filesystem::path EncryptConfig::propertiesFilePath() const {
return minifi_home_ / DEFAULT_NIFI_PROPERTIES_FILE;
}
-EncryptionKeys EncryptConfig::getEncryptionKeys() const {
+EncryptionKeys EncryptConfig::getEncryptionKeys(std::string_view property_name) const {
encrypt_config::ConfigFile bootstrap_file{std::ifstream{bootstrapFilePath()}};
- std::optional<std::string> decryption_key_hex = bootstrap_file.getValue(OLD_KEY_PROPERTY_NAME);
- std::optional<std::string> encryption_key_hex = bootstrap_file.getValue(ENCRYPTION_KEY_PROPERTY_NAME);
+
+ std::string decryption_key_name = utils::string::join_pack(property_name, ".old");
+ std::optional<std::string> decryption_key_hex = bootstrap_file.getValue(decryption_key_name);
+
+ std::string encryption_key_name{property_name};
+ std::optional<std::string> encryption_key_hex = bootstrap_file.getValue(encryption_key_name);
EncryptionKeys keys;
if (decryption_key_hex && !decryption_key_hex->empty()) {
- std::string binary_key = hexDecodeAndValidateKey(*decryption_key_hex, OLD_KEY_PROPERTY_NAME);
+ std::string binary_key = hexDecodeAndValidateKey(*decryption_key_hex, decryption_key_name);
std::cout << "Old encryption key found in " << bootstrapFilePath() << "\n";
keys.old_key = utils::crypto::stringToBytes(binary_key);
}
if (encryption_key_hex && !encryption_key_hex->empty()) {
- std::string binary_key = hexDecodeAndValidateKey(*encryption_key_hex, ENCRYPTION_KEY_PROPERTY_NAME);
- std::cout << "Using the existing encryption key found in " << bootstrapFilePath() << '\n';
+ std::string binary_key = hexDecodeAndValidateKey(*encryption_key_hex, encryption_key_name);
+ std::cout << "Using the existing encryption key " << property_name << " found in " << bootstrapFilePath() << '\n';
keys.encryption_key = utils::crypto::stringToBytes(binary_key);
} else {
std::cout << "Generating a new encryption key...\n";
utils::crypto::Bytes encryption_key = utils::crypto::generateKey();
- writeEncryptionKeyToBootstrapFile(encryption_key);
- std::cout << "Wrote the new encryption key to " << bootstrapFilePath() << '\n';
+ writeEncryptionKeyToBootstrapFile(encryption_key_name, encryption_key);
+ std::cout << "Wrote the new encryption key " << property_name << " to " << bootstrapFilePath() << '\n';
keys.encryption_key = encryption_key;
}
return keys;
@@ -150,20 +167,22 @@ std::string EncryptConfig::hexDecodeAndValidateKey(const std::string& key, const
}
}
-void EncryptConfig::writeEncryptionKeyToBootstrapFile(const utils::crypto::Bytes& encryption_key) const {
+void EncryptConfig::writeEncryptionKeyToBootstrapFile(const std::string& encryption_key_name, const utils::crypto::Bytes& encryption_key) const {
std::string key_encoded = utils::string::to_hex(utils::crypto::bytesToString(encryption_key));
encrypt_config::ConfigFile bootstrap_file{std::ifstream{bootstrapFilePath()}};
- if (bootstrap_file.hasValue(ENCRYPTION_KEY_PROPERTY_NAME)) {
- bootstrap_file.update(ENCRYPTION_KEY_PROPERTY_NAME, key_encoded);
+ if (bootstrap_file.hasValue(encryption_key_name)) {
+ bootstrap_file.update(encryption_key_name, key_encoded);
} else {
- bootstrap_file.append(ENCRYPTION_KEY_PROPERTY_NAME, key_encoded);
+ bootstrap_file.append(encryption_key_name, key_encoded);
}
bootstrap_file.writeTo(bootstrapFilePath());
}
-void EncryptConfig::encryptSensitiveProperties(const EncryptionKeys& keys) const {
+void EncryptConfig::encryptSensitiveValuesInMinifiProperties() const {
+ EncryptionKeys keys = getEncryptionKeys(ENCRYPTION_KEY_PROPERTY_NAME);
+
encrypt_config::ConfigFile properties_file{std::ifstream{propertiesFilePath()}};
if (properties_file.size() == 0) {
throw std::runtime_error{"Properties file " + propertiesFilePath().string() + " not found!"};
@@ -180,4 +199,17 @@ void EncryptConfig::encryptSensitiveProperties(const EncryptionKeys& keys) const
<< (num_properties_encrypted == 1 ? "property" : "properties") << " in " << propertiesFilePath() << '\n';
}
+void EncryptConfig::encryptSensitiveValuesInFlowConfig(
+ const std::optional<std::string>& component_id, const std::optional<std::string>& property_name, const std::optional<std::string>& property_value) const {
+ if (!component_id && !property_name && !property_value) {
+ EncryptionKeys keys = getEncryptionKeys(SENSITIVE_PROPERTIES_KEY_PROPERTY_NAME);
+ flow_config_encryptor::encryptSensitiveValuesInFlowConfig(keys, minifi_home_, flowConfigPath());
+ } else if (component_id && property_name && property_value) {
+ EncryptionKeys keys = getEncryptionKeys(SENSITIVE_PROPERTIES_KEY_PROPERTY_NAME);
+ flow_config_encryptor::encryptSensitiveValuesInFlowConfig(keys, minifi_home_, flowConfigPath(), *component_id, *property_name, *property_value);
+ } else {
+ throw std::runtime_error("either all of --component-id, --property-name and --property-value should be given (for batch mode) or none of them (for interactive mode)");
+ }
+}
+
} // namespace org::apache::nifi::minifi::encrypt_config
diff --git a/encrypt-config/EncryptConfig.h b/encrypt-config/EncryptConfig.h
index 6cdd6e440..1f48f48fb 100644
--- a/encrypt-config/EncryptConfig.h
+++ b/encrypt-config/EncryptConfig.h
@@ -21,40 +21,28 @@
#include "Utils.h"
-namespace org {
-namespace apache {
-namespace nifi {
-namespace minifi {
-namespace encrypt_config {
+namespace org::apache::nifi::minifi::encrypt_config {
class EncryptConfig {
public:
- enum class EncryptionType {
- ENCRYPT,
- RE_ENCRYPT
- };
-
explicit EncryptConfig(const std::string& minifi_home);
- EncryptionType encryptSensitiveProperties() const;
- void encryptFlowConfig() const;
+ void encryptSensitiveValuesInMinifiProperties() const;
+ void encryptSensitiveValuesInFlowConfig(const std::optional<std::string>& component_id, const std::optional<std::string>& property_name, const std::optional<std::string>& property_value) const;
+ void encryptWholeFlowConfigFile() const;
- private:
- std::filesystem::path bootstrapFilePath() const;
- std::filesystem::path propertiesFilePath() const;
+ [[nodiscard]] bool isReencrypting() const;
- EncryptionKeys getEncryptionKeys() const;
- std::string hexDecodeAndValidateKey(const std::string& key, const std::string& key_name) const;
- void writeEncryptionKeyToBootstrapFile(const utils::crypto::Bytes& encryption_key) const;
+ private:
+ [[nodiscard]] std::filesystem::path bootstrapFilePath() const;
+ [[nodiscard]] std::filesystem::path propertiesFilePath() const;
+ [[nodiscard]] std::filesystem::path flowConfigPath() const;
- void encryptSensitiveProperties(const EncryptionKeys& keys) const;
+ [[nodiscard]] EncryptionKeys getEncryptionKeys(std::string_view property_name) const;
+ [[nodiscard]] std::string hexDecodeAndValidateKey(const std::string& key, const std::string& key_name) const;
+ void writeEncryptionKeyToBootstrapFile(const std::string& encryption_key_name, const utils::crypto::Bytes& encryption_key) const;
const std::filesystem::path minifi_home_;
- EncryptionKeys keys_;
};
-} // namespace encrypt_config
-} // namespace minifi
-} // namespace nifi
-} // namespace apache
-} // namespace org
+} // namespace org::apache::nifi::minifi::encrypt_config
diff --git a/encrypt-config/EncryptConfigMain.cpp b/encrypt-config/EncryptConfigMain.cpp
index 34760781b..1faa91007 100644
--- a/encrypt-config/EncryptConfigMain.cpp
+++ b/encrypt-config/EncryptConfigMain.cpp
@@ -21,37 +21,64 @@
#include "EncryptConfig.h"
#include "argparse/argparse.hpp"
#include "agent/agent_version.h"
+#include "utils/StringUtils.h"
-using org::apache::nifi::minifi::encrypt_config::EncryptConfig;
+namespace minifi = org::apache::nifi::minifi;
+using minifi::encrypt_config::EncryptConfig;
+
+constexpr std::string_view OPERATION_MINIFI_PROPERTIES = "minifi-properties";
+constexpr std::string_view OPERATION_FLOW_CONFIG = "flow-config";
+constexpr std::string_view OPERATION_WHOLE_FLOW_CONFIG_FILE = "whole-flow-config-file";
int main(int argc, char* argv[]) try {
argparse::ArgumentParser argument_parser("Apache MiNiFi C++ Encrypt-Config", org::apache::nifi::minifi::AgentBuild::VERSION);
+ argument_parser.add_argument("operation")
+ .default_value("minifi-properties")
+ .help(minifi::utils::string::join_pack("what to encrypt: ", OPERATION_MINIFI_PROPERTIES, " | ", OPERATION_FLOW_CONFIG, " | ", OPERATION_WHOLE_FLOW_CONFIG_FILE));
argument_parser.add_argument("-m", "--minifi-home")
- .required()
- .metavar("MINIFI_HOME")
- .help("Specifies the home directory used by the minifi agent");
- argument_parser.add_argument("-e", "--encrypt-flow-config")
- .default_value(false)
- .implicit_value(true)
- .help("If set, the flow configuration file (as specified in minifi.properties) is also encrypted.");
+ .required()
+ .metavar("MINIFI_HOME")
+ .help("Specifies the home directory used by the minifi agent");
+ argument_parser.add_argument("--component-id")
+ .metavar("ID")
+ .help(minifi::utils::string::join_pack("Processor or controller service id (", OPERATION_FLOW_CONFIG, " only)"));
+ argument_parser.add_argument("--property-name")
+ .metavar("NAME")
+ .help(minifi::utils::string::join_pack("The name of the sensitive property (", OPERATION_FLOW_CONFIG, " only)"));
+ argument_parser.add_argument("--property-value")
+ .metavar("VALUE")
+ .help(minifi::utils::string::join_pack("The new value of the sensitive property (", OPERATION_FLOW_CONFIG, " only)"));
try {
argument_parser.parse_args(argc, argv);
} catch (const std::runtime_error& err) {
- std::cerr << err.what() << std::endl;
- std::cerr << argument_parser;
- std::exit(1);
+ std::cerr << err.what() << "\n\n" << argument_parser;
+ return 1;
}
EncryptConfig encrypt_config{argument_parser.get("-m")};
- EncryptConfig::EncryptionType type = encrypt_config.encryptSensitiveProperties();
- if (argument_parser.get<bool>("--encrypt-flow-config")) {
- encrypt_config.encryptFlowConfig();
- } else if (type == EncryptConfig::EncryptionType::RE_ENCRYPT) {
- std::cout << "WARNING: you did not request the flow config to be updated, "
- << "if it is currently encrypted and the old key is removed, "
- << "you won't be able to recover the flow config.\n";
+ std::string operation = argument_parser.get("operation");
+
+ if (operation == OPERATION_MINIFI_PROPERTIES) {
+ encrypt_config.encryptSensitiveValuesInMinifiProperties();
+ } else if (operation == OPERATION_FLOW_CONFIG) {
+ auto component_id = argument_parser.present("--component-id");
+ auto property_name = argument_parser.present("--property-name");
+ auto property_value = argument_parser.present("--property-value");
+ encrypt_config.encryptSensitiveValuesInFlowConfig(component_id, property_name, property_value);
+ } else if (operation == OPERATION_WHOLE_FLOW_CONFIG_FILE) {
+ encrypt_config.encryptWholeFlowConfigFile();
+ } else {
+ std::cerr << "Unknown operation: " << operation << "\n\n" << argument_parser;
+ return 4;
}
+
+ if ((operation == OPERATION_MINIFI_PROPERTIES || operation == OPERATION_WHOLE_FLOW_CONFIG_FILE) && encrypt_config.isReencrypting()) {
+ std::cout << "WARNING: an .old key was provided, which is used for both " << OPERATION_MINIFI_PROPERTIES << " and " << OPERATION_WHOLE_FLOW_CONFIG_FILE << ".\n"
+ << "If both are currently encrypted, make sure to run " << argv[0] << " to re-encrypt both before removing the .old key,\n"
+ << "otherwise you won't be able to recover the encrypted data!\n";
+ }
+
return 0;
} catch (const std::exception& ex) {
std::cerr << ex.what() << "\n(" << typeid(ex).name() << ")\n";
diff --git a/encrypt-config/FlowConfigEncryptor.cpp b/encrypt-config/FlowConfigEncryptor.cpp
new file mode 100644
index 000000000..aa9983091
--- /dev/null
+++ b/encrypt-config/FlowConfigEncryptor.cpp
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "FlowConfigEncryptor.h"
+
+#include "core/extension/ExtensionManager.h"
+#include "core/FlowConfiguration.h"
+#include "core/flow/AdaptiveConfiguration.h"
+#include "core/ProcessGroup.h"
+#include "core/RepositoryFactory.h"
+#include "core/repository/VolatileContentRepository.h"
+#include "Defaults.h"
+#include "Utils.h"
+#include "utils/file/FileSystem.h"
+#include "utils/Id.h"
+
+namespace minifi = org::apache::nifi::minifi;
+
+namespace {
+enum class Type {
+ Processor,
+ ControllerService
+};
+
+struct SensitiveProperty {
+ Type type;
+ minifi::utils::Identifier component_id;
+ std::string component_name;
+ std::string property_name;
+ std::string property_display_name;
+};
+} // namespace
+
+namespace magic_enum::customize {
+template<>
+constexpr customize_t enum_name<Type>(Type type) noexcept {
+ switch (type) {
+ case Type::Processor: return "Processor";
+ case Type::ControllerService: return "Controller service";
+ }
+ return invalid_tag;
+}
+} // namespace magic_enum::customize
+
+namespace {
+std::vector<SensitiveProperty> listSensitiveProperties(const minifi::core::ProcessGroup &process_group) {
+ std::vector<SensitiveProperty> sensitive_properties;
+
+ std::vector<minifi::core::Processor *> processors;
+ process_group.getAllProcessors(processors);
+ for (const auto *processor : processors) {
+ gsl_Expects(processor);
+ for (const auto& [_, property] : processor->getProperties()) {
+ if (property.isSensitive()) {
+ sensitive_properties.push_back(SensitiveProperty{
+ .type = Type::Processor,
+ .component_id = processor->getUUID(),
+ .component_name = processor->getName(),
+ .property_name = property.getName(),
+ .property_display_name = property.getDisplayName()});
+ }
+ }
+ }
+
+ for (const auto* controller_service_node : process_group.getAllControllerServices()) {
+ gsl_Expects(controller_service_node);
+ const auto* controller_service = controller_service_node->getControllerServiceImplementation();
+ gsl_Expects(controller_service);
+ for (const auto& [_, property] : controller_service->getProperties()) {
+ if (property.isSensitive()) {
+ sensitive_properties.push_back(SensitiveProperty{
+ .type = Type::ControllerService,
+ .component_id = controller_service->getUUID(),
+ .component_name = controller_service->getName(),
+ .property_name = property.getName(),
+ .property_display_name = property.getDisplayName()});
+ }
+ }
+ }
+
+ return sensitive_properties;
+}
+
+template<typename Func>
+void encryptSensitiveValuesInFlowConfigImpl(
+ const minifi::encrypt_config::EncryptionKeys& keys, const std::filesystem::path& minifi_home, const std::filesystem::path& flow_config_path, Func create_overrides) {
+ const auto configure = std::make_shared<minifi::Configure>();
+ configure->setHome(minifi_home);
+ configure->loadConfigureFile(DEFAULT_NIFI_PROPERTIES_FILE);
+
+ bool encrypt_whole_flow_config_file = (configure->get(minifi::Configure::nifi_flow_configuration_encrypt) | minifi::utils::andThen(minifi::utils::string::toBool)).value_or(false);
+ auto encryptor = encrypt_whole_flow_config_file ? minifi::utils::crypto::EncryptionProvider::create(minifi_home) : std::nullopt;
+ auto filesystem = std::make_shared<minifi::utils::file::FileSystem>(encrypt_whole_flow_config_file, encryptor);
+
+ minifi::core::extension::ExtensionManager::get().initialize(configure);
+
+ minifi::core::flow::AdaptiveConfiguration adaptive_configuration{minifi::core::ConfigurationContext{
+ .flow_file_repo = nullptr,
+ .content_repo = nullptr,
+ .configuration = configure,
+ .path = flow_config_path,
+ .filesystem = filesystem,
+ .sensitive_properties_encryptor = minifi::utils::crypto::EncryptionProvider{minifi::utils::crypto::XSalsa20Cipher{keys.encryption_key}}
+ }};
+
+ const auto flow_config_content = filesystem->read(flow_config_path);
+ if (!flow_config_content) {
+ throw std::runtime_error(minifi::utils::string::join_pack("Could not read the flow configuration file \"", flow_config_path.string(), "\""));
+ }
+
+ const auto process_group = adaptive_configuration.getRootFromPayload(*flow_config_content);
+ gsl_Expects(process_group);
+ const auto sensitive_properties = listSensitiveProperties(*process_group);
+
+ std::unordered_map<minifi::utils::Identifier, std::unordered_map<std::string, std::string>> overrides = create_overrides(sensitive_properties);
+ if (overrides.empty()) {
+ return;
+ }
+
+ std::string flow_config_str = adaptive_configuration.serializeWithOverrides(*process_group, overrides);
+ adaptive_configuration.persist(flow_config_str);
+}
+} // namespace
+
+namespace org::apache::nifi::minifi::encrypt_config::flow_config_encryptor {
+
+void encryptSensitiveValuesInFlowConfig(const EncryptionKeys& keys, const std::filesystem::path& minifi_home, const std::filesystem::path& flow_config_path) {
+ encryptSensitiveValuesInFlowConfigImpl(keys, minifi_home, flow_config_path,
+ [](const auto& sensitive_properties) {
+ std::unordered_map<utils::Identifier, std::unordered_map<std::string, std::string>> overrides;
+ std::cout << '\n';
+ for (const auto& sensitive_property : sensitive_properties) {
+ std::cout << magic_enum::enum_name(sensitive_property.type) << " " << sensitive_property.component_name << " (" << sensitive_property.component_id.to_string() << ") "
+ << "has sensitive property " << sensitive_property.property_display_name << "\n enter a new value or press Enter to keep the current value unchanged: ";
+ std::cout.flush();
+ std::string new_value;
+ std::getline(std::cin, new_value);
+ if (!new_value.empty()) {
+ overrides[sensitive_property.component_id].emplace(sensitive_property.property_name, new_value);
+ }
+ }
+ return overrides;
+ });
+}
+
+void encryptSensitiveValuesInFlowConfig(const EncryptionKeys& keys, const std::filesystem::path& minifi_home, const std::filesystem::path& flow_config_path,
+ const std::string& component_id, const std::string& property_name, const std::string& property_value) {
+ encryptSensitiveValuesInFlowConfigImpl(keys, minifi_home, flow_config_path,
+ [&](const auto& sensitive_properties) -> std::unordered_map<utils::Identifier, std::unordered_map<std::string, std::string>> {
+ const auto sensitive_property_it = std::ranges::find_if(sensitive_properties, [&](const auto& sensitive_property) {
+ return sensitive_property.component_id.to_string() == component_id && (sensitive_property.property_name == property_name || sensitive_property.property_display_name == property_name);
+ });
+ if (sensitive_property_it == sensitive_properties.end()) {
+ std::cout << "No sensitive property found with this component ID and property name.\n";
+ return {};
+ }
+ return {{sensitive_property_it->component_id, {{sensitive_property_it->property_name, property_value}}}};
+ });
+}
+
+} // namespace org::apache::nifi::minifi::encrypt_config::flow_config_encryptor
diff --git a/libminifi/include/core/flow/AdaptiveConfiguration.h b/encrypt-config/FlowConfigEncryptor.h
similarity index 55%
copy from libminifi/include/core/flow/AdaptiveConfiguration.h
copy to encrypt-config/FlowConfigEncryptor.h
index 101aaaa8c..b4109581b 100644
--- a/libminifi/include/core/flow/AdaptiveConfiguration.h
+++ b/encrypt-config/FlowConfigEncryptor.h
@@ -1,5 +1,4 @@
/**
- *
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -18,22 +17,15 @@
#pragma once
#include <string>
-#include <memory>
-#include <vector>
-
-#include "StructuredConfiguration.h"
-
-namespace org::apache::nifi::minifi::core::flow {
+#include <filesystem>
-class AdaptiveConfiguration : public StructuredConfiguration {
- public:
- explicit AdaptiveConfiguration(ConfigurationContext ctx);
+#include "Utils.h"
+#include "core/flow/AdaptiveConfiguration.h"
- std::vector<std::string> getSupportedFormats() const override {
- return {"application/json", "text/yml"};
- }
+namespace org::apache::nifi::minifi::encrypt_config::flow_config_encryptor {
- std::unique_ptr<core::ProcessGroup> getRootFromPayload(const std::string &payload) override;
-};
+void encryptSensitiveValuesInFlowConfig(const EncryptionKeys& keys, const std::filesystem::path& minifi_home, const std::filesystem::path& flow_config_path);
+void encryptSensitiveValuesInFlowConfig(const EncryptionKeys& keys, const std::filesystem::path& minifi_home, const std::filesystem::path& flow_config_path,
+ const std::string& component_id, const std::string& property_name, const std::string& property_value);
-} // namespace org::apache::nifi::minifi::core::flow
+} // namespace org::apache::nifi::minifi::encrypt_config::flow_config_encryptor
diff --git a/encrypt-config/tests/ConfigFileEncryptorTests.cpp b/encrypt-config/tests/ConfigFileEncryptorTests.cpp
index 4a779238f..0df8878b6 100644
--- a/encrypt-config/tests/ConfigFileEncryptorTests.cpp
+++ b/encrypt-config/tests/ConfigFileEncryptorTests.cpp
@@ -80,7 +80,7 @@ TEST_CASE("ConfigFileEncryptor can encrypt the sensitive properties", "[encrypt-
REQUIRE(test_file.size() == 110);
REQUIRE(check_encryption(test_file, Configuration::nifi_rest_api_password, original_password.length()));
- SECTION("calling encryptSensitiveProperties a second time does nothing") {
+ SECTION("calling encryptSensitiveValuesInMinifiProperties a second time does nothing") {
ConfigFile test_file_copy = test_file;
uint32_t num_properties_encrypted = encryptSensitivePropertiesInFile(test_file, KEY);
diff --git a/libminifi/include/core/FlowConfiguration.h b/libminifi/include/core/FlowConfiguration.h
index 0a93c1c73..9cc746180 100644
--- a/libminifi/include/core/FlowConfiguration.h
+++ b/libminifi/include/core/FlowConfiguration.h
@@ -107,6 +107,7 @@ class FlowConfiguration : public CoreComponent {
}
bool persist(const core::ProcessGroup& process_group);
+ bool persist(const std::string& serialized_flow);
/**
* Returns the configuration path string
@@ -150,7 +151,6 @@ class FlowConfiguration : public CoreComponent {
utils::ChecksumCalculator checksum_calculator_;
private:
- bool persist(const std::string& serialized_flow);
virtual std::string serialize(const ProcessGroup&) { return ""; }
std::shared_ptr<logging::Logger> logger_;
diff --git a/libminifi/include/core/ProcessGroup.h b/libminifi/include/core/ProcessGroup.h
index 23741f955..b43fe7790 100644
--- a/libminifi/include/core/ProcessGroup.h
+++ b/libminifi/include/core/ProcessGroup.h
@@ -205,6 +205,8 @@ class ProcessGroup : public CoreComponent {
*/
std::shared_ptr<core::controller::ControllerServiceNode> findControllerService(const std::string &nodeId) const;
+ std::vector<const core::controller::ControllerServiceNode*> getAllControllerServices() const;
+
// update property value
void updatePropertyValue(const std::string& processorName, const std::string& propertyName, const std::string& propertyValue);
diff --git a/libminifi/include/core/controller/ControllerServiceNode.h b/libminifi/include/core/controller/ControllerServiceNode.h
index 2371d32b1..5b3c70663 100644
--- a/libminifi/include/core/controller/ControllerServiceNode.h
+++ b/libminifi/include/core/controller/ControllerServiceNode.h
@@ -79,6 +79,7 @@ class ControllerServiceNode : public CoreComponent, public ConfigurableComponent
* @return the implementation of the Controller Service
*/
std::shared_ptr<ControllerService> &getControllerServiceImplementation();
+ const ControllerService* getControllerServiceImplementation() const;
std::vector<std::shared_ptr<ControllerServiceNode> > &getLinkedControllerServices();
std::vector<std::shared_ptr<ConfigurableComponent> > &getLinkedComponents();
diff --git a/libminifi/include/core/flow/AdaptiveConfiguration.h b/libminifi/include/core/flow/AdaptiveConfiguration.h
index 101aaaa8c..bc1ae9fbe 100644
--- a/libminifi/include/core/flow/AdaptiveConfiguration.h
+++ b/libminifi/include/core/flow/AdaptiveConfiguration.h
@@ -34,6 +34,8 @@ class AdaptiveConfiguration : public StructuredConfiguration {
}
std::unique_ptr<core::ProcessGroup> getRootFromPayload(const std::string &payload) override;
+
+ std::string serializeWithOverrides(const core::ProcessGroup& process_group, const std::unordered_map<utils::Identifier, std::unordered_map<std::string, std::string>>& overrides) const;
};
} // namespace org::apache::nifi::minifi::core::flow
diff --git a/libminifi/include/core/flow/FlowSerializer.h b/libminifi/include/core/flow/FlowSerializer.h
index c10b69ba3..02ec6c1cc 100644
--- a/libminifi/include/core/flow/FlowSerializer.h
+++ b/libminifi/include/core/flow/FlowSerializer.h
@@ -17,10 +17,12 @@
#pragma once
#include <string>
+#include <unordered_map>
#include "core/flow/FlowSchema.h"
#include "core/ProcessGroup.h"
#include "utils/crypto/EncryptionProvider.h"
+#include "utils/Id.h"
namespace org::apache::nifi::minifi::core::flow {
@@ -34,7 +36,8 @@ class FlowSerializer {
FlowSerializer(FlowSerializer&&) = delete;
FlowSerializer& operator=(FlowSerializer&&) = delete;
- [[nodiscard]] virtual std::string serialize(const core::ProcessGroup& process_group, const FlowSchema& schema, const utils::crypto::EncryptionProvider& encryption_provider) const = 0;
+ [[nodiscard]] virtual std::string serialize(const core::ProcessGroup& process_group, const FlowSchema& schema, const utils::crypto::EncryptionProvider& encryption_provider,
+ const std::unordered_map<utils::Identifier, std::unordered_map<std::string, std::string>>& overrides) const = 0;
};
} // namespace org::apache::nifi::minifi::core::flow
diff --git a/libminifi/include/core/json/JsonFlowSerializer.h b/libminifi/include/core/json/JsonFlowSerializer.h
index 726c78a31..1faa95c93 100644
--- a/libminifi/include/core/json/JsonFlowSerializer.h
+++ b/libminifi/include/core/json/JsonFlowSerializer.h
@@ -25,11 +25,13 @@ class JsonFlowSerializer : public core::flow::FlowSerializer {
public:
explicit JsonFlowSerializer(rapidjson::Document document) : flow_definition_json_(std::move(document)) {}
- [[nodiscard]] std::string serialize(const core::ProcessGroup& process_group, const core::flow::FlowSchema& schema, const utils::crypto::EncryptionProvider& encryption_provider) const override;
+ [[nodiscard]] std::string serialize(const core::ProcessGroup& process_group, const core::flow::FlowSchema& schema, const utils::crypto::EncryptionProvider& encryption_provider,
+ const std::unordered_map<utils::Identifier, std::unordered_map<std::string, std::string>>& overrides) const override;
private:
void encryptSensitiveProperties(rapidjson::Value& property_jsons, rapidjson::Document::AllocatorType& alloc,
- const std::map<std::string, Property>& properties, const utils::crypto::EncryptionProvider& encryption_provider) const;
+ const std::map<std::string, Property>& properties, const utils::crypto::EncryptionProvider& encryption_provider,
+ std::unordered_map<std::string, std::string> component_overrides) const;
rapidjson::Document flow_definition_json_;
std::shared_ptr<logging::Logger> logger_{logging::LoggerFactory<JsonFlowSerializer>::getLogger()};
diff --git a/libminifi/include/core/yaml/YamlFlowSerializer.h b/libminifi/include/core/yaml/YamlFlowSerializer.h
index 6fcb90124..3483b1638 100644
--- a/libminifi/include/core/yaml/YamlFlowSerializer.h
+++ b/libminifi/include/core/yaml/YamlFlowSerializer.h
@@ -25,10 +25,12 @@ class YamlFlowSerializer : public core::flow::FlowSerializer {
public:
explicit YamlFlowSerializer(const YAML::Node& flow_definition_yaml) : flow_definition_yaml_(flow_definition_yaml) {}
- [[nodiscard]] std::string serialize(const core::ProcessGroup& process_group, const core::flow::FlowSchema& schema, const utils::crypto::EncryptionProvider& encryption_provider) const override;
+ [[nodiscard]] std::string serialize(const core::ProcessGroup& process_group, const core::flow::FlowSchema& schema, const utils::crypto::EncryptionProvider& encryption_provider,
+ const std::unordered_map<utils::Identifier, std::unordered_map<std::string, std::string>>& overrides) const override;
private:
- void encryptSensitiveProperties(YAML::Node property_yamls, const std::map<std::string, Property>& properties, const utils::crypto::EncryptionProvider& encryption_provider) const;
+ void encryptSensitiveProperties(YAML::Node property_yamls, const std::map<std::string, Property>& properties, const utils::crypto::EncryptionProvider& encryption_provider,
+ std::unordered_map<std::string, std::string> component_overrides) const;
YAML::Node flow_definition_yaml_;
std::shared_ptr<logging::Logger> logger_{logging::LoggerFactory<YamlFlowSerializer>::getLogger()};
diff --git a/libminifi/src/core/ProcessGroup.cpp b/libminifi/src/core/ProcessGroup.cpp
index e0918b974..26e5142e1 100644
--- a/libminifi/src/core/ProcessGroup.cpp
+++ b/libminifi/src/core/ProcessGroup.cpp
@@ -270,6 +270,14 @@ std::shared_ptr<core::controller::ControllerServiceNode> ProcessGroup::findContr
return controller_service_map_.getControllerServiceNode(nodeId);
}
+std::vector<const core::controller::ControllerServiceNode*> ProcessGroup::getAllControllerServices() const {
+ std::vector<const core::controller::ControllerServiceNode*> controller_service_nodes;
+ for (const auto& node : controller_service_map_.getAllControllerServices()) {
+ controller_service_nodes.push_back(node.get());
+ }
+ return controller_service_nodes;
+}
+
void ProcessGroup::getAllProcessors(std::vector<Processor*>& processor_vec) const {
std::lock_guard<std::recursive_mutex> lock(mutex_);
diff --git a/libminifi/src/core/controller/ControllerServiceNode.cpp b/libminifi/src/core/controller/ControllerServiceNode.cpp
index 309757417..f600e280f 100644
--- a/libminifi/src/core/controller/ControllerServiceNode.cpp
+++ b/libminifi/src/core/controller/ControllerServiceNode.cpp
@@ -20,17 +20,16 @@
#include <memory>
#include <vector>
-namespace org {
-namespace apache {
-namespace nifi {
-namespace minifi {
-namespace core {
-namespace controller {
+namespace org::apache::nifi::minifi::core::controller {
std::shared_ptr<ControllerService> &ControllerServiceNode::getControllerServiceImplementation() {
return controller_service_;
}
+const ControllerService* ControllerServiceNode::getControllerServiceImplementation() const {
+ return controller_service_.get();
+}
+
std::vector<std::shared_ptr<ControllerServiceNode> > &ControllerServiceNode::getLinkedControllerServices() {
return linked_controller_services_;
}
@@ -39,9 +38,4 @@ std::vector<std::shared_ptr<ConfigurableComponent> > &ControllerServiceNode::get
return linked_components_;
}
-} /* namespace controller */
-} /* namespace core */
-} /* namespace minifi */
-} /* namespace nifi */
-} /* namespace apache */
-} /* namespace org */
+} // namespace org::apache::nifi::minifi::core::controller
diff --git a/libminifi/src/core/flow/AdaptiveConfiguration.cpp b/libminifi/src/core/flow/AdaptiveConfiguration.cpp
index 0a75e42ae..6eac01fbb 100644
--- a/libminifi/src/core/flow/AdaptiveConfiguration.cpp
+++ b/libminifi/src/core/flow/AdaptiveConfiguration.cpp
@@ -71,4 +71,10 @@ std::unique_ptr<core::ProcessGroup> AdaptiveConfiguration::getRootFromPayload(co
}
}
+std::string AdaptiveConfiguration::serializeWithOverrides(const core::ProcessGroup& process_group,
+ const std::unordered_map<utils::Identifier, std::unordered_map<std::string, std::string>>& overrides) const {
+ gsl_Expects(flow_serializer_);
+ return flow_serializer_->serialize(process_group, schema_, sensitive_properties_encryptor_, overrides);
+}
+
} // namespace org::apache::nifi::minifi::core::flow
diff --git a/libminifi/src/core/flow/StructuredConfiguration.cpp b/libminifi/src/core/flow/StructuredConfiguration.cpp
index 461c950bc..df79c9b6d 100644
--- a/libminifi/src/core/flow/StructuredConfiguration.cpp
+++ b/libminifi/src/core/flow/StructuredConfiguration.cpp
@@ -914,7 +914,7 @@ void StructuredConfiguration::addNewId(const std::string& uuid) {
std::string StructuredConfiguration::serialize(const core::ProcessGroup& process_group) {
gsl_Expects(flow_serializer_);
- return flow_serializer_->serialize(process_group, schema_, sensitive_properties_encryptor_);
+ return flow_serializer_->serialize(process_group, schema_, sensitive_properties_encryptor_, {});
}
} // namespace org::apache::nifi::minifi::core::flow
diff --git a/libminifi/src/core/json/JsonFlowSerializer.cpp b/libminifi/src/core/json/JsonFlowSerializer.cpp
index a68605d66..82d6d961c 100644
--- a/libminifi/src/core/json/JsonFlowSerializer.cpp
+++ b/libminifi/src/core/json/JsonFlowSerializer.cpp
@@ -38,7 +38,7 @@ rapidjson::Value& getMember(rapidjson::Value& node, const std::string& member_na
}
void JsonFlowSerializer::encryptSensitiveProperties(rapidjson::Value &property_jsons, rapidjson::Document::AllocatorType &alloc,
- const std::map<std::string, Property> &properties, const utils::crypto::EncryptionProvider &encryption_provider) const {
+ const std::map<std::string, Property> &properties, const utils::crypto::EncryptionProvider &encryption_provider, std::unordered_map<std::string, std::string> component_overrides) const {
for (auto &property : property_jsons.GetObject()) {
const std::string name{property.name.GetString(), property.name.GetStringLength()};
if (!properties.contains(name)) {
@@ -47,14 +47,22 @@ void JsonFlowSerializer::encryptSensitiveProperties(rapidjson::Value &property_j
}
if (properties.at(name).isSensitive()) {
auto& value = property.value;
- const std::string_view value_sv{value.GetString(), value.GetStringLength()};
+ const std::string_view value_sv = component_overrides.contains(name) ? component_overrides.at(name) : std::string_view{value.GetString(), value.GetStringLength()};
const std::string encrypted_value = utils::crypto::property_encryption::encrypt(value_sv, encryption_provider);
value.SetString(encrypted_value.c_str(), encrypted_value.size(), alloc);
}
+ component_overrides.erase(name);
+ }
+
+ for (const auto& [name, value] : component_overrides) {
+ gsl_Expects(properties.contains(name) && properties.at(name).isSensitive());
+ const std::string encrypted_value = utils::crypto::property_encryption::encrypt(value, encryption_provider);
+ property_jsons.AddMember(rapidjson::Value(name, alloc), rapidjson::Value(encrypted_value, alloc), alloc);
}
}
-std::string JsonFlowSerializer::serialize(const core::ProcessGroup &process_group, const core::flow::FlowSchema &schema, const utils::crypto::EncryptionProvider &encryption_provider) const {
+std::string JsonFlowSerializer::serialize(const core::ProcessGroup &process_group, const core::flow::FlowSchema &schema, const utils::crypto::EncryptionProvider &encryption_provider,
+ const std::unordered_map<utils::Identifier, std::unordered_map<std::string, std::string>>& overrides) const {
gsl_Expects(schema.root_group.size() == 1 && schema.identifier.size() == 1 &&
schema.processors.size() == 1 && schema.processor_properties.size() == 1 &&
schema.controller_services.size() == 1 && schema.controller_service_properties.size() == 1);
@@ -78,23 +86,32 @@ std::string JsonFlowSerializer::serialize(const core::ProcessGroup &process_grou
logger_->log_warn("Processor {} not found in the flow definition", processor_id->to_string());
continue;
}
- encryptSensitiveProperties(getMember(processor_json, schema.processor_properties[0]), alloc, processor->getProperties(), encryption_provider);
+ const auto& processor_overrides = overrides.contains(*processor_id) ? overrides.at(*processor_id) : std::unordered_map<std::string, std::string>{};
+ encryptSensitiveProperties(getMember(processor_json, schema.processor_properties[0]), alloc, processor->getProperties(), encryption_provider,
+ processor_overrides);
}
auto controller_services = getMember(root_group, schema.controller_services[0]).GetArray();
for (auto &controller_service_json : controller_services) {
- const std::string controller_service_id{getMember(controller_service_json, schema.identifier[0]).GetString(), getMember(controller_service_json, schema.identifier[0]).GetStringLength()};
- const auto controller_service_node = process_group.findControllerService(controller_service_id);
+ const std::string controller_service_id_str{getMember(controller_service_json, schema.identifier[0]).GetString(), getMember(controller_service_json, schema.identifier[0]).GetStringLength()};
+ const auto controller_service_id = utils::Identifier::parse(controller_service_id_str);
+ if (!controller_service_id) {
+ logger_->log_warn("Invalid controller service ID found in the flow definition: {}", controller_service_id_str);
+ continue;
+ }
+ const auto controller_service_node = process_group.findControllerService(controller_service_id_str);
if (!controller_service_node) {
- logger_->log_warn("Controller service node {} not found in the flow definition", controller_service_id);
+ logger_->log_warn("Controller service node {} not found in the flow definition", controller_service_id_str);
continue;
}
const auto controller_service = controller_service_node->getControllerServiceImplementation();
if (!controller_service) {
- logger_->log_warn("Controller service {} not found in the flow definition", controller_service_id);
+ logger_->log_warn("Controller service {} not found in the flow definition", controller_service_id_str);
continue;
}
- encryptSensitiveProperties(getMember(controller_service_json, schema.controller_service_properties[0]), alloc, controller_service->getProperties(), encryption_provider);
+ const auto& controller_service_overrides = overrides.contains(*controller_service_id) ? overrides.at(*controller_service_id) : std::unordered_map<std::string, std::string>{};
+ encryptSensitiveProperties(getMember(controller_service_json, schema.controller_service_properties[0]), alloc, controller_service->getProperties(), encryption_provider,
+ controller_service_overrides);
}
rapidjson::StringBuffer buffer;
diff --git a/libminifi/src/core/yaml/YamlFlowSerializer.cpp b/libminifi/src/core/yaml/YamlFlowSerializer.cpp
index b61f67ae3..00bd87db3 100644
--- a/libminifi/src/core/yaml/YamlFlowSerializer.cpp
+++ b/libminifi/src/core/yaml/YamlFlowSerializer.cpp
@@ -21,7 +21,8 @@
namespace org::apache::nifi::minifi::core::yaml {
-void YamlFlowSerializer::encryptSensitiveProperties(YAML::Node property_yamls, const std::map<std::string, Property>& properties, const utils::crypto::EncryptionProvider& encryption_provider) const {
+void YamlFlowSerializer::encryptSensitiveProperties(YAML::Node property_yamls, const std::map<std::string, Property>& properties, const utils::crypto::EncryptionProvider& encryption_provider,
+ std::unordered_map<std::string, std::string> component_overrides) const {
for (auto kv : property_yamls) {
auto name = kv.first.as<std::string>();
if (!properties.contains(name)) {
@@ -31,18 +32,25 @@ void YamlFlowSerializer::encryptSensitiveProperties(YAML::Node property_yamls, c
if (properties.at(name).isSensitive()) {
if (kv.second.IsSequence()) {
for (auto property_item : kv.second) {
- auto value = property_item["value"].as<std::string>();
+ auto value = component_overrides.contains(name) ? component_overrides.at(name) : property_item["value"].as<std::string>();
property_item["value"] = utils::crypto::property_encryption::encrypt(value, encryption_provider);
}
} else {
- auto value = kv.second.as<std::string>();
+ auto value = component_overrides.contains(name) ? component_overrides.at(name) : kv.second.as<std::string>();
property_yamls[name] = utils::crypto::property_encryption::encrypt(value, encryption_provider);
}
+ component_overrides.erase(name);
}
}
+
+ for (const auto& [name, value] : component_overrides) {
+ gsl_Expects(properties.contains(name) && properties.at(name).isSensitive());
+ property_yamls[name] = utils::crypto::property_encryption::encrypt(value, encryption_provider);
+ }
}
-std::string YamlFlowSerializer::serialize(const core::ProcessGroup& process_group, const core::flow::FlowSchema& schema, const utils::crypto::EncryptionProvider& encryption_provider) const {
+std::string YamlFlowSerializer::serialize(const core::ProcessGroup& process_group, const core::flow::FlowSchema& schema, const utils::crypto::EncryptionProvider& encryption_provider,
+ const std::unordered_map<utils::Identifier, std::unordered_map<std::string, std::string>>& overrides) const {
gsl_Expects(schema.identifier.size() == 1 &&
schema.processors.size() == 1 && schema.processor_properties.size() == 1 &&
schema.controller_services.size() == 1 && schema.controller_service_properties.size() == 1);
@@ -60,22 +68,29 @@ std::string YamlFlowSerializer::serialize(const core::ProcessGroup& process_grou
logger_->log_warn("Processor {} not found in the flow definition", processor_id->to_string());
continue;
}
- encryptSensitiveProperties(processor_yaml[schema.processor_properties[0]], processor->getProperties(), encryption_provider);
+ const auto& processor_overrides = overrides.contains(*processor_id) ? overrides.at(*processor_id) : std::unordered_map<std::string, std::string>{};
+ encryptSensitiveProperties(processor_yaml[schema.processor_properties[0]], processor->getProperties(), encryption_provider, processor_overrides);
}
for (auto controller_service_yaml : flow_definition_yaml[schema.controller_services[0]]) {
- const auto controller_service_id = controller_service_yaml[schema.identifier[0]].Scalar();
- const auto controller_service_node = process_group.findControllerService(controller_service_id);
+ const auto controller_service_id_str = controller_service_yaml[schema.identifier[0]].Scalar();
+ const auto controller_service_id = utils::Identifier::parse(controller_service_id_str);
+ if (!controller_service_id) {
+ logger_->log_warn("Invalid controller service ID found in the flow definition: {}", controller_service_id_str);
+ continue;
+ }
+ const auto controller_service_node = process_group.findControllerService(controller_service_id_str);
if (!controller_service_node) {
- logger_->log_warn("Controller service node {} not found in the flow definition", controller_service_id);
+ logger_->log_warn("Controller service node {} not found in the flow definition", controller_service_id_str);
continue;
}
const auto controller_service = controller_service_node->getControllerServiceImplementation();
if (!controller_service) {
- logger_->log_warn("Controller service {} not found in the flow definition", controller_service_id);
+ logger_->log_warn("Controller service {} not found in the flow definition", controller_service_id_str);
continue;
}
- encryptSensitiveProperties(controller_service_yaml[schema.controller_service_properties[0]], controller_service->getProperties(), encryption_provider);
+ const auto& controller_service_overrides = overrides.contains(*controller_service_id) ? overrides.at(*controller_service_id) : std::unordered_map<std::string, std::string>{};
+ encryptSensitiveProperties(controller_service_yaml[schema.controller_service_properties[0]], controller_service->getProperties(), encryption_provider, controller_service_overrides);
}
return YAML::Dump(flow_definition_yaml) + '\n';
diff --git a/libminifi/test/unit/JsonFlowSerializerTests.cpp b/libminifi/test/unit/JsonFlowSerializerTests.cpp
index 3412376a7..49cd913aa 100644
--- a/libminifi/test/unit/JsonFlowSerializerTests.cpp
+++ b/libminifi/test/unit/JsonFlowSerializerTests.cpp
@@ -19,6 +19,7 @@
#include "../Catch.h"
#include "../ConfigurationTestController.h"
+#include "catch2/generators/catch_generators.hpp"
#include "core/flow/FlowSchema.h"
#include "core/json/JsonFlowSerializer.h"
#include "core/json/JsonNode.h"
@@ -104,7 +105,8 @@ constexpr std::string_view config_json_with_default_schema = R"({
}
)";
-constexpr std::string_view config_json_with_nifi_schema = R"({
+// in two parts because Visual Studio doesn't like long string constants
+constexpr std::string_view config_json_with_nifi_schema_part_1 = R"({
"encodingVersion": {
"majorVersion": 2,
"minorVersion": 0
@@ -390,7 +392,9 @@ constexpr std::string_view config_json_with_nifi_schema = R"({
}
],
"labels": [],
- "funnels": [],
+ "funnels": [],)";
+
+constexpr std::string_view config_json_with_nifi_schema_part_2 = R"(
"controllerServices": [
{
"identifier": "b9801278-7b5d-4314-aed6-713fd4b5f933",
@@ -452,6 +456,66 @@ constexpr std::string_view config_json_with_nifi_schema = R"({
],
"componentType": "CONTROLLER_SERVICE",
"groupIdentifier": "8b4d66dc-9085-4722-b35b-3492f363baa3"
+ },
+ {
+ "identifier": "b418f4ff-e598-4ea2-921f-14f9dd864482",
+ "instanceIdentifier": "dbb76c00-97ad-4b3a-bf0c-d5d1b88e79d3",
+ "name": "Second SSLContextService",
+ "position": {
+ "x": 0.0,
+ "y": 0.0
+ },
+ "type": "org.apache.nifi.minifi.controllers.SSLContextService",
+ "bundle": {
+ "group": "org.apache.nifi.minifi",
+ "artifact": "minifi-system",
+ "version": "1.23.06"
+ },
+ "properties": {
+ "Private Key": "second_ssl_service_certs/agent-key.pem",
+ "Client Certificate": "second_ssl_service_certs/agent-cert.pem",
+ "CA Certificate": "second_ssl_service_certs/ca-cert.pem",
+ "Use System Cert Store": "false"
+ },
+ "propertyDescriptors": {
+ "Private Key": {
+ "name": "Private Key",
+ "identifiesControllerService": false,
+ "sensitive": false
+ },
+ "Client Certificate": {
+ "name": "Client Certificate",
+ "identifiesControllerService": false,
+ "sensitive": false
+ },
+ "Passphrase": {
+ "name": "Passphrase",
+ "identifiesControllerService": false,
+ "sensitive": true
+ },
+ "CA Certificate": {
+ "name": "CA Certificate",
+ "identifiesControllerService": false,
+ "sensitive": false
+ },
+ "Use System Cert Store": {
+ "name": "Use System Cert Store",
+ "identifiesControllerService": false,
+ "sensitive": false
+ }
+ },
+ "controllerServiceApis": [
+ {
+ "type": "org.apache.nifi.minifi.controllers.SSLContextService",
+ "bundle": {
+ "group": "org.apache.nifi.minifi",
+ "artifact": "minifi-system",
+ "version": "1.23.06"
+ }
+ }
+ ],
+ "componentType": "CONTROLLER_SERVICE",
+ "groupIdentifier": "910ec043-2372-4edb-9ef4-8ce720c50685"
}
],
"variables": {},
@@ -467,16 +531,9 @@ TEST_CASE("JsonFlowSerializer can encrypt the sensitive properties") {
ConfigurationTestController test_controller;
core::flow::AdaptiveConfiguration json_configuration{test_controller.getContext()};
- core::flow::FlowSchema schema;
- std::string flow_definition;
- SECTION("Default schema") {
- schema = core::flow::FlowSchema::getDefault();
- flow_definition = std::string{config_json_with_default_schema};
- }
- SECTION("NiFi schema") {
- schema = core::flow::FlowSchema::getNiFiFlowJson();
- flow_definition = std::string{config_json_with_nifi_schema};
- }
+ const auto [schema, flow_definition] = GENERATE(
+ std::make_tuple(core::flow::FlowSchema::getDefault(), std::string{config_json_with_default_schema}),
+ std::make_tuple(core::flow::FlowSchema::getNiFiFlowJson(), utils::string::join_pack(config_json_with_nifi_schema_part_1, config_json_with_nifi_schema_part_2)));
const auto process_group = json_configuration.getRootFromPayload(flow_definition);
REQUIRE(process_group);
@@ -486,7 +543,21 @@ TEST_CASE("JsonFlowSerializer can encrypt the sensitive properties") {
REQUIRE(res);
const auto flow_serializer = core::json::JsonFlowSerializer{std::move(doc)};
- std::string config_json_encrypted = flow_serializer.serialize(*process_group, schema, encryption_provider);
+ using Overrides = std::unordered_map<utils::Identifier, std::unordered_map<std::string, std::string>>;
+ const auto processor_id = utils::Identifier::parse("469617f1-3898-4bbf-91fe-27d8f4dd2a75").value();
+ const auto controller_service_id = utils::Identifier::parse("b9801278-7b5d-4314-aed6-713fd4b5f933").value();
+
+ const auto [overrides, expected_results] = GENERATE_REF(
+ std::make_tuple(Overrides{},
+ std::array{"very_secure_password", "very_secure_passphrase"}),
+ std::make_tuple(Overrides{{processor_id, {{"invokehttp-proxy-password", "password123"}}}},
+ std::array{"password123", "very_secure_passphrase"}),
+ std::make_tuple(Overrides{{controller_service_id, {{"Passphrase", "speak friend and enter"}}}},
+ std::array{"very_secure_password", "speak friend and enter"}),
+ std::make_tuple(Overrides{{processor_id, {{"invokehttp-proxy-password", "password123"}}}, {controller_service_id, {{"Passphrase", "speak friend and enter"}}}},
+ std::array{"password123", "speak friend and enter"}));
+
+ std::string config_json_encrypted = flow_serializer.serialize(*process_group, schema, encryption_provider, overrides);
{
std::regex regex{R"_("invokehttp-proxy-password": "(enc\{.*\})",)_"};
@@ -495,7 +566,7 @@ TEST_CASE("JsonFlowSerializer can encrypt the sensitive properties") {
REQUIRE(match_results.size() == 2);
std::string encrypted_value = match_results[1];
- CHECK(utils::crypto::property_encryption::decrypt(encrypted_value, encryption_provider) == "very_secure_password");
+ CHECK(utils::crypto::property_encryption::decrypt(encrypted_value, encryption_provider) == expected_results[0]);
}
{
@@ -505,6 +576,36 @@ TEST_CASE("JsonFlowSerializer can encrypt the sensitive properties") {
REQUIRE(match_results.size() == 2);
std::string encrypted_value = match_results[1];
- CHECK(utils::crypto::property_encryption::decrypt(encrypted_value, encryption_provider) == "very_secure_passphrase");
+ CHECK(utils::crypto::property_encryption::decrypt(encrypted_value, encryption_provider) == expected_results[1]);
}
}
+
+TEST_CASE("JsonFlowSerializer with an override can add a new property to the flow config file") {
+ ConfigurationTestController test_controller;
+ core::flow::AdaptiveConfiguration json_configuration{test_controller.getContext()};
+
+ const auto schema = core::flow::FlowSchema::getNiFiFlowJson();
+ const auto config_json_with_nifi_schema = utils::string::join_pack(config_json_with_nifi_schema_part_1, config_json_with_nifi_schema_part_2);
+ const auto process_group = json_configuration.getRootFromPayload(config_json_with_nifi_schema);
+ REQUIRE(process_group);
+
+ rapidjson::Document doc;
+ rapidjson::ParseResult res = doc.Parse(config_json_with_nifi_schema.data(), config_json_with_nifi_schema.size());
+ REQUIRE(res);
+ const auto flow_serializer = core::json::JsonFlowSerializer{std::move(doc)};
+
+ const auto second_controller_service_id = utils::Identifier::parse("b418f4ff-e598-4ea2-921f-14f9dd864482").value();
+ const auto overrides = std::unordered_map<utils::Identifier, std::unordered_map<std::string, std::string>>{{second_controller_service_id, {{"Passphrase", "new passphrase"}}}};
+
+ std::string config_json_encrypted = flow_serializer.serialize(*process_group, schema, encryption_provider, overrides);
+
+ // find the second match
+ std::regex regex{R"_("Passphrase": "(enc\{.*\})")_"};
+ std::smatch match_results;
+ REQUIRE(std::regex_search(config_json_encrypted.cbegin(), config_json_encrypted.cend(), match_results, regex));
+ REQUIRE(std::regex_search(match_results.suffix().first, config_json_encrypted.cend(), match_results, regex));
+ REQUIRE(match_results.size() == 2);
+
+ std::string encrypted_value = match_results[1];
+ CHECK(utils::crypto::property_encryption::decrypt(encrypted_value, encryption_provider) == "new passphrase");
+}
diff --git a/libminifi/test/unit/YamlFlowSerializerTests.cpp b/libminifi/test/unit/YamlFlowSerializerTests.cpp
index a35162ccd..25dff1514 100644
--- a/libminifi/test/unit/YamlFlowSerializerTests.cpp
+++ b/libminifi/test/unit/YamlFlowSerializerTests.cpp
@@ -19,6 +19,7 @@
#include "../Catch.h"
#include "../ConfigurationTestController.h"
+#include "catch2/generators/catch_generators.hpp"
#include "core/flow/FlowSchema.h"
#include "core/yaml/YamlFlowSerializer.h"
#include "core/yaml/YamlNode.h"
@@ -142,6 +143,14 @@ Controller Services:
Passphrase: very_secure_passphrase
Private Key: certs/agent-key.pem
Use System Cert Store: false
+ - id: b418f4ff-e598-4ea2-921f-14f9dd864482
+ name: Second SSLContextService
+ type: org.apache.nifi.minifi.controllers.SSLContextService
+ Properties:
+ CA Certificate: second_ssl_service_certs/ca-cert.pem
+ Client Certificate: second_ssl_service_certs/agent-cert.pem
+ Private Key: second_ssl_service_certs/agent-key.pem
+ Use System Cert Store: false
Process Groups: []
Input Ports: []
Output Ports: []
@@ -175,7 +184,21 @@ TEST_CASE("YamlFlowSerializer can encrypt the sensitive properties") {
YAML::Node root_yaml_node = YAML::Load(std::string{config_yaml});
const auto flow_serializer = core::yaml::YamlFlowSerializer{root_yaml_node};
- std::string config_yaml_encrypted = flow_serializer.serialize(*process_group, schema, encryption_provider);
+ using Overrides = std::unordered_map<utils::Identifier, std::unordered_map<std::string, std::string>>;
+ const auto processor_id = utils::Identifier::parse("469617f1-3898-4bbf-91fe-27d8f4dd2a75").value();
+ const auto controller_service_id = utils::Identifier::parse("b9801278-7b5d-4314-aed6-713fd4b5f933").value();
+
+ const auto [overrides, expected_results] = GENERATE_REF(
+ std::make_tuple(Overrides{},
+ std::array{"very_secure_password", "very_secure_passphrase"}),
+ std::make_tuple(Overrides{{processor_id, {{"invokehttp-proxy-password", "password123"}}}},
+ std::array{"password123", "very_secure_passphrase"}),
+ std::make_tuple(Overrides{{controller_service_id, {{"Passphrase", "speak friend and enter"}}}},
+ std::array{"very_secure_password", "speak friend and enter"}),
+ std::make_tuple(Overrides{{processor_id, {{"invokehttp-proxy-password", "password123"}}}, {controller_service_id, {{"Passphrase", "speak friend and enter"}}}},
+ std::array{"password123", "speak friend and enter"}));
+
+ std::string config_yaml_encrypted = flow_serializer.serialize(*process_group, schema, encryption_provider, overrides);
{
std::regex regex{R"_(invokehttp-proxy-password: (enc\{.*\}))_"};
@@ -184,7 +207,7 @@ TEST_CASE("YamlFlowSerializer can encrypt the sensitive properties") {
REQUIRE(match_results.size() == 2);
std::string encrypted_value = match_results[1];
- CHECK(utils::crypto::property_encryption::decrypt(encrypted_value, encryption_provider) == "very_secure_password");
+ CHECK(utils::crypto::property_encryption::decrypt(encrypted_value, encryption_provider) == expected_results[0]);
}
{
@@ -194,6 +217,33 @@ TEST_CASE("YamlFlowSerializer can encrypt the sensitive properties") {
REQUIRE(match_results.size() == 2);
std::string encrypted_value = match_results[1];
- CHECK(utils::crypto::property_encryption::decrypt(encrypted_value, encryption_provider) == "very_secure_passphrase");
+ CHECK(utils::crypto::property_encryption::decrypt(encrypted_value, encryption_provider) == expected_results[1]);
}
}
+
+TEST_CASE("YamlFlowSerializer with an override can add a new property to the flow config file") {
+ ConfigurationTestController test_controller;
+ core::YamlConfiguration yaml_configuration{test_controller.getContext()};
+ const auto process_group = yaml_configuration.getRootFromPayload(std::string{config_yaml});
+ REQUIRE(process_group);
+
+ const auto schema = core::flow::FlowSchema::getDefault();
+
+ YAML::Node root_yaml_node = YAML::Load(std::string{config_yaml});
+ const auto flow_serializer = core::yaml::YamlFlowSerializer{root_yaml_node};
+
+ const auto second_controller_service_id = utils::Identifier::parse("b418f4ff-e598-4ea2-921f-14f9dd864482").value();
+ const auto overrides = std::unordered_map<utils::Identifier, std::unordered_map<std::string, std::string>>{{second_controller_service_id, {{"Passphrase", "new passphrase"}}}};
+
+ std::string config_yaml_encrypted = flow_serializer.serialize(*process_group, schema, encryption_provider, overrides);
+
+ // find the second match
+ std::regex regex{R"_(Passphrase: (enc\{.*\}))_"};
+ std::smatch match_results;
+ REQUIRE(std::regex_search(config_yaml_encrypted.cbegin(), config_yaml_encrypted.cend(), match_results, regex));
+ REQUIRE(std::regex_search(match_results.suffix().first, config_yaml_encrypted.cend(), match_results, regex));
+ REQUIRE(match_results.size() == 2);
+
+ std::string encrypted_value = match_results[1];
+ CHECK(utils::crypto::property_encryption::decrypt(encrypted_value, encryption_provider) == "new passphrase");
+}