You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ex...@apache.org on 2021/08/06 12:40:48 UTC
[nifi] branch main updated: NIFI-8696: Added HashiCorp Vault
KeyValue SPP
This is an automated email from the ASF dual-hosted git repository.
exceptionfactory pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new cc1e966 NIFI-8696: Added HashiCorp Vault KeyValue SPP
cc1e966 is described below
commit cc1e9665cda8ae91c2c067f567013efe69fe448c
Author: Joe Gresock <jg...@gmail.com>
AuthorDate: Tue Jul 27 21:30:02 2021 -0400
NIFI-8696: Added HashiCorp Vault KeyValue SPP
This closes #5255
Signed-off-by: David Handermann <ex...@apache.org>
---
...actHashiCorpVaultSensitivePropertyProvider.java | 4 +-
...orpVaultKeyValueSensitivePropertyProvider.java} | 54 +++++-----
...iCorpVaultTransitSensitivePropertyProvider.java | 5 -
.../nifi/properties/PropertyProtectionScheme.java | 1 +
.../StandardSensitivePropertyProviderFactory.java | 2 +
.../HashiCorpVaultCommunicationService.java | 32 ++++--
...StandardHashiCorpVaultCommunicationService.java | 61 ++++++++++-
...andardHashiCorpVaultCommunicationServiceIT.java | 17 ++-
.../src/main/asciidoc/administration-guide.adoc | 117 ++++++++++++++++++++-
nifi-docs/src/main/asciidoc/toolkit-guide.adoc | 49 +--------
.../resources/conf/bootstrap-hashicorp-vault.conf | 3 +
.../resources/conf/bootstrap-hashicorp-vault.conf | 3 +
.../src/main/resources/conf/bootstrap.conf | 2 +-
13 files changed, 256 insertions(+), 94 deletions(-)
diff --git a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/AbstractHashiCorpVaultSensitivePropertyProvider.java b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/AbstractHashiCorpVaultSensitivePropertyProvider.java
index 4570608..63c8c62 100644
--- a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/AbstractHashiCorpVaultSensitivePropertyProvider.java
+++ b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/AbstractHashiCorpVaultSensitivePropertyProvider.java
@@ -116,7 +116,9 @@ public abstract class AbstractHashiCorpVaultSensitivePropertyProvider extends Ab
* @param vaultBootstrapProperties The Vault-specific bootstrap properties
* @return true if the relevant Secrets Engine-specific properties are configured
*/
- protected abstract boolean hasRequiredSecretsEngineProperties(final BootstrapProperties vaultBootstrapProperties);
+ protected boolean hasRequiredSecretsEngineProperties(final BootstrapProperties vaultBootstrapProperties) {
+ return getSecretsEnginePath(vaultBootstrapProperties) != null;
+ }
/**
* Returns the key used to identify the provider implementation in {@code nifi.properties},
diff --git a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/HashiCorpVaultTransitSensitivePropertyProvider.java b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/HashiCorpVaultKeyValueSensitivePropertyProvider.java
similarity index 55%
copy from nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/HashiCorpVaultTransitSensitivePropertyProvider.java
copy to nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/HashiCorpVaultKeyValueSensitivePropertyProvider.java
index 6c7efd2..d373a5a 100644
--- a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/HashiCorpVaultTransitSensitivePropertyProvider.java
+++ b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/HashiCorpVaultKeyValueSensitivePropertyProvider.java
@@ -18,17 +18,16 @@ package org.apache.nifi.properties;
import org.apache.commons.lang3.StringUtils;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
+import java.util.Objects;
/**
- * Uses the HashiCorp Vault Transit Secrets Engine to encrypt sensitive values at rest.
+ * Uses the HashiCorp Vault Key/Value (unversioned) Secrets Engine to store sensitive values.
*/
-public class HashiCorpVaultTransitSensitivePropertyProvider extends AbstractHashiCorpVaultSensitivePropertyProvider {
- private static final Charset PROPERTY_CHARSET = StandardCharsets.UTF_8;
- private static final String TRANSIT_PATH = "vault.transit.path";
+public class HashiCorpVaultKeyValueSensitivePropertyProvider extends AbstractHashiCorpVaultSensitivePropertyProvider {
- HashiCorpVaultTransitSensitivePropertyProvider(final BootstrapProperties bootstrapProperties) {
+ private static final String KEY_VALUE_PATH = "vault.kv.path";
+
+ HashiCorpVaultKeyValueSensitivePropertyProvider(final BootstrapProperties bootstrapProperties) {
super(bootstrapProperties);
}
@@ -37,58 +36,55 @@ public class HashiCorpVaultTransitSensitivePropertyProvider extends AbstractHash
if (vaultBootstrapProperties == null) {
return null;
}
- final String transitPath = vaultBootstrapProperties.getProperty(TRANSIT_PATH);
+ final String kvPath = vaultBootstrapProperties.getProperty(KEY_VALUE_PATH);
// Validate transit path
try {
- PropertyProtectionScheme.fromIdentifier(getProtectionScheme().getIdentifier(transitPath));
+ PropertyProtectionScheme.fromIdentifier(getProtectionScheme().getIdentifier(kvPath));
} catch (IllegalArgumentException e) {
- throw new SensitivePropertyProtectionException(String.format("%s [%s] contains unsupported characters", TRANSIT_PATH, transitPath), e);
+ throw new SensitivePropertyProtectionException(String.format("%s [%s] contains unsupported characters", KEY_VALUE_PATH, kvPath), e);
}
- return transitPath;
+ return kvPath;
}
@Override
protected PropertyProtectionScheme getProtectionScheme() {
- return PropertyProtectionScheme.HASHICORP_VAULT_TRANSIT;
- }
-
- @Override
- protected boolean hasRequiredSecretsEngineProperties(final BootstrapProperties vaultBootstrapProperties) {
- return getSecretsEnginePath(vaultBootstrapProperties) != null;
+ return PropertyProtectionScheme.HASHICORP_VAULT_KV;
}
/**
- * Returns the encrypted cipher text.
+ * Stores the sensitive value in Vault and returns a description of the secret.
*
* @param unprotectedValue the sensitive value
* @param context The property context, unused in this provider
* @return the value to persist in the {@code nifi.properties} file
- * @throws SensitivePropertyProtectionException if there is an exception encrypting the value
+ * @throws SensitivePropertyProtectionException if there is an exception writing the secret
*/
@Override
public String protect(final String unprotectedValue, final ProtectedPropertyContext context) throws SensitivePropertyProtectionException {
if (StringUtils.isBlank(unprotectedValue)) {
- throw new IllegalArgumentException("Cannot encrypt an empty value");
+ throw new IllegalArgumentException("Cannot protect an empty value");
}
+ Objects.requireNonNull(context, "Context is required to protect a value");
- return getVaultCommunicationService().encrypt(getPath(), unprotectedValue.getBytes(PROPERTY_CHARSET));
+ getVaultCommunicationService().writeKeyValueSecret(getPath(), context.getContextKey(), unprotectedValue);
+ return String.format("%s/%s", getPath(), context.getContextKey());
}
/**
- * Returns the decrypted plaintext.
+ * Returns the secret value, as read from Vault.
*
- * @param protectedValue the cipher text read from the {@code nifi.properties} file
- * @param context The property context, unused in this provider
+ * @param protectedValue The value read from {@code nifi.properties} file. Ignored in this provider.
+ * @param context The property context, from which the Vault secret name is pulled
* @return the raw value to be used by the application
- * @throws SensitivePropertyProtectionException if there is an error decrypting the cipher text
+ * @throws SensitivePropertyProtectionException if there is an error retrieving the scret
*/
@Override
public String unprotect(final String protectedValue, final ProtectedPropertyContext context) throws SensitivePropertyProtectionException {
- if (StringUtils.isBlank(protectedValue)) {
- throw new IllegalArgumentException("Cannot decrypt an empty value");
- }
+ Objects.requireNonNull(context, "Context is required to unprotect a value");
- return new String(getVaultCommunicationService().decrypt(getPath(), protectedValue), PROPERTY_CHARSET);
+ return getVaultCommunicationService().readKeyValueSecret(getPath(), context.getContextKey())
+ .orElseThrow(() -> new SensitivePropertyProtectionException(String
+ .format("Secret [%s] not found in Vault Key/Value engine at [%s]", context.getContextKey(), getPath())));
}
}
diff --git a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/HashiCorpVaultTransitSensitivePropertyProvider.java b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/HashiCorpVaultTransitSensitivePropertyProvider.java
index 6c7efd2..452ee14 100644
--- a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/HashiCorpVaultTransitSensitivePropertyProvider.java
+++ b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/HashiCorpVaultTransitSensitivePropertyProvider.java
@@ -53,11 +53,6 @@ public class HashiCorpVaultTransitSensitivePropertyProvider extends AbstractHash
return PropertyProtectionScheme.HASHICORP_VAULT_TRANSIT;
}
- @Override
- protected boolean hasRequiredSecretsEngineProperties(final BootstrapProperties vaultBootstrapProperties) {
- return getSecretsEnginePath(vaultBootstrapProperties) != null;
- }
-
/**
* Returns the encrypted cipher text.
*
diff --git a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/PropertyProtectionScheme.java b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/PropertyProtectionScheme.java
index 0e2a72c..ee3859b 100644
--- a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/PropertyProtectionScheme.java
+++ b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/PropertyProtectionScheme.java
@@ -26,6 +26,7 @@ import java.util.Objects;
public enum PropertyProtectionScheme {
AES_GCM("aes/gcm/(128|192|256)", "aes/gcm/%s", "AES Sensitive Property Provider", true),
AWS_KMS("aws/kms", "aws/kms", "AWS KMS Sensitive Property Provider", false),
+ HASHICORP_VAULT_KV("hashicorp/vault/kv/[a-zA-Z0-9_-]+", "hashicorp/vault/kv/%s", "HashiCorp Vault Key/Value Engine Sensitive Property Provider", false),
HASHICORP_VAULT_TRANSIT("hashicorp/vault/transit/[a-zA-Z0-9_-]+", "hashicorp/vault/transit/%s", "HashiCorp Vault Transit Engine Sensitive Property Provider", false);
PropertyProtectionScheme(final String identifierPattern, final String identifierFormat, final String name, final boolean requiresSecretKey) {
diff --git a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/StandardSensitivePropertyProviderFactory.java b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/StandardSensitivePropertyProviderFactory.java
index cfbb90d..9ba0178 100644
--- a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/StandardSensitivePropertyProviderFactory.java
+++ b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/StandardSensitivePropertyProviderFactory.java
@@ -124,6 +124,8 @@ public class StandardSensitivePropertyProviderFactory implements SensitiveProper
return providerMap.computeIfAbsent(protectionScheme, s -> new AWSSensitivePropertyProvider(getBootstrapProperties()));
case HASHICORP_VAULT_TRANSIT:
return providerMap.computeIfAbsent(protectionScheme, s -> new HashiCorpVaultTransitSensitivePropertyProvider(getBootstrapProperties()));
+ case HASHICORP_VAULT_KV:
+ return providerMap.computeIfAbsent(protectionScheme, s -> new HashiCorpVaultKeyValueSensitivePropertyProvider(getBootstrapProperties()));
default:
throw new SensitivePropertyProtectionException("Unsupported protection scheme " + protectionScheme);
}
diff --git a/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/HashiCorpVaultCommunicationService.java b/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/HashiCorpVaultCommunicationService.java
index 977b369..bf43268 100644
--- a/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/HashiCorpVaultCommunicationService.java
+++ b/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/HashiCorpVaultCommunicationService.java
@@ -16,6 +16,8 @@
*/
package org.apache.nifi.vault.hashicorp;
+import java.util.Optional;
+
/**
* A service to handle all communication with an instance of HashiCorp Vault.
* @see <a href="https://www.vaultproject.io/">https://www.vaultproject.io/</a>
@@ -26,21 +28,39 @@ public interface HashiCorpVaultCommunicationService {
* Encrypts the given plaintext using Vault's Transit Secrets Engine.
*
* @see <a href="https://www.vaultproject.io/api-docs/secret/transit">https://www.vaultproject.io/api-docs/secret/transit</a>
- * @param transitKey A named encryption key used in the Transit Secrets Engine. The key is expected to have
- * already been configured in the Vault instance.
+ * @param transitPath The Vault path to use for the configured Transit Secrets Engine
* @param plainText The plaintext to encrypt
* @return The cipher text
*/
- String encrypt(String transitKey, byte[] plainText);
+ String encrypt(String transitPath, byte[] plainText);
/**
* Decrypts the given cipher text using Vault's Transit Secrets Engine.
*
* @see <a href="https://www.vaultproject.io/api-docs/secret/transit">https://www.vaultproject.io/api-docs/secret/transit</a>
- * @param transitKey A named encryption key used in the Transit Secrets Engine. The key is expected to have
- * already been configured in the Vault instance.
+ * @param transitPath The Vault path to use for the configured Transit Secrets Engine
* @param cipherText The cipher text to decrypt
* @return The decrypted plaintext
*/
- byte[] decrypt(String transitKey, String cipherText);
+ byte[] decrypt(String transitPath, String cipherText);
+
+ /**
+ * Writes a secret using Vault's unversioned Key/Value Secrets Engine.
+ *
+ * @see <a href="https://www.vaultproject.io/api-docs/secret/kv/kv-v1">https://www.vaultproject.io/api-docs/secret/kv/kv-v1</a>
+ * @param keyValuePath The Vault path to use for the configured Key/Value v1 Secrets Engine
+ * @param key The secret key
+ * @param value The secret value
+ */
+ void writeKeyValueSecret(String keyValuePath, String key, String value);
+
+ /**
+ * Reads a secret from Vault's unversioned Key/Value Secrets Engine.
+ *
+ * @see <a href="https://www.vaultproject.io/api-docs/secret/kv/kv-v1">https://www.vaultproject.io/api-docs/secret/kv/kv-v1</a>
+ * @param keyValuePath The Vault path to use for the configured Key/Value v1 Secrets Engine
+ * @param key The secret key
+ * @return The secret value, or empty if not found
+ */
+ Optional<String> readKeyValueSecret(String keyValuePath, String key);
}
diff --git a/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/StandardHashiCorpVaultCommunicationService.java b/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/StandardHashiCorpVaultCommunicationService.java
index 61f0176..21c9213 100644
--- a/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/StandardHashiCorpVaultCommunicationService.java
+++ b/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/StandardHashiCorpVaultCommunicationService.java
@@ -16,25 +16,35 @@
*/
package org.apache.nifi.vault.hashicorp;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultConfiguration;
import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultProperties;
import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultPropertySource;
import org.springframework.core.env.PropertySource;
import org.springframework.vault.authentication.SimpleSessionManager;
import org.springframework.vault.client.ClientHttpRequestFactoryFactory;
+import org.springframework.vault.core.VaultKeyValueOperations;
import org.springframework.vault.core.VaultTemplate;
import org.springframework.vault.core.VaultTransitOperations;
import org.springframework.vault.support.Ciphertext;
import org.springframework.vault.support.Plaintext;
+import org.springframework.vault.support.VaultResponseSupport;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+import static org.springframework.vault.core.VaultKeyValueOperationsSupport.KeyValueBackend.KV_1;
/**
* Implements the VaultCommunicationService using Spring Vault
*/
public class StandardHashiCorpVaultCommunicationService implements HashiCorpVaultCommunicationService {
-
private final HashiCorpVaultConfiguration vaultConfiguration;
private final VaultTemplate vaultTemplate;
private final VaultTransitOperations transitOperations;
+ private final Map<String, VaultKeyValueOperations> keyValueOperationsMap;
/**
* Creates a VaultCommunicationService that uses Spring Vault.
@@ -49,6 +59,7 @@ public class StandardHashiCorpVaultCommunicationService implements HashiCorpVaul
new SimpleSessionManager(vaultConfiguration.clientAuthentication()));
transitOperations = vaultTemplate.opsForTransit();
+ keyValueOperationsMap = new HashMap<>();
}
/**
@@ -61,12 +72,52 @@ public class StandardHashiCorpVaultCommunicationService implements HashiCorpVaul
}
@Override
- public String encrypt(final String transitKey, final byte[] plainText) {
- return transitOperations.encrypt(transitKey, Plaintext.of(plainText)).getCiphertext();
+ public String encrypt(final String transitPath, final byte[] plainText) {
+ return transitOperations.encrypt(transitPath, Plaintext.of(plainText)).getCiphertext();
}
@Override
- public byte[] decrypt(final String transitKey, final String cipherText) {
- return transitOperations.decrypt(transitKey, Ciphertext.of(cipherText)).getPlaintext();
+ public byte[] decrypt(final String transitPath, final String cipherText) {
+ return transitOperations.decrypt(transitPath, Ciphertext.of(cipherText)).getPlaintext();
+ }
+
+ /**
+ * Writes the value to the "value" key of the secret with the path [keyValuePath]/[key].
+ * @param keyValuePath The Vault path to use for the configured Key/Value v1 Secrets Engine
+ * @param key The secret key
+ * @param value The secret value
+ */
+ @Override
+ public void writeKeyValueSecret(final String keyValuePath, final String key, final String value) {
+ final VaultKeyValueOperations keyValueOperations = keyValueOperationsMap
+ .computeIfAbsent(keyValuePath, path -> vaultTemplate.opsForKeyValue(path, KV_1));
+ keyValueOperations.put(key, new SecretData(value));
+ }
+
+ /**
+ * Returns the value of the "value" key from the secret at the path [keyValuePath]/[key].
+ * @param keyValuePath The Vault path to use for the configured Key/Value v1 Secrets Engine
+ * @param key The secret key
+ * @return The value of the secret
+ */
+ @Override
+ public Optional<String> readKeyValueSecret(final String keyValuePath, final String key) {
+ final VaultKeyValueOperations keyValueOperations = keyValueOperationsMap
+ .computeIfAbsent(keyValuePath, path -> vaultTemplate.opsForKeyValue(path, KV_1));
+ final VaultResponseSupport<SecretData> response = keyValueOperations.get(key, SecretData.class);
+ return response == null ? Optional.empty() : Optional.ofNullable(response.getRequiredData().getValue());
+ }
+
+ private static class SecretData {
+ private final String value;
+
+ @JsonCreator
+ public SecretData(@JsonProperty("value") final String value) {
+ this.value = value;
+ }
+
+ public String getValue() {
+ return value;
+ }
}
}
diff --git a/nifi-commons/nifi-vault-utils/src/test/java/org/apache/nifi/vault/hashicorp/StandardHashiCorpVaultCommunicationServiceIT.java b/nifi-commons/nifi-vault-utils/src/test/java/org/apache/nifi/vault/hashicorp/StandardHashiCorpVaultCommunicationServiceIT.java
index 60d64a9..f347801 100644
--- a/nifi-commons/nifi-vault-utils/src/test/java/org/apache/nifi/vault/hashicorp/StandardHashiCorpVaultCommunicationServiceIT.java
+++ b/nifi-commons/nifi-vault-utils/src/test/java/org/apache/nifi/vault/hashicorp/StandardHashiCorpVaultCommunicationServiceIT.java
@@ -17,17 +17,19 @@
package org.apache.nifi.vault.hashicorp;
import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultProperties;
-import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.nio.charset.StandardCharsets;
+import static org.junit.Assert.assertEquals;
+
/**
* The simplest way to run this test is by installing Vault locally, then running:
*
* vault server -dev
* vault secrets enable transit
+ * vault secrets enable kv
* vault write -f transit/keys/nifi
*
* Make note of the Root Token and create a properties file with the contents:
@@ -62,6 +64,17 @@ public class StandardHashiCorpVaultCommunicationServiceIT {
byte[] decrypted = vcs.decrypt(TRANSIT_KEY, ciphertext);
- Assert.assertEquals(plaintext, new String(decrypted, StandardCharsets.UTF_8));
+ assertEquals(plaintext, new String(decrypted, StandardCharsets.UTF_8));
+ }
+
+ @Test
+ public void testReadWriteSecret() {
+ final String key = "key";
+ final String value = "value";
+
+ vcs.writeKeyValueSecret("kv", key, value);
+
+ final String resultValue = vcs.readKeyValueSecret("kv", key).orElseThrow(() -> new NullPointerException("Missing secret for kv/key"));
+ assertEquals(value, resultValue);
}
}
diff --git a/nifi-docs/src/main/asciidoc/administration-guide.adoc b/nifi-docs/src/main/asciidoc/administration-guide.adoc
index 309ead9..4f71047 100644
--- a/nifi-docs/src/main/asciidoc/administration-guide.adoc
+++ b/nifi-docs/src/main/asciidoc/administration-guide.adoc
@@ -1765,7 +1765,7 @@ All options require a password (`nifi.sensitive.props.key` value) of *at least 1
[[encrypt-config_tool]]
== Encrypted Passwords in Configuration Files
-In order to facilitate the secure setup of NiFi, you can use the `encrypt-config` command line utility to encrypt raw configuration values that NiFi decrypts in memory on startup. This extensible protection scheme transparently allows NiFi to use raw values in operation, while protecting them at rest. In addition to the default AES encryption provider, a HashiCorp Vault encryption provider can be configured in the `bootstrap-hashicorp-vault.properties` file.
+In order to facilitate the secure setup of NiFi, you can use the `encrypt-config` command line utility to encrypt raw configuration values that NiFi decrypts in memory on startup. This extensible protection scheme transparently allows NiFi to use raw values in operation, while protecting them at rest.
This is a change in behavior; prior to 1.0, all configuration values were stored in plaintext on the file system. POSIX file permissions were recommended to limit unauthorized access to these files.
@@ -1773,6 +1773,121 @@ If no administrator action is taken, the configuration values remain unencrypted
For more information, see the <<toolkit-guide.adoc#encrypt_config_tool,Encrypt-Config Tool>> section in the link:toolkit-guide.html[NiFi Toolkit Guide].
+In addition to the default AES encryption provider, other providers can be configured in their respective `bootstrap-*.conf` files. Following is a list of additional encryption providers and their configuration:
+
+=== HashiCorp Vault providers
+Two encryption providers are currently configurable in the `bootstrap-hashicorp-vault.conf` file:
+
+[options="header,footer"]
+|===
+|Provider|Provider Identifier|Description
+|HashiCorp Vault Transit provider|`hashicorp/vault/kv/{vault.transit.path}`|Uses HashiCorp Vault's Transit Secrets Engine to decrypt sensitive properties.
+|HashiCorp Vault Key/Value provider|`hashicorp/vault/kv/{vault.transit.path}`|Retrieves sensitive values from Secrets stored in a HashiCorp Vault Key/Value (unversioned) Secrets Engine.
+|===
+
+Note that all HashiCorp Vault encryption providers require a running Vault instance in order to decrypt these values at NiFi's startup.
+
+Following are the configuration properties available inside the `bootstrap-hashicorp-vault.conf` file:
+
+==== Required properties
+
+[options="header,footer"]
+|===
+|Property Name|Description|Default
+|`vault.uri`|The HashiCorp Vault URI (e.g., `https://vault-server:8200`). If not set, all HashiCorp Vault providers will be disabled.|_none_
+|`vault.authentication.properties.file`|Filename of a properties file containing Vault authentication properties. See the `Authentication-specific property keys` section of https://docs.spring.io/spring-vault/docs/2.3.x/reference/html/#vault.core.environment-vault-configuration for all authentication property keys. If not set, all Spring Vault authentication properties must be configured directly in bootstrap-hashicorp-vault.conf.|_none_
+|`vault.transit.path`|If set, enables the HashiCorp Vault Transit provider. The value should be the Vault `path` of a Transit Secrets Engine (e.g., `nifi-transit`). Valid characters include alphanumeric, dash, and underscore.|_none_
+|`vault.kv.path`|If set, enables the HashiCorp Vault Key/Value provider. The value should be the Vault `path` of a K/V (v1) Secrets Engine (e.g., `nifi-kv`). Valid characters include alphanumeric, dash, and underscore.|_none_
+|===
+
+==== Optional properties
+[options="header,footer"]
+|===
+|Property Name|Description|Default
+|`vault.connection.timeout`|The connection timeout of the Vault client|`5 secs`
+|`vault.read.timeout`|The read timeout of the Vault client|`15 secs`
+|`vault.ssl.enabledCipherSuites`|A comma-separated list of the enabled TLS cipher suites|_none_
+|`vault.ssl.enabledProtocols`|A comma-separated list of the enabled TLS protocols|_none_
+|`vault.ssl.key-store`|Path to a keystore. Required if the Vault server is TLS-enabled|_none_
+|`vault.ssl.key-store-type`|Keystore type (JKS, BCFKS or PKCS12). Required if the Vault server is TLS-enabled|_none_
+|`vault.ssl.key-store-password`|Keystore password. Required if the Vault server is TLS-enabled|_none_
+|`vault.ssl.trust-store`|Path to a truststore. Required if the Vault server is TLS-enabled|_none_
+|`vault.ssl.trust-store-type`|Truststore type (JKS, BCFKS or PKCS12). Required if the Vault server is TLS-enabled|_none_
+|`vault.ssl.trust-store-password`|Truststore password. Required if the Vault server is TLS-enabled|_none_
+|===
+
+=== AWS KMS provider
+This provider uses AWS Key Management Service (https://aws.amazon.com/kms/) for decryption. AWS KMS configuration properties can be stored in the `bootstrap-aws.conf` file, as referenced in `bootstrap.conf`. If the configuration properties are not specified in `bootstrap-aws.conf`, then the provider will attempt to use the AWS default credentials provider, which checks standard environment variables and system properties.
+
+==== Required properties
+[options="header,footer"]
+|===
+|Property Name|Description|Default
+|`aws.kms.key.id`|The identifier or ARN that the AWS KMS client uses for encryption and decryption.|_none_
+|===
+
+==== Optional properties
+===== All of the following must be configured, or will be ignored entirely.
+[options="header,footer"]
+|===
+|Property Name|Description|Default
+|`aws.region`|The AWS region used to configure the AWS KMS Client.|_none_
+|`aws.access.key.id`|The access key ID credential used to access AWS KMS.|_none_
+|`aws.secret.access.key`|The secret access key used to access AWS KMS.|_none_
+|===
+
+=== Property Context Mapping
+Some encryption providers store protected values in an external service instead of persisting the encrypted values directly in the configuration file. To support this use case, a property context is defined for each protected property in NiFi's configuration files, in the format: `{context-name}/{property-name}`
+
+* `context-name` - represents a namespace for properties in order to disambiguate properties with the same name. Without additional configuration, all protected properties are assigned the `default` context.
+* `property-name` - contains the name of the property.
+
+In order to support logical context names, mapping properties may be provided in `bootstrap.conf`, as follows:
+
+```
+nifi.bootstrap.protection.context.mapping.<context-name>=<identifier matching regex>
+```
+
+Here, `context-name` would determine the context name above, and `<identifier matching regex>` would map any property whose *group identifier* matched the provided Regular Expression. *Group identifiers* are defined per configuration file type, and are described as follows:
+[options="header,footer"]
+|===
+|Configuration File|Group Identifier Description|Assigned Context
+|`nifi.properties`|There is no concept of a group identifier here, since all property names should be unique.|_default_
+|`authorizers.xml`|The `<identifier>` value of the XML block surrounding the property.|The mapped context name if RegEx matches the identifier, otherwise _default_
+|`login-identity-providers.xml`|The `<identifier>` value of the XML block surrounding the property.|The mapped context name if RegEx matches the identifier, otherwise _default_
+|===
+
+==== Example
+In the NiFi binary distribution, the `login-identity-providers.xml` file comes with a provider with the identifier `ldap-provider` and a property called `Manager Password`:
+
+```
+ <provider>
+ <identifier>ldap-provider</identifier>
+ <class>org.apache.nifi.ldap.LdapProvider</class>
+ ...
+ <property name="Manager Password"/>
+ ...
+ </provider>
+```
+Similarly, the `authorizers.xml` file comes with a `ldap-user-group-provider` and a property also called `Manager Password`:
+
+```
+ <userGroupProvider>
+ <identifier>ldap-user-group-provider</identifier>
+ <class>org.apache.nifi.ldap.tenants.LdapUserGroupProvider</class>
+ ...
+ <property name="Manager Password"/>
+ ...
+ </userGroupProvider>
+```
+
+If the Manager Password is desired to reference the same exact property (e.g., the same Secret in the HashiCorp Vault K/V provider) but still be distinguished from any other `Manager Password` property unrelated to LDAP, the following mapping could be added:
+
+```
+nifi.bootstrap.protection.context.mapping.ldap=ldap-.*
+```
+
+This would cause both of the above to be assigned a context of `"ldap/Manager Password"` instead of `"default/Manager Password"`.
[[admin-toolkit]]
== NiFi Toolkit Administrative Tools
In addition to `tls-toolkit` and `encrypt-config`, the NiFi Toolkit also contains command line utilities for administrators to support NiFi maintenance in standalone and clustered environments. These utilities include:
diff --git a/nifi-docs/src/main/asciidoc/toolkit-guide.adoc b/nifi-docs/src/main/asciidoc/toolkit-guide.adoc
index f61780b..3b173a8 100644
--- a/nifi-docs/src/main/asciidoc/toolkit-guide.adoc
+++ b/nifi-docs/src/main/asciidoc/toolkit-guide.adoc
@@ -477,52 +477,13 @@ The protection scheme can be selected during encryption using the `--protectionS
The default protection scheme, `AES-G/CM` simply encrypts sensitive properties and marks their protection as either `aes/gcm/256` or `aes/gcm/256` as appropriate. This protection is all done within NiFi itself.
==== HASHICORP_VAULT_TRANSIT
-This protection scheme uses HashiCorp Vault's Transit Secrets Engine (https://www.vaultproject.io/docs/secrets/transit) to outsource encryption to a configured Vault server. All HashiCorp Vault configuration is stored in the `bootstrap-hashicorp-vault.conf` file, as referenced in the `bootstrap.conf` of a NiFi or NiFi Registry instance. Therefore, when using the HASHICORP_VAULT_TRANSIT protection scheme, the `nifi(.registry)?.bootstrap.protection.hashicorp.vault.conf` property in the `b [...]
-
-===== Required properties
-[options="header,footer"]
-|===
-|Property Name|Description|Default
-|`vault.uri`|The HashiCorp Vault URI (e.g., `https://vault-server:8200`). If not set, this provider will be disabled.|_none_
-|`vault.authentication.properties.file`|Filename of a properties file containing Vault authentication properties. See the `Authentication-specific property keys` section of https://docs.spring.io/spring-vault/docs/2.3.x/reference/html/#vault.core.environment-vault-configuration for all authentication property keys. If not set, all Spring Vault authentication properties must be configured directly in bootstrap-hashicorp-vault.conf.|_none_
-|`vault.transit.path`|The HashiCorp Vault `path` specifying the Transit Secrets Engine (e.g., `nifi-transit`). Valid characters include alphanumeric, dash, and underscore. If not set, this provider will be disabled.|_none_
-|===
-
-===== Optional properties
-[options="header,footer"]
-|===
-|Property Name|Description|Default
-|`vault.connection.timeout`|The connection timeout of the Vault client|`5 secs`
-|`vault.read.timeout`|The read timeout of the Vault client|`15 secs`
-|`vault.ssl.enabledCipherSuites`|A comma-separated list of the enabled TLS cipher suites|_none_
-|`vault.ssl.enabledProtocols`|A comma-separated list of the enabled TLS protocols|_none_
-|`vault.ssl.key-store`|Path to a keystore. Required if the Vault server is TLS-enabled|_none_
-|`vault.ssl.key-store-type`|Keystore type (JKS, BCFKS or PKCS12). Required if the Vault server is TLS-enabled|_none_
-|`vault.ssl.key-store-password`|Keystore password. Required if the Vault server is TLS-enabled|_none_
-|`vault.ssl.trust-store`|Path to a truststore. Required if the Vault server is TLS-enabled|_none_
-|`vault.ssl.trust-store-type`|Truststore type (JKS, BCFKS or PKCS12). Required if the Vault server is TLS-enabled|_none_
-|`vault.ssl.trust-store-password`|Truststore password. Required if the Vault server is TLS-enabled|_none_
-|===
+This protection scheme uses HashiCorp Vault's Transit Secrets Engine (https://www.vaultproject.io/docs/secrets/transit) to outsource encryption to a configured Vault server. All HashiCorp Vault configuration is stored in the `bootstrap-hashicorp-vault.conf` file, as referenced in the `bootstrap.conf` of a NiFi or NiFi Registry instance. Therefore, when using the HASHICORP_VAULT_TRANSIT protection scheme, the `nifi(.registry)?.bootstrap.protection.hashicorp.vault.conf` property in the `b [...]
+
+==== HASHICORP_VAULT_KV
+This protection scheme uses HashiCorp Vault's Transit unversioned Key/Value Engine (https://www.vaultproject.io/docs/secrets/kv/kv-v1) to store sensitive values as Vault Secrets. All HashiCorp Vault configuration is stored in the `bootstrap-hashicorp-vault.conf` file, as referenced in the `bootstrap.conf` of a NiFi or NiFi Registry instance. Therefore, when using the HASHICORP_VAULT_KV protection scheme, the `nifi(.registry)?.bootstrap.protection.hashicorp.vault.conf` property in the `b [...]
==== AWS_KMS
-This protection scheme uses AWS Key Management Service (https://aws.amazon.com/kms/) for encryption and decryption. AWS KMS configuration properties can be stored in the `bootstrap-aws.conf` file, as referenced in the `bootstrap.conf` of NiFi or NiFi Registry. If the configuration properties are not specified in `bootstrap-aws.conf`, then the provider will attempt to use the AWS default credentials provider, which checks standard environment variables and system properties.
-
-===== Required properties
-[options="header,footer"]
-|===
-|Property Name|Description|Default
-|`aws.kms.key.id`|The identifier or ARN that the AWS KMS client uses for encryption and decryption.|_none_
-|===
-
-===== Optional properties
-====== All of the following must be configured, or will be ignored entirely.
-[options="header,footer"]
-|===
-|Property Name|Description|Default
-|`aws.region`|The AWS region used to configure the AWS KMS Client.|_none_
-|`aws.access.key.id`|The access key ID credential used to access AWS KMS.|_none_
-|`aws.secret.access.key`|The secret access key used to access AWS KMS.|_none_
-|===
+This protection scheme uses AWS Key Management Service (https://aws.amazon.com/kms/) for encryption and decryption. AWS KMS configuration properties can be stored in the `bootstrap-aws.conf` file, as referenced in the `bootstrap.conf` of NiFi or NiFi Registry. If the configuration properties are not specified in `bootstrap-aws.conf`, then the provider will attempt to use the AWS default credentials provider, which checks standard environment variables and system properties. Therefore, w [...]
=== Examples
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap-hashicorp-vault.conf b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap-hashicorp-vault.conf
index 1d1a409..bbf54f5 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap-hashicorp-vault.conf
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap-hashicorp-vault.conf
@@ -21,6 +21,9 @@ vault.uri=
# Transit Path is required to enable the Sensitive Properties Provider Protection Scheme 'hashicorp/vault/transit/{path}'
vault.transit.path=
+# Key/Value Path is required to enable the Sensitive Properties Provider Protection Scheme 'hashicorp/vault/kv/{path}'
+vault.kv.path=
+
# Token Authentication example properties
# vault.authentication=TOKEN
# vault.token=<token value>
diff --git a/nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/bootstrap-hashicorp-vault.conf b/nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/bootstrap-hashicorp-vault.conf
index 1d1a409..bbf54f5 100644
--- a/nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/bootstrap-hashicorp-vault.conf
+++ b/nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/bootstrap-hashicorp-vault.conf
@@ -21,6 +21,9 @@ vault.uri=
# Transit Path is required to enable the Sensitive Properties Provider Protection Scheme 'hashicorp/vault/transit/{path}'
vault.transit.path=
+# Key/Value Path is required to enable the Sensitive Properties Provider Protection Scheme 'hashicorp/vault/kv/{path}'
+vault.kv.path=
+
# Token Authentication example properties
# vault.authentication=TOKEN
# vault.token=<token value>
diff --git a/nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/bootstrap.conf b/nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/bootstrap.conf
index 31e397c..046d252 100644
--- a/nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/bootstrap.conf
+++ b/nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/bootstrap.conf
@@ -59,4 +59,4 @@ nifi.registry.bootstrap.sensitive.key=
nifi.registry.bootstrap.protection.hashicorp.vault.conf=./conf/bootstrap-hashicorp-vault.conf
# AWS KMS Sensitive Property Providers
-nifi.registry.bootstrap.protection.aws.kms.conf=./conf/bootstrap-aws.conf
\ No newline at end of file
+nifi.registry.bootstrap.protection.aws.kms.conf=./conf/bootstrap-aws.conf