You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ex...@apache.org on 2021/06/25 03:07:37 UTC
[nifi] branch main updated: NIFI-8447 Added HashiCorp Vault Transit
Sensitive Properties Provider
This is an automated email from the ASF dual-hosted git repository.
exceptionfactory pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 726082f NIFI-8447 Added HashiCorp Vault Transit Sensitive Properties Provider
726082f is described below
commit 726082ffa6c9f4b350fd6026152c50dc5bae2151
Author: Joe Gresock <jg...@gmail.com>
AuthorDate: Mon Jun 14 06:53:37 2021 -0400
NIFI-8447 Added HashiCorp Vault Transit Sensitive Properties Provider
- Added default bootstrap-hashicorp-vault.conf
- Updated Toolkit Guide documentation with HashiCorp Vault properties
This closes #5154
Signed-off-by: David Handermann <ex...@apache.org>
---
.../AbstractBootstrapPropertiesLoader.java | 33 ++++--
.../nifi/properties/BootstrapProperties.java | 75 +++++++++++-
.../nifi-sensitive-property-provider/pom.xml | 11 ++
.../properties/AESSensitivePropertyProvider.java | 2 +-
...actHashiCorpVaultSensitivePropertyProvider.java | 132 +++++++++++++++++++++
.../AbstractSensitivePropertyProvider.java | 13 --
...iCorpVaultTransitSensitivePropertyProvider.java | 92 ++++++++++++++
.../nifi/properties/PropertyProtectionScheme.java | 3 +-
.../StandardSensitivePropertyProviderFactory.java | 10 +-
...andardSensitivePropertyProviderFactoryTest.java | 85 +++++++++++--
...StandardHashiCorpVaultCommunicationService.java | 48 ++------
.../config/HashiCorpVaultConfiguration.java | 105 +++++++++++++++-
.../hashicorp/config/HashiCorpVaultProperties.java | 1 +
.../hashicorp/config/HashiCorpVaultProperty.java | 1 +
.../config/HashiCorpVaultPropertySource.java | 4 +-
.../config/lookup/BeanPropertyLookup.java | 17 +--
.../hashicorp/TestHashiCorpVaultConfiguration.java | 18 +--
...StandardHashiCorpVaultCommunicationService.java | 8 +-
.../src/main/asciidoc/administration-guide.adoc | 2 +-
nifi-docs/src/main/asciidoc/toolkit-guide.adoc | 46 ++++++-
.../NiFiPropertiesLoaderGroovyTest.groovy | 6 +-
.../resources/conf/bootstrap-hashicorp-vault.conf | 48 ++++++++
.../src/main/resources/conf/bootstrap.conf | 5 +
.../properties/NiFiRegistryPropertiesLoader.java | 5 +-
.../NiFiRegistryBootstrapUtilsGroovyTest.groovy | 4 +-
.../resources/conf/bootstrap-hashicorp-vault.conf | 48 ++++++++
.../src/main/resources/conf/bootstrap.conf | 7 +-
.../nifi/properties/ConfigEncryptionTool.groovy | 6 +-
.../encryptconfig/NiFiRegistryDecryptMode.groovy | 3 +-
.../toolkit/encryptconfig/NiFiRegistryMode.groovy | 26 +++-
30 files changed, 733 insertions(+), 131 deletions(-)
diff --git a/nifi-commons/nifi-property-utils/src/main/java/org/apache/nifi/properties/AbstractBootstrapPropertiesLoader.java b/nifi-commons/nifi-property-utils/src/main/java/org/apache/nifi/properties/AbstractBootstrapPropertiesLoader.java
index cbeea84..0bc378e 100644
--- a/nifi-commons/nifi-property-utils/src/main/java/org/apache/nifi/properties/AbstractBootstrapPropertiesLoader.java
+++ b/nifi-commons/nifi-property-utils/src/main/java/org/apache/nifi/properties/AbstractBootstrapPropertiesLoader.java
@@ -16,6 +16,7 @@
*/
package org.apache.nifi.properties;
+import org.apache.nifi.properties.BootstrapProperties.BootstrapPropertyKey;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -24,6 +25,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.util.Objects;
import java.util.Properties;
/**
@@ -74,14 +76,27 @@ public abstract class AbstractBootstrapPropertiesLoader {
* @throws IOException If the file is not readable
*/
public BootstrapProperties loadBootstrapProperties(final String bootstrapPath) throws IOException {
- final Properties properties = new Properties();
final Path bootstrapFilePath = getBootstrapFile(bootstrapPath).toPath();
- try (final InputStream bootstrapInput = Files.newInputStream(bootstrapFilePath)) {
+ return loadBootstrapProperties(bootstrapFilePath, getApplicationPrefix());
+ }
+
+ /**
+ * Loads a properties file into a BootstrapProperties object.
+ * @param bootstrapPath The path to the properties file
+ * @param propertyPrefix The property prefix to enforce
+ * @return The BootstrapProperties
+ * @throws IOException If the properties file could not be read
+ */
+ public static BootstrapProperties loadBootstrapProperties(final Path bootstrapPath, final String propertyPrefix) throws IOException {
+ Objects.requireNonNull(bootstrapPath, "Bootstrap path must be provided");
+ Objects.requireNonNull(propertyPrefix, "Property prefix must be provided");
+
+ final Properties properties = new Properties();
+ try (final InputStream bootstrapInput = Files.newInputStream(bootstrapPath)) {
properties.load(bootstrapInput);
- return new BootstrapProperties(getApplicationPrefix(), properties, bootstrapFilePath);
+ return new BootstrapProperties(propertyPrefix, properties, bootstrapPath);
} catch (final IOException e) {
- logger.error("Cannot read from bootstrap.conf file at {}", bootstrapFilePath);
- throw new IOException("Cannot read from bootstrap.conf", e);
+ throw new IOException("Cannot read from " + bootstrapPath, e);
}
}
@@ -97,7 +112,7 @@ public abstract class AbstractBootstrapPropertiesLoader {
public String extractKeyFromBootstrapFile(final String bootstrapPath) throws IOException {
final BootstrapProperties bootstrapProperties = loadBootstrapProperties(bootstrapPath);
- return bootstrapProperties.getBootstrapSensitiveKey().orElseGet(() -> {
+ return bootstrapProperties.getProperty(BootstrapPropertyKey.SENSITIVE_KEY).orElseGet(() -> {
logger.warn("No encryption key present in the bootstrap.conf file at {}", bootstrapProperties.getConfigFilePath());
return "";
});
@@ -121,8 +136,7 @@ public abstract class AbstractBootstrapPropertiesLoader {
if (confDir.exists() && confDir.canRead()) {
expectedBootstrapFile = new File(confDir, BOOTSTRAP_CONF);
} else {
- logger.error("Cannot read from bootstrap.conf file at {} -- conf/ directory is missing or permissions are incorrect", confDir.getAbsolutePath());
- throw new IOException("Cannot read from bootstrap.conf");
+ throw new IOException(String.format("Cannot read %s directory for %s", confDir, bootstrapPath));
}
} else {
expectedBootstrapFile = new File(bootstrapPath);
@@ -131,8 +145,7 @@ public abstract class AbstractBootstrapPropertiesLoader {
if (expectedBootstrapFile.exists() && expectedBootstrapFile.canRead()) {
return expectedBootstrapFile;
} else {
- logger.error("Cannot read from bootstrap.conf file at {} -- file is missing or permissions are incorrect", expectedBootstrapFile.getAbsolutePath());
- throw new IOException("Cannot read from bootstrap.conf");
+ throw new IOException("Cannot read from " + expectedBootstrapFile.getAbsolutePath());
}
}
diff --git a/nifi-commons/nifi-property-utils/src/main/java/org/apache/nifi/properties/BootstrapProperties.java b/nifi-commons/nifi-property-utils/src/main/java/org/apache/nifi/properties/BootstrapProperties.java
index 4713e27..bee82e1 100644
--- a/nifi-commons/nifi-property-utils/src/main/java/org/apache/nifi/properties/BootstrapProperties.java
+++ b/nifi-commons/nifi-property-utils/src/main/java/org/apache/nifi/properties/BootstrapProperties.java
@@ -17,17 +17,29 @@
package org.apache.nifi.properties;
import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.Enumeration;
import java.util.Objects;
import java.util.Optional;
import java.util.Properties;
+import java.util.Set;
/**
* Properties representing bootstrap.conf.
*/
public class BootstrapProperties extends StandardReadableProperties {
private static final String PROPERTY_KEY_FORMAT = "%s.%s";
- private static final String BOOTSTRAP_SENSITIVE_KEY = "bootstrap.sensitive.key";
+
+ public enum BootstrapPropertyKey {
+ SENSITIVE_KEY("bootstrap.sensitive.key"),
+ HASHICORP_VAULT_SENSITIVE_PROPERTY_PROVIDER_CONF("bootstrap.protection.hashicorp.vault.conf");
+
+ private final String key;
+
+ BootstrapPropertyKey(final String key) {
+ this.key = key;
+ }
+ }
private final String propertyPrefix;
private final Path configFilePath;
@@ -44,6 +56,29 @@ public class BootstrapProperties extends StandardReadableProperties {
}
/**
+ * Ensures that blank or empty properties are returned as null.
+ * @param key The property key
+ * @param defaultValue The default value to use if the value is null or empty
+ * @return The property value (null if empty or blank)
+ */
+ @Override
+ public String getProperty(final String key, final String defaultValue) {
+ final String property = super.getProperty(key, defaultValue);
+ return isBlank(property) ? null : property;
+ }
+
+ /**
+ * Ensures that blank or empty properties are returned as null.
+ * @param key The property key
+ * @return The property value (null if empty or blank)
+ */
+ @Override
+ public String getProperty(final String key) {
+ final String property = super.getProperty(key);
+ return isBlank(property) ? null : property;
+ }
+
+ /**
* Returns the path to the bootstrap config file.
* @return The path to the file
*/
@@ -72,15 +107,45 @@ public class BootstrapProperties extends StandardReadableProperties {
}
/**
- * Returns the bootstrap sensitive key.
- * @return The bootstrap sensitive key
+ * Returns the optional property value with the given BootstrapPropertyKey.
+ * @param key A BootstrapPropertyKey, representing properties in bootstrap.conf
+ * @return The property value
*/
- public Optional<String> getBootstrapSensitiveKey() {
- return Optional.ofNullable(getProperty(getPropertyKey(BOOTSTRAP_SENSITIVE_KEY)));
+ public Optional<String> getProperty(final BootstrapPropertyKey key) {
+ return Optional.ofNullable(getProperty(getPropertyKey(key.key)));
}
@Override
public String toString() {
return String.format("Bootstrap properties [%s] with prefix [%s]", configFilePath, propertyPrefix);
}
+
+ /**
+ * An empty instance of BootstrapProperties.
+ */
+ public static final BootstrapProperties EMPTY = new BootstrapProperties("", new Properties(), Paths.get("conf/bootstrap.conf")) {
+ @Override
+ public Set<String> getPropertyKeys() {
+ return null;
+ }
+
+ @Override
+ public String getProperty(String key) {
+ return null;
+ }
+
+ @Override
+ public String getProperty(String key, String defaultValue) {
+ return null;
+ }
+
+ @Override
+ public int size() {
+ return 0;
+ }
+ };
+
+ private static boolean isBlank(final String string) {
+ return (string == null) || string.isEmpty() || string.trim().isEmpty();
+ }
}
diff --git a/nifi-commons/nifi-sensitive-property-provider/pom.xml b/nifi-commons/nifi-sensitive-property-provider/pom.xml
index bdfa9ba..59660e4 100644
--- a/nifi-commons/nifi-sensitive-property-provider/pom.xml
+++ b/nifi-commons/nifi-sensitive-property-provider/pom.xml
@@ -42,6 +42,17 @@
<artifactId>nifi-security-utils</artifactId>
<version>1.14.0-SNAPSHOT</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-vault-utils</artifactId>
+ <version>1.14.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>2.10.0</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
<!-- Required to run Groovy tests without any Java tests -->
diff --git a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/AESSensitivePropertyProvider.java b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/AESSensitivePropertyProvider.java
index 3999a3a..6d30375 100644
--- a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/AESSensitivePropertyProvider.java
+++ b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/AESSensitivePropertyProvider.java
@@ -81,7 +81,7 @@ public class AESSensitivePropertyProvider extends AbstractSensitivePropertyProvi
}
@Override
- protected boolean isSupported(final BootstrapProperties bootstrapProperties) {
+ public boolean isSupported() {
return true; // AES protection is always supported
}
diff --git a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/AbstractHashiCorpVaultSensitivePropertyProvider.java b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/AbstractHashiCorpVaultSensitivePropertyProvider.java
new file mode 100644
index 0000000..3a06157
--- /dev/null
+++ b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/AbstractHashiCorpVaultSensitivePropertyProvider.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.properties;
+
+import org.apache.nifi.properties.BootstrapProperties.BootstrapPropertyKey;
+import org.apache.nifi.vault.hashicorp.HashiCorpVaultCommunicationService;
+import org.apache.nifi.vault.hashicorp.StandardHashiCorpVaultCommunicationService;
+import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultConfiguration;
+import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultConfiguration.VaultConfigurationKey;
+import org.springframework.core.env.PropertySource;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+
+public abstract class AbstractHashiCorpVaultSensitivePropertyProvider extends AbstractSensitivePropertyProvider {
+ private static final String VAULT_PREFIX = "vault";
+
+ private final String path;
+ private final HashiCorpVaultCommunicationService vaultCommunicationService;
+ private final BootstrapProperties vaultBootstrapProperties;
+
+ AbstractHashiCorpVaultSensitivePropertyProvider(final BootstrapProperties bootstrapProperties) {
+ super(bootstrapProperties);
+
+ final String vaultBootstrapConfFilename = bootstrapProperties
+ .getProperty(BootstrapPropertyKey.HASHICORP_VAULT_SENSITIVE_PROPERTY_PROVIDER_CONF).orElse(null);
+ vaultBootstrapProperties = getVaultBootstrapProperties(vaultBootstrapConfFilename);
+ path = getSecretsEnginePath(vaultBootstrapProperties);
+ if (hasRequiredVaultProperties()) {
+ try {
+ vaultCommunicationService = new StandardHashiCorpVaultCommunicationService(getVaultPropertySource(vaultBootstrapConfFilename));
+ } catch (IOException e) {
+ throw new SensitivePropertyProtectionException("Error configuring HashiCorpVaultCommunicationService", e);
+ }
+ } else {
+ vaultCommunicationService = null;
+ }
+ }
+
+ /**
+ * Return the configured Secrets Engine path for this sensitive property provider.
+ * @param vaultBootstrapProperties The Properties from the file located at bootstrap.protection.hashicorp.vault.conf
+ * @return The Secrets Engine path
+ */
+ protected abstract String getSecretsEnginePath(final BootstrapProperties vaultBootstrapProperties);
+
+ private static BootstrapProperties getVaultBootstrapProperties(final String vaultBootstrapConfFilename) {
+ final BootstrapProperties vaultBootstrapProperties;
+ if (vaultBootstrapConfFilename != null) {
+ try {
+ vaultBootstrapProperties = AbstractBootstrapPropertiesLoader.loadBootstrapProperties(
+ Paths.get(vaultBootstrapConfFilename), VAULT_PREFIX);
+ } catch (IOException e) {
+ throw new SensitivePropertyProtectionException("Could not load " + vaultBootstrapConfFilename, e);
+ }
+ } else {
+ vaultBootstrapProperties = null;
+ }
+ return vaultBootstrapProperties;
+ }
+
+ private PropertySource<?> getVaultPropertySource(final String vaultBootstrapConfFilename) throws IOException {
+ return HashiCorpVaultConfiguration.createPropertiesFileSource(vaultBootstrapConfFilename);
+ }
+
+ /**
+ * Returns the Secrets Engine path.
+ * @return The Secrets Engine path
+ */
+ protected String getPath() {
+ return path;
+ }
+
+ protected HashiCorpVaultCommunicationService getVaultCommunicationService() {
+ if (vaultCommunicationService == null) {
+ throw new SensitivePropertyProtectionException(getIdentifierKey() + " protection scheme is not fully configured in hashicorp-vault-bootstrap.conf");
+ }
+ return vaultCommunicationService;
+ }
+
+ @Override
+ public boolean isSupported() {
+ return hasRequiredVaultProperties();
+ }
+
+ /**
+ * Returns the Vault-specific bootstrap properties (e.g., bootstrap-vault.properties)
+ * @return The Vault-specific bootstrap properties
+ */
+ protected BootstrapProperties getVaultBootstrapProperties() {
+ return vaultBootstrapProperties;
+ }
+
+ private boolean hasRequiredVaultProperties() {
+ return vaultBootstrapProperties != null
+ && (vaultBootstrapProperties.getProperty(VaultConfigurationKey.URI.getKey()) != null)
+ && hasRequiredSecretsEngineProperties(vaultBootstrapProperties);
+ }
+
+ /**
+ * Return true if the relevant Secrets Engine-specific properties are configured.
+ * @param vaultBootstrapProperties The Vault-specific bootstrap properties
+ * @return true if the relevant Secrets Engine-specific properties are configured
+ */
+ protected abstract boolean hasRequiredSecretsEngineProperties(final BootstrapProperties vaultBootstrapProperties);
+
+ /**
+ * Returns the key used to identify the provider implementation in {@code nifi.properties},
+ * in the format 'vault/{secretsEngine}/{secretsEnginePath}'.
+ *
+ * @return the key to persist in the sibling property
+ */
+ @Override
+ public String getIdentifierKey() {
+ return getProtectionScheme().getIdentifier(path);
+ }
+
+}
diff --git a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/AbstractSensitivePropertyProvider.java b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/AbstractSensitivePropertyProvider.java
index b52fb73..e163747 100644
--- a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/AbstractSensitivePropertyProvider.java
+++ b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/AbstractSensitivePropertyProvider.java
@@ -33,14 +33,6 @@ public abstract class AbstractSensitivePropertyProvider implements SensitiveProp
*/
protected abstract PropertyProtectionScheme getProtectionScheme();
- /**
- * Return true if this SensitivePropertyProvider is supported, given the provided
- * Bootstrap properties.
- * @param bootstrapProperties The Bootstrap properties
- * @return True if this SensitivePropertyProvider is supported
- */
- protected abstract boolean isSupported(BootstrapProperties bootstrapProperties);
-
@Override
public String getName() {
return getProtectionScheme().getName();
@@ -55,9 +47,4 @@ public abstract class AbstractSensitivePropertyProvider implements SensitiveProp
public String getIdentifierKey() {
return getProtectionScheme().getIdentifier();
}
-
- @Override
- public boolean isSupported() {
- return isSupported(bootstrapProperties);
- }
}
diff --git a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/HashiCorpVaultTransitSensitivePropertyProvider.java b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/HashiCorpVaultTransitSensitivePropertyProvider.java
new file mode 100644
index 0000000..b670cb1
--- /dev/null
+++ b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/HashiCorpVaultTransitSensitivePropertyProvider.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.properties;
+
+import org.apache.commons.lang3.StringUtils;
+
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+
+/**
+ * Uses the HashiCorp Vault Transit Secrets Engine to encrypt sensitive values at rest.
+ */
+public class HashiCorpVaultTransitSensitivePropertyProvider extends AbstractHashiCorpVaultSensitivePropertyProvider {
+ private static final Charset PROPERTY_CHARSET = StandardCharsets.UTF_8;
+ private static final String TRANSIT_PATH = "vault.transit.path";
+
+ HashiCorpVaultTransitSensitivePropertyProvider(final BootstrapProperties bootstrapProperties) {
+ super(bootstrapProperties);
+ }
+
+ @Override
+ protected String getSecretsEnginePath(final BootstrapProperties vaultBootstrapProperties) {
+ if (vaultBootstrapProperties == null) {
+ return null;
+ }
+ final String transitPath = vaultBootstrapProperties.getProperty(TRANSIT_PATH);
+ // Validate transit path
+ try {
+ PropertyProtectionScheme.fromIdentifier(getProtectionScheme().getIdentifier(transitPath));
+ } catch (IllegalArgumentException e) {
+ throw new SensitivePropertyProtectionException(String.format("%s [%s] contains unsupported characters", TRANSIT_PATH, transitPath), e);
+ }
+
+ return transitPath;
+ }
+
+ @Override
+ protected PropertyProtectionScheme getProtectionScheme() {
+ return PropertyProtectionScheme.HASHICORP_VAULT_TRANSIT;
+ }
+
+ @Override
+ protected boolean hasRequiredSecretsEngineProperties(final BootstrapProperties vaultBootstrapProperties) {
+ return getSecretsEnginePath(vaultBootstrapProperties) != null;
+ }
+
+ /**
+ * Returns the encrypted cipher text.
+ *
+ * @param unprotectedValue the sensitive value
+ * @return the value to persist in the {@code nifi.properties} file
+ * @throws SensitivePropertyProtectionException if there is an exception encrypting the value
+ */
+ @Override
+ public String protect(final String unprotectedValue) throws SensitivePropertyProtectionException {
+ if (StringUtils.isBlank(unprotectedValue)) {
+ throw new IllegalArgumentException("Cannot encrypt an empty value");
+ }
+
+ return getVaultCommunicationService().encrypt(getPath(), unprotectedValue.getBytes(PROPERTY_CHARSET));
+ }
+
+ /**
+ * Returns the decrypted plaintext.
+ *
+ * @param protectedValue the cipher text read from the {@code nifi.properties} file
+ * @return the raw value to be used by the application
+ * @throws SensitivePropertyProtectionException if there is an error decrypting the cipher text
+ */
+ @Override
+ public String unprotect(final String protectedValue) throws SensitivePropertyProtectionException {
+ if (StringUtils.isBlank(protectedValue)) {
+ throw new IllegalArgumentException("Cannot decrypt an empty value");
+ }
+
+ return new String(getVaultCommunicationService().decrypt(getPath(), protectedValue), PROPERTY_CHARSET);
+ }
+}
diff --git a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/PropertyProtectionScheme.java b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/PropertyProtectionScheme.java
index 4017406..8c321f2 100644
--- a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/PropertyProtectionScheme.java
+++ b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/PropertyProtectionScheme.java
@@ -24,7 +24,8 @@ import java.util.Objects;
* SensitivePropertyProvider.
*/
public enum PropertyProtectionScheme {
- AES_GCM("aes/gcm/(128|192|256)", "aes/gcm/%s", "AES Sensitive Property Provider", true);
+ AES_GCM("aes/gcm/(128|192|256)", "aes/gcm/%s", "AES Sensitive Property Provider", true),
+ HASHICORP_VAULT_TRANSIT("hashicorp/vault/transit/[a-zA-Z0-9_-]+", "hashicorp/vault/transit/%s", "HashiCorp Vault Transit Engine Sensitive Property Provider", false);
PropertyProtectionScheme(final String identifierPattern, final String identifierFormat, final String name, final boolean requiresSecretKey) {
this.identifierPattern = identifierPattern;
diff --git a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/StandardSensitivePropertyProviderFactory.java b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/StandardSensitivePropertyProviderFactory.java
index ef59333..230bb22 100644
--- a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/StandardSensitivePropertyProviderFactory.java
+++ b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/StandardSensitivePropertyProviderFactory.java
@@ -16,6 +16,7 @@
*/
package org.apache.nifi.properties;
+import org.apache.nifi.properties.BootstrapProperties.BootstrapPropertyKey;
import org.apache.nifi.util.NiFiBootstrapUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -76,7 +77,7 @@ public class StandardSensitivePropertyProviderFactory implements SensitiveProper
}
private String getKeyHex() {
- return keyHex.orElseGet(() -> getBootstrapProperties().getBootstrapSensitiveKey()
+ return keyHex.orElseGet(() -> getBootstrapProperties().getProperty(BootstrapPropertyKey.SENSITIVE_KEY)
.orElseThrow(() -> new SensitivePropertyProtectionException("Could not read root key from bootstrap.conf")));
}
@@ -90,8 +91,8 @@ public class StandardSensitivePropertyProviderFactory implements SensitiveProper
try {
return NiFiBootstrapUtils.loadBootstrapProperties();
} catch (final IOException e) {
- logger.error("Error extracting root key from bootstrap.conf for login identity provider decryption", e);
- throw new SensitivePropertyProtectionException("Could not read root key from bootstrap.conf");
+ logger.debug("Could not load bootstrap.conf from disk, so using empty bootstrap.conf", e);
+ return BootstrapProperties.EMPTY;
}
});
}
@@ -104,7 +105,8 @@ public class StandardSensitivePropertyProviderFactory implements SensitiveProper
switch (protectionScheme) {
case AES_GCM:
return providerMap.computeIfAbsent(protectionScheme, s -> new AESSensitivePropertyProvider(keyHex));
- // Other providers may choose to pass getBootstrapProperties() into the constructor
+ case HASHICORP_VAULT_TRANSIT:
+ return providerMap.computeIfAbsent(protectionScheme, s -> new HashiCorpVaultTransitSensitivePropertyProvider(getBootstrapProperties()));
default:
throw new SensitivePropertyProtectionException("Unsupported protection scheme " + protectionScheme);
}
diff --git a/nifi-commons/nifi-sensitive-property-provider/src/test/java/org/apache/nifi/properties/StandardSensitivePropertyProviderFactoryTest.java b/nifi-commons/nifi-sensitive-property-provider/src/test/java/org/apache/nifi/properties/StandardSensitivePropertyProviderFactoryTest.java
index f36f259..81995bf 100644
--- a/nifi-commons/nifi-sensitive-property-provider/src/test/java/org/apache/nifi/properties/StandardSensitivePropertyProviderFactoryTest.java
+++ b/nifi-commons/nifi-sensitive-property-provider/src/test/java/org/apache/nifi/properties/StandardSensitivePropertyProviderFactoryTest.java
@@ -16,6 +16,7 @@
*/
package org.apache.nifi.properties;
+import org.apache.commons.io.FilenameUtils;
import org.apache.nifi.util.NiFiProperties;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.junit.AfterClass;
@@ -23,8 +24,10 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.internal.util.io.IOUtil;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.Security;
@@ -32,8 +35,10 @@ import java.util.Properties;
import java.util.function.Supplier;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
public class StandardSensitivePropertyProviderFactoryTest {
@@ -45,8 +50,9 @@ public class StandardSensitivePropertyProviderFactoryTest {
private static final String AD_HOC_KEY_HEX = "123456789ABCDEFFEDCBA98765432101";
private static Path tempConfDir;
- private static Path mockBootstrapConf;
- private static Path mockNifiProperties;
+ private static Path bootstrapConf;
+ private static Path hashicorpVaultBootstrapConf;
+ private static Path nifiProperties;
private static NiFiProperties niFiProperties;
@@ -54,23 +60,28 @@ public class StandardSensitivePropertyProviderFactoryTest {
public static void initOnce() throws IOException {
Security.addProvider(new BouncyCastleProvider());
tempConfDir = Files.createTempDirectory("conf");
- mockBootstrapConf = Files.createTempFile("bootstrap", ".conf").toAbsolutePath();
+ bootstrapConf = Files.createTempFile("bootstrap", ".conf").toAbsolutePath();
+ hashicorpVaultBootstrapConf = Files.createTempFile("bootstrap-hashicorp-vault", ".conf").toAbsolutePath();
- mockNifiProperties = Files.createTempFile("nifi", ".properties").toAbsolutePath();
+ nifiProperties = Files.createTempFile("nifi", ".properties").toAbsolutePath();
- mockBootstrapConf = Files.move(mockBootstrapConf, tempConfDir.resolve("bootstrap.conf"));
- mockNifiProperties = Files.move(mockNifiProperties, tempConfDir.resolve("nifi.properties"));
+ bootstrapConf = Files.move(bootstrapConf, tempConfDir.resolve("bootstrap.conf"));
+ nifiProperties = Files.move(nifiProperties, tempConfDir.resolve("nifi.properties"));
- IOUtil.writeText("nifi.bootstrap.sensitive.key=" + BOOTSTRAP_KEY_HEX, mockBootstrapConf.toFile());
- System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, mockNifiProperties.toString());
+ final String bootstrapConfText = String.format("%s=%s\n%s=%s",
+ "nifi.bootstrap.sensitive.key", BOOTSTRAP_KEY_HEX,
+ "nifi.bootstrap.protection.hashicorp.vault.conf", FilenameUtils.separatorsToUnix(hashicorpVaultBootstrapConf.toString()));
+ IOUtil.writeText(bootstrapConfText, bootstrapConf.toFile());
+ System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, FilenameUtils.separatorsToUnix(nifiProperties.toString()));
niFiProperties = new NiFiProperties();
}
@AfterClass
public static void tearDownOnce() throws IOException {
- Files.deleteIfExists(mockBootstrapConf);
- Files.deleteIfExists(mockNifiProperties);
+ Files.deleteIfExists(bootstrapConf);
+ Files.deleteIfExists(hashicorpVaultBootstrapConf);
+ Files.deleteIfExists(nifiProperties);
Files.deleteIfExists(tempConfDir);
System.clearProperty(NiFiProperties.PROPERTIES_FILE_PATH);
}
@@ -99,12 +110,62 @@ public class StandardSensitivePropertyProviderFactoryTest {
private Supplier<BootstrapProperties> mockBootstrapProperties() throws IOException {
final Properties bootstrapProperties = new Properties();
- try (final InputStream inputStream = Files.newInputStream(mockBootstrapConf)) {
+ try (final InputStream inputStream = Files.newInputStream(bootstrapConf)) {
bootstrapProperties.load(inputStream);
- return () -> new BootstrapProperties("nifi", bootstrapProperties, mockBootstrapConf);
+ return () -> new BootstrapProperties("nifi", bootstrapProperties, bootstrapConf);
}
}
+ private void configureHashicorpVault(final Properties properties) throws IOException {
+ try (OutputStream out = new FileOutputStream(hashicorpVaultBootstrapConf.toFile())) {
+ properties.store(out, "HashiCorpVault test");
+ }
+ }
+
+ @Test
+ public void testHashicorpVaultTransit() throws IOException {
+ configureDefaultFactory();
+ final Properties properties = new Properties();
+ properties.put("vault.transit.path", "nifi-transit");
+ configureHashicorpVault(properties);
+
+ final SensitivePropertyProvider spp = factory.getProvider(PropertyProtectionScheme.HASHICORP_VAULT_TRANSIT);
+ }
+
+ @Test
+ public void testHashicorpVaultTransit_isSupported() throws IOException {
+ configureDefaultFactory();
+ final Properties properties = new Properties();
+ properties.put("vault.transit.path", "nifi-transit");
+ properties.put("vault.uri", "http://localhost:8200");
+ properties.put("vault.token", "test-token");
+ configureHashicorpVault(properties);
+
+ SensitivePropertyProvider spp = factory.getProvider(PropertyProtectionScheme.HASHICORP_VAULT_TRANSIT);
+ assertTrue(spp.isSupported());
+
+ properties.remove("vault.uri");
+ configureHashicorpVault(properties);
+ configureDefaultFactory();
+ spp = factory.getProvider(PropertyProtectionScheme.HASHICORP_VAULT_TRANSIT);
+ assertFalse(spp.isSupported());
+
+ properties.put("vault.uri", "http://localhost:8200");
+ properties.remove("vault.transit.path");
+ spp = factory.getProvider(PropertyProtectionScheme.HASHICORP_VAULT_TRANSIT);
+ assertFalse(spp.isSupported());
+ }
+
+ @Test
+ public void testHashicorpVaultTransit_invalidCharacters() throws IOException {
+ configureDefaultFactory();
+ final Properties properties = new Properties();
+ properties.put("vault.transit.path", "invalid/characters");
+ configureHashicorpVault(properties);
+
+ assertThrows(SensitivePropertyProtectionException.class, () -> factory.getProvider(PropertyProtectionScheme.HASHICORP_VAULT_TRANSIT));
+ }
+
@Test
public void testAES_GCM() throws IOException {
configureDefaultFactory();
diff --git a/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/StandardHashiCorpVaultCommunicationService.java b/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/StandardHashiCorpVaultCommunicationService.java
index 8f92fb3..61f0176 100644
--- a/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/StandardHashiCorpVaultCommunicationService.java
+++ b/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/StandardHashiCorpVaultCommunicationService.java
@@ -16,27 +16,21 @@
*/
package org.apache.nifi.vault.hashicorp;
-import org.apache.nifi.util.FormatUtils;
import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultConfiguration;
import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultProperties;
+import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultPropertySource;
+import org.springframework.core.env.PropertySource;
import org.springframework.vault.authentication.SimpleSessionManager;
import org.springframework.vault.client.ClientHttpRequestFactoryFactory;
import org.springframework.vault.core.VaultTemplate;
import org.springframework.vault.core.VaultTransitOperations;
import org.springframework.vault.support.Ciphertext;
-import org.springframework.vault.support.ClientOptions;
import org.springframework.vault.support.Plaintext;
-import org.springframework.vault.support.SslConfiguration;
-
-import java.time.Duration;
-import java.util.Optional;
-import java.util.concurrent.TimeUnit;
/**
* Implements the VaultCommunicationService using Spring Vault
*/
public class StandardHashiCorpVaultCommunicationService implements HashiCorpVaultCommunicationService {
- private static final String HTTPS = "https";
private final HashiCorpVaultConfiguration vaultConfiguration;
private final VaultTemplate vaultTemplate;
@@ -44,42 +38,26 @@ public class StandardHashiCorpVaultCommunicationService implements HashiCorpVaul
/**
* Creates a VaultCommunicationService that uses Spring Vault.
- * @param vaultProperties Properties to configure the service
+ * @param propertySources Property sources to configure the service
* @throws HashiCorpVaultConfigurationException If the configuration was invalid
*/
- public StandardHashiCorpVaultCommunicationService(final HashiCorpVaultProperties vaultProperties) throws HashiCorpVaultConfigurationException {
- this.vaultConfiguration = new HashiCorpVaultConfiguration(vaultProperties);
-
- final SslConfiguration sslConfiguration = vaultProperties.getUri().contains(HTTPS)
- ? vaultConfiguration.sslConfiguration() : SslConfiguration.unconfigured();
-
- final ClientOptions clientOptions = getClientOptions(vaultProperties);
+ public StandardHashiCorpVaultCommunicationService(final PropertySource<?>... propertySources) throws HashiCorpVaultConfigurationException {
+ vaultConfiguration = new HashiCorpVaultConfiguration(propertySources);
vaultTemplate = new VaultTemplate(vaultConfiguration.vaultEndpoint(),
- ClientHttpRequestFactoryFactory.create(clientOptions, sslConfiguration),
+ ClientHttpRequestFactoryFactory.create(vaultConfiguration.clientOptions(), vaultConfiguration.sslConfiguration()),
new SimpleSessionManager(vaultConfiguration.clientAuthentication()));
transitOperations = vaultTemplate.opsForTransit();
}
- private static ClientOptions getClientOptions(HashiCorpVaultProperties vaultProperties) {
- final ClientOptions clientOptions = new ClientOptions();
- Duration readTimeoutDuration = clientOptions.getReadTimeout();
- Duration connectionTimeoutDuration = clientOptions.getConnectionTimeout();
- final Optional<String> configuredReadTimeout = vaultProperties.getReadTimeout();
- if (configuredReadTimeout.isPresent()) {
- readTimeoutDuration = getDuration(configuredReadTimeout.get());
- }
- final Optional<String> configuredConnectionTimeout = vaultProperties.getConnectionTimeout();
- if (configuredConnectionTimeout.isPresent()) {
- connectionTimeoutDuration = getDuration(configuredConnectionTimeout.get());
- }
- return new ClientOptions(connectionTimeoutDuration, readTimeoutDuration);
- }
-
- private static Duration getDuration(String formattedDuration) {
- final double duration = FormatUtils.getPreciseTimeDuration(formattedDuration, TimeUnit.MILLISECONDS);
- return Duration.ofMillis(Double.valueOf(duration).longValue());
+ /**
+ * Creates a VaultCommunicationService that uses Spring Vault.
+ * @param vaultProperties Properties to configure the service
+ * @throws HashiCorpVaultConfigurationException If the configuration was invalid
+ */
+ public StandardHashiCorpVaultCommunicationService(final HashiCorpVaultProperties vaultProperties) throws HashiCorpVaultConfigurationException {
+ this(new HashiCorpVaultPropertySource(vaultProperties));
}
@Override
diff --git a/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/HashiCorpVaultConfiguration.java b/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/HashiCorpVaultConfiguration.java
index 44ad4e6..34ce6c6 100644
--- a/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/HashiCorpVaultConfiguration.java
+++ b/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/HashiCorpVaultConfiguration.java
@@ -16,31 +16,124 @@
*/
package org.apache.nifi.vault.hashicorp.config;
+import org.apache.nifi.util.FormatUtils;
import org.apache.nifi.vault.hashicorp.HashiCorpVaultConfigurationException;
import org.springframework.core.env.ConfigurableEnvironment;
+import org.springframework.core.env.PropertySource;
import org.springframework.core.env.StandardEnvironment;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.support.ResourcePropertySource;
+import org.springframework.vault.client.RestTemplateFactory;
import org.springframework.vault.config.EnvironmentVaultConfiguration;
+import org.springframework.vault.support.ClientOptions;
+import org.springframework.vault.support.SslConfiguration;
import java.io.IOException;
import java.nio.file.Paths;
+import java.time.Duration;
+import java.util.concurrent.TimeUnit;
/**
* A Vault configuration that uses the NiFiVaultEnvironment.
*/
public class HashiCorpVaultConfiguration extends EnvironmentVaultConfiguration {
+ public enum VaultConfigurationKey {
+ AUTHENTICATION_PROPERTIES_FILE("vault.authentication.properties.file"),
+ READ_TIMEOUT("vault.read.timeout"),
+ CONNECTION_TIMEOUT("vault.connection.timeout"),
+ URI("vault.uri");
- public HashiCorpVaultConfiguration(final HashiCorpVaultProperties vaultProperties) throws HashiCorpVaultConfigurationException {
+ private final String key;
+
+ VaultConfigurationKey(final String key) {
+ this.key = key;
+ }
+
+ /**
+ * Returns the property key.
+ * @return The property key
+ */
+ public String getKey() {
+ return key;
+ }
+ }
+
+ private static final String HTTPS = "https";
+
+ private final SslConfiguration sslConfiguration;
+ private final ClientOptions clientOptions;
+
+ /**
+ * Creates a HashiCorpVaultConfiguration from property sources
+ * @param propertySources A series of Spring PropertySource objects
+ * @throws HashiCorpVaultConfigurationException If the authentication properties file could not be read
+ */
+ public HashiCorpVaultConfiguration(final PropertySource<?>... propertySources) {
final ConfigurableEnvironment env = new StandardEnvironment();
+ for(final PropertySource<?> propertySource : propertySources) {
+ env.getPropertySources().addFirst(propertySource);
+ }
- try {
- env.getPropertySources().addFirst(new ResourcePropertySource(new FileSystemResource(Paths.get(vaultProperties.getAuthPropertiesFilename()))));
- } catch (IOException e) {
- throw new HashiCorpVaultConfigurationException("Could not load auth properties", e);
+ if (env.containsProperty(VaultConfigurationKey.AUTHENTICATION_PROPERTIES_FILE.key)) {
+ final String authPropertiesFilename = env.getProperty(VaultConfigurationKey.AUTHENTICATION_PROPERTIES_FILE.key);
+ try {
+ final PropertySource<?> authPropertiesSource = createPropertiesFileSource(authPropertiesFilename);
+ env.getPropertySources().addFirst(authPropertiesSource);
+ } catch (IOException e) {
+ throw new HashiCorpVaultConfigurationException("Could not load HashiCorp Vault authentication properties " + authPropertiesFilename, e);
+ }
}
- env.getPropertySources().addFirst(new HashiCorpVaultPropertySource(vaultProperties));
this.setApplicationContext(new HashiCorpVaultApplicationContext(env));
+
+ sslConfiguration = env.getProperty(VaultConfigurationKey.URI.key).contains(HTTPS)
+ ? super.sslConfiguration() : SslConfiguration.unconfigured();
+
+ clientOptions = getClientOptions();
+ }
+
+ /**
+ * A convenience method to create a PropertySource from a file on disk.
+ * @param filename The properties filename.
+ * @return A PropertySource containing the properties in the given file
+ * @throws IOException If the file could not be read
+ */
+ public static PropertySource<?> createPropertiesFileSource(final String filename) throws IOException {
+ return new ResourcePropertySource(new FileSystemResource(Paths.get(filename)));
+ }
+
+ @Override
+ public ClientOptions clientOptions() {
+ return clientOptions;
+ }
+
+ @Override
+ protected RestTemplateFactory getRestTemplateFactory() {
+ return this.restTemplateFactory(clientHttpRequestFactoryWrapper());
+ }
+
+ @Override
+ public SslConfiguration sslConfiguration() {
+ return sslConfiguration;
+ }
+
+ private ClientOptions getClientOptions() {
+ final ClientOptions clientOptions = new ClientOptions();
+ Duration readTimeoutDuration = clientOptions.getReadTimeout();
+ Duration connectionTimeoutDuration = clientOptions.getConnectionTimeout();
+ final String configuredReadTimeout = getEnvironment().getProperty(VaultConfigurationKey.READ_TIMEOUT.key);
+ if (configuredReadTimeout != null) {
+ readTimeoutDuration = getDuration(configuredReadTimeout);
+ }
+ final String configuredConnectionTimeout = getEnvironment().getProperty(VaultConfigurationKey.CONNECTION_TIMEOUT.key);
+ if (configuredConnectionTimeout != null) {
+ connectionTimeoutDuration = getDuration(configuredConnectionTimeout);
+ }
+ return new ClientOptions(connectionTimeoutDuration, readTimeoutDuration);
+ }
+
+ private static Duration getDuration(String formattedDuration) {
+ final double duration = FormatUtils.getPreciseTimeDuration(formattedDuration, TimeUnit.MILLISECONDS);
+ return Duration.ofMillis(Double.valueOf(duration).longValue());
}
}
diff --git a/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/HashiCorpVaultProperties.java b/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/HashiCorpVaultProperties.java
index 867ca0c..3a84bc9 100644
--- a/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/HashiCorpVaultProperties.java
+++ b/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/HashiCorpVaultProperties.java
@@ -79,6 +79,7 @@ public class HashiCorpVaultProperties {
return ssl;
}
+ @HashiCorpVaultProperty(key = "authentication.properties.file")
public String getAuthPropertiesFilename() {
return authPropertiesFilename;
}
diff --git a/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/HashiCorpVaultProperty.java b/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/HashiCorpVaultProperty.java
index 81bd87e..dcee774 100644
--- a/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/HashiCorpVaultProperty.java
+++ b/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/HashiCorpVaultProperty.java
@@ -28,4 +28,5 @@ import java.lang.annotation.Target;
@Target(ElementType.METHOD)
@Retention(RetentionPolicy.RUNTIME)
public @interface HashiCorpVaultProperty {
+ String key() default "";
}
diff --git a/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/HashiCorpVaultPropertySource.java b/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/HashiCorpVaultPropertySource.java
index 446efc1..8e64d08 100644
--- a/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/HashiCorpVaultPropertySource.java
+++ b/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/HashiCorpVaultPropertySource.java
@@ -30,10 +30,10 @@ public class HashiCorpVaultPropertySource extends PropertySource<HashiCorpVaultP
private PropertyLookup propertyLookup;
- public HashiCorpVaultPropertySource(HashiCorpVaultProperties source) {
+ public HashiCorpVaultPropertySource(final HashiCorpVaultProperties source) {
super(HashiCorpVaultPropertySource.class.getName(), source);
- propertyLookup = new BeanPropertyLookup(PREFIX, HashiCorpVaultProperties.class, HashiCorpVaultProperty.class);
+ propertyLookup = new BeanPropertyLookup(PREFIX, HashiCorpVaultProperties.class);
}
@Override
diff --git a/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/lookup/BeanPropertyLookup.java b/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/lookup/BeanPropertyLookup.java
index 2ad1ac1..da873af 100644
--- a/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/lookup/BeanPropertyLookup.java
+++ b/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/lookup/BeanPropertyLookup.java
@@ -17,10 +17,10 @@
package org.apache.nifi.vault.hashicorp.config.lookup;
import org.apache.nifi.vault.hashicorp.HashiCorpVaultConfigurationException;
+import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultProperty;
import org.springframework.beans.BeanUtils;
import java.beans.PropertyDescriptor;
-import java.lang.annotation.Annotation;
import java.lang.reflect.InvocationTargetException;
import java.util.Arrays;
import java.util.Map;
@@ -34,24 +34,25 @@ public class BeanPropertyLookup extends PropertyLookup {
private final Map<String, PropertyLookup> propertyLookupMap;
- public BeanPropertyLookup(final String prefix, final Class<?> beanClass, final Class<? extends Annotation> propertyFilter) {
- this(prefix, beanClass, propertyFilter, null);
+ public BeanPropertyLookup(final String prefix, final Class<?> beanClass) {
+ this(prefix, beanClass, null);
}
- private BeanPropertyLookup(final String prefix, final Class<?> beanClass, final Class<? extends Annotation> propertyFilter,
- final PropertyDescriptor propertyDescriptor) {
+ private BeanPropertyLookup(final String prefix, final Class<?> beanClass, final PropertyDescriptor propertyDescriptor) {
super(propertyDescriptor);
propertyLookupMap = Arrays.stream(BeanUtils.getPropertyDescriptors(beanClass))
- .filter(pd -> pd.getReadMethod().getAnnotation(propertyFilter) != null)
+ .filter(pd -> pd.getReadMethod().getAnnotation(HashiCorpVaultProperty.class) != null)
.collect(Collectors.toMap(
pd -> getPropertyKey(prefix, pd),
pd -> pd.getReadMethod().getReturnType().equals(String.class) ? new ValuePropertyLookup(pd)
- : new BeanPropertyLookup(getPropertyKey(prefix, pd), pd.getReadMethod().getReturnType(), propertyFilter, pd)
+ : new BeanPropertyLookup(getPropertyKey(prefix, pd), pd.getReadMethod().getReturnType(), pd)
));
}
private static String getPropertyKey(final String prefix, final PropertyDescriptor propertyDescriptor) {
- return prefix == null ? propertyDescriptor.getDisplayName() : String.join(SEPARATOR, prefix, propertyDescriptor.getDisplayName());
+ final HashiCorpVaultProperty propertyAnnotation = propertyDescriptor.getReadMethod().getAnnotation(HashiCorpVaultProperty.class);
+ final String unqualifiedPropertyKey = !propertyAnnotation.key().isEmpty() ? propertyAnnotation.key() : propertyDescriptor.getDisplayName();
+ return prefix == null ? unqualifiedPropertyKey: String.join(SEPARATOR, prefix, unqualifiedPropertyKey);
}
@Override
diff --git a/nifi-commons/nifi-vault-utils/src/test/java/org/apache/nifi/vault/hashicorp/TestHashiCorpVaultConfiguration.java b/nifi-commons/nifi-vault-utils/src/test/java/org/apache/nifi/vault/hashicorp/TestHashiCorpVaultConfiguration.java
index 38cb2d9..e1ec59c 100644
--- a/nifi-commons/nifi-vault-utils/src/test/java/org/apache/nifi/vault/hashicorp/TestHashiCorpVaultConfiguration.java
+++ b/nifi-commons/nifi-vault-utils/src/test/java/org/apache/nifi/vault/hashicorp/TestHashiCorpVaultConfiguration.java
@@ -18,6 +18,7 @@ package org.apache.nifi.vault.hashicorp;
import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultConfiguration;
import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultProperties;
+import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultPropertySource;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
@@ -113,13 +114,13 @@ public class TestHashiCorpVaultConfiguration {
}
}
- public void runTest() {
- config = new HashiCorpVaultConfiguration(propertiesBuilder.build());
+ public void runTest(final String expectedScheme) {
+ config = new HashiCorpVaultConfiguration(new HashiCorpVaultPropertySource(propertiesBuilder.build()));
VaultEndpoint endpoint = config.vaultEndpoint();
Assert.assertEquals("localhost", endpoint.getHost());
Assert.assertEquals(8200, endpoint.getPort());
- Assert.assertEquals("http", endpoint.getScheme());
+ Assert.assertEquals(expectedScheme, endpoint.getScheme());
ClientAuthentication clientAuthentication = config.clientAuthentication();
Assert.assertNotNull(clientAuthentication);
@@ -127,7 +128,7 @@ public class TestHashiCorpVaultConfiguration {
@Test
public void testBasicProperties() {
- this.runTest();
+ this.runTest("http");
}
@Test
@@ -140,8 +141,9 @@ public class TestHashiCorpVaultConfiguration {
propertiesBuilder.setTrustStoreType(TRUSTSTORE_TYPE_VALUE);
propertiesBuilder.setEnabledTlsProtocols(TLS_V_1_3_VALUE);
propertiesBuilder.setEnabledTlsCipherSuites(TEST_CIPHER_SUITE_VALUE);
+ propertiesBuilder.setUri(URI_VALUE.replace("http", "https"));
- this.runTest();
+ this.runTest("https");
SslConfiguration sslConfiguration = config.sslConfiguration();
Assert.assertEquals(keystoreFile.toFile().getAbsolutePath(), sslConfiguration.getKeyStoreConfiguration().getResource().getFile().getAbsolutePath());
@@ -157,7 +159,7 @@ public class TestHashiCorpVaultConfiguration {
@Test
public void testInvalidTLS() {
propertiesBuilder.setUri(URI_VALUE.replace("http", "https"));
- Assert.assertThrows(NullPointerException.class, () -> this.runTest());
+ Assert.assertThrows(NullPointerException.class, () -> this.runTest("https"));
}
@Test
@@ -169,7 +171,7 @@ public class TestHashiCorpVaultConfiguration {
authProperties = writeVaultAuthProperties(props);
propertiesBuilder.setAuthPropertiesFilename(authProperties.getAbsolutePath());
- Assert.assertThrows(IllegalArgumentException.class, () -> this.runTest());
+ Assert.assertThrows(IllegalArgumentException.class, () -> this.runTest("http"));
} finally {
if (authProperties != null) {
Files.deleteIfExists(authProperties.toPath());
@@ -185,7 +187,7 @@ public class TestHashiCorpVaultConfiguration {
authProperties = writeVaultAuthProperties(props);
propertiesBuilder.setAuthPropertiesFilename(authProperties.getAbsolutePath());
- Assert.assertThrows(IllegalArgumentException.class, () -> this.runTest());
+ Assert.assertThrows(IllegalArgumentException.class, () -> this.runTest("http"));
} finally {
if (authProperties != null) {
Files.deleteIfExists(authProperties.toPath());
diff --git a/nifi-commons/nifi-vault-utils/src/test/java/org/apache/nifi/vault/hashicorp/TestStandardHashiCorpVaultCommunicationService.java b/nifi-commons/nifi-vault-utils/src/test/java/org/apache/nifi/vault/hashicorp/TestStandardHashiCorpVaultCommunicationService.java
index f9b102a..6527ca2 100644
--- a/nifi-commons/nifi-vault-utils/src/test/java/org/apache/nifi/vault/hashicorp/TestStandardHashiCorpVaultCommunicationService.java
+++ b/nifi-commons/nifi-vault-utils/src/test/java/org/apache/nifi/vault/hashicorp/TestStandardHashiCorpVaultCommunicationService.java
@@ -70,9 +70,8 @@ public class TestStandardHashiCorpVaultCommunicationService {
// Once to check if the URI is https, and once by VaultTemplate
Mockito.verify(properties, Mockito.times(2)).getUri();
- // Once each to check if they are configured
- Mockito.verify(properties, Mockito.times(1)).getConnectionTimeout();
- Mockito.verify(properties, Mockito.times(1)).getReadTimeout();
+ // Once to check if the property is set, and once to retrieve the value
+ Mockito.verify(properties, Mockito.times(2)).getAuthPropertiesFilename();
// These should not be called because TLS is not configured
this.ensureTlsPropertiesAccessed(0);
@@ -94,9 +93,6 @@ public class TestStandardHashiCorpVaultCommunicationService {
Mockito.when(properties.getConnectionTimeout()).thenReturn(Optional.of("20 secs"));
Mockito.when(properties.getReadTimeout()).thenReturn(Optional.of("40 secs"));
this.configureService();
-
- Mockito.verify(properties, Mockito.times(1)).getConnectionTimeout();
- Mockito.verify(properties, Mockito.times(1)).getReadTimeout();
}
@Test
diff --git a/nifi-docs/src/main/asciidoc/administration-guide.adoc b/nifi-docs/src/main/asciidoc/administration-guide.adoc
index 2e9e423..98146ab 100644
--- a/nifi-docs/src/main/asciidoc/administration-guide.adoc
+++ b/nifi-docs/src/main/asciidoc/administration-guide.adoc
@@ -1759,7 +1759,7 @@ All options require a password (`nifi.sensitive.props.key` value) of *at least 1
[[encrypt-config_tool]]
== Encrypted Passwords in Configuration Files
-In order to facilitate the secure setup of NiFi, you can use the `encrypt-config` command line utility to encrypt raw configuration values that NiFi decrypts in memory on startup. This extensible protection scheme transparently allows NiFi to use raw values in operation, while protecting them at rest. In the future, hardware security modules (HSM) and external secure storage mechanisms will be integrated, but for now, an AES encryption provider is the default implementation.
+In order to facilitate the secure setup of NiFi, you can use the `encrypt-config` command line utility to encrypt raw configuration values that NiFi decrypts in memory on startup. This extensible protection scheme transparently allows NiFi to use raw values in operation, while protecting them at rest. In addition to the default AES encryption provider, a HashiCorp Vault encryption provider can be configured in the `bootstrap-hashicorp-vault.properties` file.
This is a change in behavior; prior to 1.0, all configuration values were stored in plaintext on the file system. POSIX file permissions were recommended to limit unauthorized access to these files.
diff --git a/nifi-docs/src/main/asciidoc/toolkit-guide.adoc b/nifi-docs/src/main/asciidoc/toolkit-guide.adoc
index b59d45c..bdfc282 100644
--- a/nifi-docs/src/main/asciidoc/toolkit-guide.adoc
+++ b/nifi-docs/src/main/asciidoc/toolkit-guide.adoc
@@ -434,13 +434,15 @@ The following are available options when targeting NiFi:
* `-u`,`--outputAuthorizers <file>` The destination _authorizers.xml_ file containing protected config values (will not modify input _authorizers.xml_)
* `-f`,`--flowXml <file>` The _flow.xml.gz_ file currently protected with old password (will be overwritten unless `-g` is specified)
* `-g`,`--outputFlowXml <file>` The destination _flow.xml.gz_ file containing protected config values (will not modify input _flow.xml.gz_)
- * `-b`,`--bootstrapConf <file>` The _bootstrap.conf_ file to persist root key
+ * `-b`,`--bootstrapConf <file>` The bootstrap.conf file to persist root key and to optionally provide any configuration for the protection scheme.
+ * `-S`,`--protectionScheme <protectionScheme>` Selects the protection scheme for encrypted properties. Valid values are: [AES_GCM, HASHICORP_VAULT_TRANSIT] (default is AES_GCM)
* `-k`,`--key <keyhex>` The raw hexadecimal key to use to encrypt the sensitive properties
* `-e`,`--oldKey <keyhex>` The old raw hexadecimal key to use during key migration
+ * `-H`,`--oldProtectionScheme <protectionScheme>` The old protection scheme to use during encryption migration (see --protectionScheme for possible values). Default is AES_GCM
* `-p`,`--password <password>` The password from which to derive the key to use to encrypt the sensitive properties
* `-w`,`--oldPassword <password>` The old password from which to derive the key during migration
* `-r`,`--useRawKey` If provided, the secure console will prompt for the raw key value in hexadecimal form
- * `-m`,`--migrate` If provided, the _nifi.properties_ and/or _login-identity-providers.xml_ sensitive properties will be re-encrypted with a new key
+ * `-m`,`--migrate` If provided, the _nifi.properties_ and/or _login-identity-providers.xml_ sensitive properties will be re-encrypted with the new scheme
* `-x`,`--encryptFlowXmlOnly` If provided, the properties in _flow.xml.gz_ will be re-encrypted with a new key but the _nifi.properties_ and/or _login-identity-providers.xml_ files will not be modified
* `-s`,`--propsKey <password|keyhex>` The password or key to use to encrypt the sensitive processor properties in _flow.xml.gz_
* `-A`,`--newFlowAlgorithm <algorithm>` The algorithm to use to encrypt the sensitive processor properties in _flow.xml.gz_
@@ -454,9 +456,11 @@ The following are available options when targeting NiFi Registry using the `--ni
* `-v`,`--verbose` Sets verbose mode (default false)
* `-p`,`--password <password>` Protect the files using a password-derived key. If an argument is not provided to this flag, interactive mode will be triggered to prompt the user to enter the password.
* `-k`,`--key <keyhex>` Protect the files using a raw hexadecimal key. If an argument is not provided to this flag, interactive mode will be triggered to prompt the user to enter the key.
+ * `-S`,`--protectionScheme <protectionScheme>` Selects the protection scheme for encrypted properties. Valid values are: [AES_GCM, HASHICORP_VAULT_TRANSIT] (default is AES_GCM)
* `--oldPassword <password>` If the input files are already protected using a password-derived key, this specifies the old password so that the files can be unprotected before re-protecting.
* `--oldKey <keyhex>` If the input files are already protected using a key, this specifies the raw hexadecimal key so that the files can be unprotected before re-protecting.
- * `-b`,`--bootstrapConf <file>` The _bootstrap.conf_ file containing no root key or an existing root key. If a new password or key is specified (using `-p` or `-k`) and no output _bootstrap.conf_ file is specified, then this file will be overwritten to persist the new root key.
+ * `-H`,`--oldProtectionScheme <protectionScheme>`The old protection scheme to use during encryption migration (see --protectionScheme for possible values). Default is AES_GCM.
+ * `-b`,`--bootstrapConf <file>` The _bootstrap.conf_ file containing no root key or an existing root key, and any other protection scheme configuration properties. If a new password or key is specified (using -p or -k) and no output _bootstrap.conf_ file is specified, then this file will be overwritten to persist the new master key.
* `-B`,`--outputBootstrapConf <file>` The destination _bootstrap.conf_ file to persist root key. If specified, the input _bootstrap.conf_ will not be modified.
* `-r`,`--nifiRegistryProperties <file>` The _nifi-registry.properties_ file containing unprotected config values, overwritten if no output file specified.
* `-R`,`--outputNifiRegistryProperties <file>` The destination _nifi-registry.properties_ file containing protected config values.
@@ -466,6 +470,40 @@ The following are available options when targeting NiFi Registry using the `--ni
* `-I`,`--outputIdentityProvidersXml <file>` The destination _identity-providers.xml_ file containing protected config values.
* `--decrypt` Can be used with `-r` to decrypt a previously encrypted NiFi Registry Properties file. Decrypted content is printed to STDOUT.
+=== Protection Schemes
+The protection scheme can be selected during encryption using the `--protectionScheme` flag. During migration, the former protection scheme is specified using the `--oldProtectionScheme` flag. This distinction allows a set of protected configuration files to be migrated not only to a new key, but to a completely different protection scheme.
+
+==== AES_GCM
+The default protection scheme, `AES-G/CM` simply encrypts sensitive properties and marks their protection as either `aes/gcm/256` or `aes/gcm/256` as appropriate. This protection is all done within NiFi itself.
+
+==== HASHICORP_VAULT_TRANSIT
+This protection scheme uses HashiCorp Vault's Transit Secrets Engine (https://www.vaultproject.io/docs/secrets/transit) to outsource encryption to a configured Vault server. All HashiCorp Vault configuration is stored in the `bootstrap-hashicorp-vault.conf` file, as referenced in the `bootstrap.conf` of a NiFi or NiFi Registry instance. Therefore, when using the HASHICORP_VAULT_TRANSIT protection scheme, the `nifi(.registry)?.bootstrap.protection.hashicorp.vault.conf` property in the `b [...]
+
+===== Required properties
+[options="header,footer"]
+|===
+|Property Name|Description|Default
+|`vault.uri`|The HashiCorp Vault URI (e.g., `https://vault-server:8200`). If not set, this provider will be disabled.|_none_
+|`vault.authentication.properties.file`|Filename of a properties file containing Vault authentication properties. See the `Authentication-specific property keys` section of https://docs.spring.io/spring-vault/docs/2.3.x/reference/html/#vault.core.environment-vault-configuration for all authentication property keys. If not set, all Spring Vault authentication properties must be configured directly in bootstrap-hashicorp-vault.conf.|_none_
+|`vault.transit.path`|The HashiCorp Vault `path` specifying the Transit Secrets Engine (e.g., `nifi-transit`). Valid characters include alphanumeric, dash, and underscore. If not set, this provider will be disabled.|_none_
+|===
+
+===== Optional properties
+[options="header,footer"]
+|===
+|Property Name|Description|Default
+|`vault.connection.timeout`|The connection timeout of the Vault client|`5 secs`
+|`vault.read.timeout`|The read timeout of the Vault client|`15 secs`
+|`vault.ssl.enabledCipherSuites`|A comma-separated list of the enabled TLS cipher suites|_none_
+|`vault.ssl.enabledProtocols`|A comma-separated list of the enabled TLS protocols|_none_
+|`vault.ssl.key-store`|Path to a keystore. Required if the Vault server is TLS-enabled|_none_
+|`vault.ssl.key-store-type`|Keystore type (JKS, BCFKS or PKCS12). Required if the Vault server is TLS-enabled|_none_
+|`vault.ssl.key-store-password`|Keystore password. Required if the Vault server is TLS-enabled|_none_
+|`vault.ssl.trust-store`|Path to a truststore. Required if the Vault server is TLS-enabled|_none_
+|`vault.ssl.trust-store-type`|Truststore type (JKS, BCFKS or PKCS12). Required if the Vault server is TLS-enabled|_none_
+|`vault.ssl.trust-store-password`|Truststore password. Required if the Vault server is TLS-enabled|_none_
+|===
+
=== Examples
==== NiFi
@@ -658,6 +696,8 @@ for each phase (old vs. new), and any combination is sufficient:
* old password -> new key
* old password -> new password
+In order to change the protection scheme (e.g., migrating from AES encryption to Vault encryption), specify the `--protectionScheme`
+and `--oldProtectionScheme` in the migration command.
== File Manager
The File Manager utility (invoked as `./bin/file-manager.sh` or `bin\file-manager.bat`) allows system administrators to take a backup of an existing NiFi installation, install a new version of NiFi in a designated location (while migrating any previous configuration settings) or restore an installation from a previous backup. File Manager supports NiFi version 1.0.0 and higher.
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/NiFiPropertiesLoaderGroovyTest.groovy b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/NiFiPropertiesLoaderGroovyTest.groovy
index b73a970..4809923 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/NiFiPropertiesLoaderGroovyTest.groovy
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/NiFiPropertiesLoaderGroovyTest.groovy
@@ -290,6 +290,8 @@ class NiFiPropertiesLoaderGroovyTest extends GroovyTestCase {
void testShouldLoadUnprotectedPropertiesFromProtectedFile() throws Exception {
// Arrange
File protectedFile = new File("src/test/resources/conf/nifi_with_sensitive_properties_protected_aes.properties")
+
+ System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, protectedFile.path)
NiFiPropertiesLoader niFiPropertiesLoader = NiFiPropertiesLoader.withKey(KEY_HEX)
final def EXPECTED_PLAIN_VALUES = [
@@ -378,7 +380,7 @@ class NiFiPropertiesLoaderGroovyTest extends GroovyTestCase {
logger.expected(msg)
// Assert
- assert msg == "Cannot read from bootstrap.conf"
+ assert msg =~ "Cannot read from .*bootstrap.conf"
}
@Test
@@ -399,7 +401,7 @@ class NiFiPropertiesLoaderGroovyTest extends GroovyTestCase {
logger.expected(msg)
// Assert
- assert msg == "Cannot read from bootstrap.conf"
+ assert msg =~ "Cannot read from .*bootstrap.conf"
// Clean up to allow for indexing, etc.
Files.setPosixFilePermissions(unreadableFile.toPath(), originalPermissions)
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap-hashicorp-vault.conf b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap-hashicorp-vault.conf
new file mode 100644
index 0000000..1d1a409
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap-hashicorp-vault.conf
@@ -0,0 +1,48 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# HTTP or HTTPS URI for HashiCorp Vault is required to enable the Sensitive Properties Provider
+vault.uri=
+
+# Transit Path is required to enable the Sensitive Properties Provider Protection Scheme 'hashicorp/vault/transit/{path}'
+vault.transit.path=
+
+# Token Authentication example properties
+# vault.authentication=TOKEN
+# vault.token=<token value>
+
+# Optional file supports authentication properties described in the Spring Vault Environment Configuration
+# https://docs.spring.io/spring-vault/docs/2.3.x/reference/html/#vault.core.environment-vault-configuration
+#
+# All authentication properties must be included in bootstrap-hashicorp-vault.conf when this property is not specified.
+# Properties in bootstrap-hashicorp-vault.conf take precedence when the same values are defined in both files.
+# Token Authentication is the default when the 'vault.authentication' property is not specified.
+vault.authentication.properties.file=
+
+# Optional Timeout properties
+vault.connection.timeout=5 secs
+vault.read.timeout=15 secs
+
+# Optional TLS properties
+vault.ssl.enabledCipherSuites=
+vault.ssl.enabledProtocols=
+vault.ssl.key-store=
+vault.ssl.key-store-type=
+vault.ssl.key-store-password=
+vault.ssl.trust-store=
+vault.ssl.trust-store-type=
+vault.ssl.trust-store-password=
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap.conf b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap.conf
index 40486c0..778a699 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap.conf
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap.conf
@@ -58,6 +58,11 @@ java.arg.14=-Djava.awt.headless=true
# Root key in hexadecimal format for encrypted sensitive configuration values
nifi.bootstrap.sensitive.key=
+# Sensitive Property Provider configuration
+
+# HashiCorp Vault Sensitive Property Providers
+nifi.bootstrap.protection.hashicorp.vault.conf=./conf/bootstrap-hashicorp-vault.conf
+
# Sets the provider of SecureRandom to /dev/urandom to prevent blocking on VMs
java.arg.15=-Djava.security.egd=file:/dev/urandom
diff --git a/nifi-registry/nifi-registry-core/nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/NiFiRegistryPropertiesLoader.java b/nifi-registry/nifi-registry-core/nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/NiFiRegistryPropertiesLoader.java
index be9e282..a05c0f7 100644
--- a/nifi-registry/nifi-registry-core/nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/NiFiRegistryPropertiesLoader.java
+++ b/nifi-registry/nifi-registry-core/nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/NiFiRegistryPropertiesLoader.java
@@ -16,7 +16,7 @@
*/
package org.apache.nifi.registry.properties;
-import org.apache.nifi.properties.SensitivePropertyProtectionException;
+import org.apache.nifi.properties.BootstrapProperties;
import org.apache.nifi.properties.SensitivePropertyProvider;
import org.apache.nifi.properties.SensitivePropertyProviderFactory;
import org.apache.nifi.properties.StandardSensitivePropertyProviderFactory;
@@ -82,7 +82,8 @@ public class NiFiRegistryPropertiesLoader {
try {
return NiFiRegistryBootstrapUtils.loadBootstrapProperties();
} catch (IOException e) {
- throw new SensitivePropertyProtectionException("Could not load bootstrap.conf for sensitive property provider configuration.", e);
+ logger.debug("Cannot read bootstrap.conf -- file is missing or not readable. Defaulting to empty bootstrap.conf");
+ return BootstrapProperties.EMPTY;
}
});
}
diff --git a/nifi-registry/nifi-registry-core/nifi-registry-properties/src/test/groovy/org/apache/nifi/registry/properties/util/NiFiRegistryBootstrapUtilsGroovyTest.groovy b/nifi-registry/nifi-registry-core/nifi-registry-properties/src/test/groovy/org/apache/nifi/registry/properties/util/NiFiRegistryBootstrapUtilsGroovyTest.groovy
index 465a122..c95bbb6 100644
--- a/nifi-registry/nifi-registry-core/nifi-registry-properties/src/test/groovy/org/apache/nifi/registry/properties/util/NiFiRegistryBootstrapUtilsGroovyTest.groovy
+++ b/nifi-registry/nifi-registry-core/nifi-registry-properties/src/test/groovy/org/apache/nifi/registry/properties/util/NiFiRegistryBootstrapUtilsGroovyTest.groovy
@@ -95,7 +95,7 @@ class NiFiRegistryBootstrapUtilsGroovyTest extends GroovyTestCase {
logger.info(msg)
// Assert
- assert msg == "Cannot read from bootstrap.conf"
+ assert msg =~ "Cannot read from .*bootstrap.missing.conf"
}
@Test
@@ -114,7 +114,7 @@ class NiFiRegistryBootstrapUtilsGroovyTest extends GroovyTestCase {
logger.info(msg)
// Assert
- assert msg == "Cannot read from bootstrap.conf"
+ assert msg =~ "Cannot read from .*bootstrap.unreadable_file_permissions.conf"
} finally {
// Clean up to allow for indexing, etc.
Files.setPosixFilePermissions(unreadableFile.toPath(), originalPermissions)
diff --git a/nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/bootstrap-hashicorp-vault.conf b/nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/bootstrap-hashicorp-vault.conf
new file mode 100644
index 0000000..1d1a409
--- /dev/null
+++ b/nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/bootstrap-hashicorp-vault.conf
@@ -0,0 +1,48 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# HTTP or HTTPS URI for HashiCorp Vault is required to enable the Sensitive Properties Provider
+vault.uri=
+
+# Transit Path is required to enable the Sensitive Properties Provider Protection Scheme 'hashicorp/vault/transit/{path}'
+vault.transit.path=
+
+# Token Authentication example properties
+# vault.authentication=TOKEN
+# vault.token=<token value>
+
+# Optional file supports authentication properties described in the Spring Vault Environment Configuration
+# https://docs.spring.io/spring-vault/docs/2.3.x/reference/html/#vault.core.environment-vault-configuration
+#
+# All authentication properties must be included in bootstrap-hashicorp-vault.conf when this property is not specified.
+# Properties in bootstrap-hashicorp-vault.conf take precedence when the same values are defined in both files.
+# Token Authentication is the default when the 'vault.authentication' property is not specified.
+vault.authentication.properties.file=
+
+# Optional Timeout properties
+vault.connection.timeout=5 secs
+vault.read.timeout=15 secs
+
+# Optional TLS properties
+vault.ssl.enabledCipherSuites=
+vault.ssl.enabledProtocols=
+vault.ssl.key-store=
+vault.ssl.key-store-type=
+vault.ssl.key-store-password=
+vault.ssl.trust-store=
+vault.ssl.trust-store-type=
+vault.ssl.trust-store-password=
diff --git a/nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/bootstrap.conf b/nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/bootstrap.conf
index e4bf313..3663ba7 100644
--- a/nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/bootstrap.conf
+++ b/nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/bootstrap.conf
@@ -51,4 +51,9 @@ java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true
java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol
# Master key in hexadecimal format for encrypted sensitive configuration values
-nifi.registry.bootstrap.sensitive.key=
\ No newline at end of file
+nifi.registry.bootstrap.sensitive.key=
+
+# Sensitive Property Provider configuration
+
+# HashiCorp Vault Sensitive Property Providers
+nifi.registry.bootstrap.protection.hashicorp.vault.conf=./conf/bootstrap-hashicorp-vault.conf
\ No newline at end of file
diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy
index 9f48d6c..27cf192 100644
--- a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy
+++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy
@@ -31,7 +31,6 @@ import org.apache.nifi.encrypt.PropertyEncryptor
import org.apache.nifi.encrypt.PropertyEncryptorFactory
import org.apache.nifi.flow.encryptor.FlowEncryptor
import org.apache.nifi.flow.encryptor.StandardFlowEncryptor
-import org.apache.nifi.registry.properties.util.NiFiRegistryBootstrapUtils
import org.apache.nifi.toolkit.tls.commandLine.CommandLineParseException
import org.apache.nifi.toolkit.tls.commandLine.ExitCode
import org.apache.nifi.util.NiFiBootstrapUtils
@@ -599,7 +598,8 @@ class ConfigEncryptionTool {
}
private NiFiPropertiesLoader getNiFiPropertiesLoader(final String keyHex) {
- return protectionScheme.requiresSecretKey() ? NiFiPropertiesLoader.withKey(keyHex) : new NiFiPropertiesLoader()
+ return protectionScheme.requiresSecretKey() || migrationProtectionScheme.requiresSecretKey()
+ ? NiFiPropertiesLoader.withKey(keyHex) : new NiFiPropertiesLoader()
}
/**
@@ -1566,7 +1566,7 @@ class ConfigEncryptionTool {
@Override
BootstrapProperties get() {
try {
- NiFiRegistryBootstrapUtils.loadBootstrapProperties(bootstrapConfPath)
+ NiFiBootstrapUtils.loadBootstrapProperties(bootstrapConfPath)
} catch (final IOException e) {
throw new SensitivePropertyProtectionException(e.getCause(), e)
}
diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/NiFiRegistryDecryptMode.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/NiFiRegistryDecryptMode.groovy
index 919e2b0..b8433cf 100644
--- a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/NiFiRegistryDecryptMode.groovy
+++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/NiFiRegistryDecryptMode.groovy
@@ -17,7 +17,6 @@
package org.apache.nifi.toolkit.encryptconfig
import groovy.cli.commons.CliBuilder
-import org.apache.nifi.properties.ConfigEncryptionTool
import org.apache.nifi.properties.PropertyProtectionScheme
import org.apache.nifi.properties.StandardSensitivePropertyProviderFactory
import org.apache.nifi.toolkit.encryptconfig.util.BootstrapUtil
@@ -117,7 +116,7 @@ class NiFiRegistryDecryptMode extends DecryptMode {
}
config.decryptionProvider = StandardSensitivePropertyProviderFactory
- .withKeyAndBootstrapSupplier(config.key, ConfigEncryptionTool.getBootstrapSupplier(config.inputBootstrapPath))
+ .withKeyAndBootstrapSupplier(config.key, NiFiRegistryMode.getBootstrapSupplier(config.inputBootstrapPath))
.getProvider(config.protectionScheme)
run(config)
diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/NiFiRegistryMode.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/NiFiRegistryMode.groovy
index 3b5bbce..ff36b22 100644
--- a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/NiFiRegistryMode.groovy
+++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/NiFiRegistryMode.groovy
@@ -20,10 +20,13 @@ import groovy.cli.commons.CliBuilder
import groovy.cli.commons.OptionAccessor
import org.apache.commons.cli.HelpFormatter
import org.apache.commons.cli.Options
+import org.apache.nifi.properties.BootstrapProperties
import org.apache.nifi.properties.ConfigEncryptionTool
import org.apache.nifi.properties.PropertyProtectionScheme
+import org.apache.nifi.properties.SensitivePropertyProtectionException
import org.apache.nifi.properties.SensitivePropertyProvider
import org.apache.nifi.properties.StandardSensitivePropertyProviderFactory
+import org.apache.nifi.registry.properties.util.NiFiRegistryBootstrapUtils
import org.apache.nifi.toolkit.encryptconfig.util.BootstrapUtil
import org.apache.nifi.toolkit.encryptconfig.util.NiFiRegistryAuthorizersXmlEncryptor
import org.apache.nifi.toolkit.encryptconfig.util.NiFiRegistryIdentityProvidersXmlEncryptor
@@ -33,6 +36,8 @@ import org.apache.nifi.util.console.TextDevices
import org.slf4j.Logger
import org.slf4j.LoggerFactory
+import java.util.function.Supplier
+
class NiFiRegistryMode implements ToolMode {
private static final Logger logger = LoggerFactory.getLogger(NiFiRegistryMode.class)
@@ -45,6 +50,19 @@ class NiFiRegistryMode implements ToolMode {
verboseEnabled = false
}
+ static Supplier<BootstrapProperties> getBootstrapSupplier(final String bootstrapConfPath) {
+ new Supplier<BootstrapProperties>() {
+ @Override
+ BootstrapProperties get() {
+ try {
+ NiFiRegistryBootstrapUtils.loadBootstrapProperties(bootstrapConfPath)
+ } catch (final IOException e) {
+ throw new SensitivePropertyProtectionException(e.getCause(), e)
+ }
+ }
+ }
+ }
+
@Override
void run(String[] args) {
try {
@@ -318,12 +336,12 @@ class NiFiRegistryMode implements ToolMode {
throw new RuntimeException("Failed to configure tool, could not determine encryption key. Must provide -p, -k, or -b. If using -b, bootstrap.conf argument must already contain root key.")
}
encryptionProvider = StandardSensitivePropertyProviderFactory
- .withKeyAndBootstrapSupplier(encryptionKey, ConfigEncryptionTool.getBootstrapSupplier(inputBootstrapPath))
- .getProvider(oldProtectionScheme)
+ .withKeyAndBootstrapSupplier(encryptionKey, getBootstrapSupplier(inputBootstrapPath))
+ .getProvider(protectionScheme)
decryptionProvider = decryptionKey ? StandardSensitivePropertyProviderFactory
- .withKeyAndBootstrapSupplier(decryptionKey, ConfigEncryptionTool.getBootstrapSupplier(inputBootstrapPath))
- .getProvider(protectionScheme) : null
+ .withKeyAndBootstrapSupplier(decryptionKey, getBootstrapSupplier(inputBootstrapPath))
+ .getProvider(oldProtectionScheme) : null
if (handlingNiFiRegistryProperties) {
propertiesEncryptor = new NiFiRegistryPropertiesEncryptor(encryptionProvider, decryptionProvider)