You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by al...@apache.org on 2016/11/23 21:30:45 UTC
[2/2] nifi git commit: NIFI-3024 Added key migration for sensitive
processor properties contained in flow.xml.gz. (nifi.sensitive.props.key)
NIFI-3024 Added key migration for sensitive processor properties contained in flow.xml.gz. (nifi.sensitive.props.key)
This closes #1261.
Signed-off-by: Andy LoPresto <al...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/2c371453
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/2c371453
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/2c371453
Branch: refs/heads/master
Commit: 2c3714536fc516fc1712dac2a7a9ccd855e6f25b
Parents: cdb9b81
Author: Andy LoPresto <al...@apache.org>
Authored: Mon Nov 21 21:19:18 2016 -0800
Committer: Andy LoPresto <al...@apache.org>
Committed: Wed Nov 23 13:26:18 2016 -0800
----------------------------------------------------------------------
.../src/main/asciidoc/administration-guide.adoc | 32 +-
.../nifi/properties/NiFiPropertiesLoader.java | 64 +-
.../properties/ProtectedNiFiProperties.java | 2 +-
.../AESSensitivePropertyProviderTest.groovy | 2 +-
.../nifi-toolkit-encrypt-config/pom.xml | 12 +
.../nifi/properties/ConfigEncryptionTool.groovy | 483 ++++++++-
.../properties/ConfigEncryptionToolTest.groovy | 981 ++++++++++++++++++-
.../src/test/resources/flow.xml | 154 +++
.../src/test/resources/flow.xml.gz | Bin 0 -> 1674 bytes
.../src/test/resources/flow_default_key.xml | 154 +++
.../src/test/resources/flow_default_key.xml.gz | Bin 0 -> 1686 bytes
.../src/test/resources/nifi_default.properties | 125 +++
...erties_protected_aes_password_128.properties | 34 +
13 files changed, 1960 insertions(+), 83 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/2c371453/nifi-docs/src/main/asciidoc/administration-guide.adoc
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/administration-guide.adoc b/nifi-docs/src/main/asciidoc/administration-guide.adoc
index 2d63a9a..287760a 100644
--- a/nifi-docs/src/main/asciidoc/administration-guide.adoc
+++ b/nifi-docs/src/main/asciidoc/administration-guide.adoc
@@ -1002,19 +1002,25 @@ The default encryption algorithm utilized is AES/GCM 128/256-bit. 128-bit is use
You can use the following command line options with the `encrypt-config` tool:
-* `-b,--bootstrapConf <arg>` The bootstrap.conf file to persist master key
-* `-e,--oldKey <arg>` The old raw hexadecimal key to use during key migration
-* `-h,--help` Prints this usage message
-* `-k,--key <arg>` The raw hexadecimal key to use to encrypt the sensitive properties
-* `-m,--migrate` If provided, the sensitive properties will be re-encrypted with a new key
-* `-n,--niFiProperties <arg>` The nifi.properties file containing unprotected config values (will be overwritten)
-* `-o,--outputNiFiProperties <arg>` The destination nifi.properties file containing protected config values (will not modify input nifi.properties)
-* `-p,--password <arg>` The password from which to derive the key to use to encrypt the sensitive properties
-* `-r,--useRawKey` If provided, the secure console will prompt for the raw key value in hexadecimal form
-* `-v,--verbose` Sets verbose mode (default false)
-* `-w,--oldPassword <arg>` The old password from which to derive the key during migration
-* `-l,--loginIdentityProviders <arg>` The login-identity-providers.xml file containing unprotected config values (will be overwritten)
-* `-i,--outputLoginIdentityProviders <arg>` The destination login-identity-providers.xml file containing protected config values (will not modify input login-identity-providers.xml)
+ * `-A`,`--newFlowAlgorithm <arg>` The algorithm to use to encrypt the sensitive processor properties in flow.xml.gz
+ * `-b`,`--bootstrapConf <arg>` The bootstrap.conf file to persist master key
+ * `-e`,`--oldKey <arg>` The old raw hexadecimal key to use during key migration
+ * `-f`,`--flowXml <arg>` The flow.xml.gz file currently protected with old password (will be overwritten)
+ * `-g`,`--outputFlowXml <arg>` The destination flow.xml.gz file containing protected config values (will not modify input flow.xml.gz)
+ * `-h`,`--help` Prints this usage message
+ * `-i`,`--outputLoginIdentityProviders <arg>` The destination login-identity-providers.xml file containing protected config values (will not modify input login-identity-providers.xml)
+ * `-k`,`--key <arg>` The raw hexadecimal key to use to encrypt the sensitive properties
+ * `-l`,`--loginIdentityProviders <arg>` The login-identity-providers.xml file containing unprotected config values (will be overwritten)
+ * `-m`,`--migrate` If provided, the nifi.properties and/or login-identity-providers.xml sensitive properties will be re-encrypted with a new key
+ * `-n`,`--niFiProperties <arg>` The nifi.properties file containing unprotected config values (will be overwritten)
+ * `-o`,`--outputNiFiProperties <arg>` The destination nifi.properties file containing protected config values (will not modify input nifi.properties)
+ * `-p`,`--password <arg>` The password from which to derive the key to use to encrypt the sensitive properties
+ * `-P`,`--newFlowProvider <arg>` The security provider to use to encrypt the sensitive processor properties in flow.xml.gz
+ * `-r`,`--useRawKey` If provided, the secure console will prompt for the raw key value in hexadecimal form
+ * `-s`,`--propsKey <arg>` The password or key to use to encrypt the sensitive processor properties in flow.xml.gz
+ * `-v`,`--verbose` Sets verbose mode (default false)
+ * `-w`,`--oldPassword <arg>` The old password from which to derive the key during migration
+ * `-x`,`--encryptFlowXmlOnly` If provided, the properties in flow.xml.gz will be re-encrypted with a new key but the nifi.properties and/or login-identity-providers.xml files will not be modified
As an example of how the tool works, assume that you have installed the tool on a machine supporting 256-bit encryption and with the following existing values in the 'nifi.properties' file:
http://git-wip-us.apache.org/repos/asf/nifi/blob/2c371453/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/NiFiPropertiesLoader.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/NiFiPropertiesLoader.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/NiFiPropertiesLoader.java
index 20b5191..b9230c3 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/NiFiPropertiesLoader.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/NiFiPropertiesLoader.java
@@ -29,6 +29,7 @@ import java.util.Optional;
import java.util.Properties;
import java.util.stream.Stream;
import javax.crypto.Cipher;
+import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.util.NiFiProperties;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.slf4j.Logger;
@@ -53,7 +54,7 @@ public class NiFiPropertiesLoader {
/**
* Returns an instance of the loader configured with the key.
- *
+ * <p>
* <p>
* NOTE: This method is used reflectively by the process which starts NiFi
* so changes to it must be made in conjunction with that mechanism.</p>
@@ -109,31 +110,48 @@ public class NiFiPropertiesLoader {
* @throws IOException if the file is not readable
*/
public static String extractKeyFromBootstrapFile() throws IOException {
- // Guess at location of bootstrap.conf file from nifi.properties file
- String defaultNiFiPropertiesPath = getDefaultFilePath();
- File propertiesFile = new File(defaultNiFiPropertiesPath);
- File confDir = new File(propertiesFile.getParent());
- if (confDir.exists() && confDir.canRead()) {
- File expectedBootstrapFile = new File(confDir, "bootstrap.conf");
- if (expectedBootstrapFile.exists() && expectedBootstrapFile.canRead()) {
- try (Stream<String> stream = Files.lines(Paths.get(expectedBootstrapFile.getAbsolutePath()))) {
- Optional<String> keyLine = stream.filter(l -> l.startsWith(BOOTSTRAP_KEY_PREFIX)).findFirst();
- if (keyLine.isPresent()) {
- return keyLine.get().split("=", 2)[1];
- } else {
- logger.warn("No encryption key present in the bootstrap.conf file at {}", expectedBootstrapFile.getAbsolutePath());
- return "";
- }
- } catch (IOException e) {
- logger.error("Cannot read from bootstrap.conf file at {} to extract encryption key", expectedBootstrapFile.getAbsolutePath());
- throw new IOException("Cannot read from bootstrap.conf", e);
- }
+ return extractKeyFromBootstrapFile("");
+ }
+
+ /**
+ * Returns the key (if any) used to encrypt sensitive properties, extracted from {@code $NIFI_HOME/conf/bootstrap.conf}.
+ *
+ * @param bootstrapPath the path to the bootstrap file
+ * @return the key in hexadecimal format
+ * @throws IOException if the file is not readable
+ */
+ public static String extractKeyFromBootstrapFile(String bootstrapPath) throws IOException {
+ File expectedBootstrapFile;
+ if (StringUtils.isBlank(bootstrapPath)) {
+ // Guess at location of bootstrap.conf file from nifi.properties file
+ String defaultNiFiPropertiesPath = getDefaultFilePath();
+ File propertiesFile = new File(defaultNiFiPropertiesPath);
+ File confDir = new File(propertiesFile.getParent());
+ if (confDir.exists() && confDir.canRead()) {
+ expectedBootstrapFile = new File(confDir, "bootstrap.conf");
} else {
- logger.error("Cannot read from bootstrap.conf file at {} to extract encryption key -- file is missing or permissions are incorrect", expectedBootstrapFile.getAbsolutePath());
+ logger.error("Cannot read from bootstrap.conf file at {} to extract encryption key -- conf/ directory is missing or permissions are incorrect", confDir.getAbsolutePath());
throw new IOException("Cannot read from bootstrap.conf");
}
} else {
- logger.error("Cannot read from bootstrap.conf file at {} to extract encryption key -- conf/ directory is missing or permissions are incorrect", confDir.getAbsolutePath());
+ expectedBootstrapFile = new File(bootstrapPath);
+ }
+
+ if (expectedBootstrapFile.exists() && expectedBootstrapFile.canRead()) {
+ try (Stream<String> stream = Files.lines(Paths.get(expectedBootstrapFile.getAbsolutePath()))) {
+ Optional<String> keyLine = stream.filter(l -> l.startsWith(BOOTSTRAP_KEY_PREFIX)).findFirst();
+ if (keyLine.isPresent()) {
+ return keyLine.get().split("=", 2)[1];
+ } else {
+ logger.warn("No encryption key present in the bootstrap.conf file at {}", expectedBootstrapFile.getAbsolutePath());
+ return "";
+ }
+ } catch (IOException e) {
+ logger.error("Cannot read from bootstrap.conf file at {} to extract encryption key", expectedBootstrapFile.getAbsolutePath());
+ throw new IOException("Cannot read from bootstrap.conf", e);
+ }
+ } else {
+ logger.error("Cannot read from bootstrap.conf file at {} to extract encryption key -- file is missing or permissions are incorrect", expectedBootstrapFile.getAbsolutePath());
throw new IOException("Cannot read from bootstrap.conf");
}
}
@@ -254,7 +272,7 @@ public class NiFiPropertiesLoader {
/**
* Returns the loaded {@link NiFiProperties} instance. If none is currently
* loaded, attempts to load the default instance.
- *
+ * <p>
* <p>
* NOTE: This method is used reflectively by the process which starts NiFi
* so changes to it must be made in conjunction with that mechanism.</p>
http://git-wip-us.apache.org/repos/asf/nifi/blob/2c371453/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/ProtectedNiFiProperties.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/ProtectedNiFiProperties.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/ProtectedNiFiProperties.java
index 83320a0..4774dc7 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/ProtectedNiFiProperties.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/ProtectedNiFiProperties.java
@@ -284,7 +284,7 @@ class ProtectedNiFiProperties extends StandardNiFiProperties {
* @param key the key identifying the sensitive property
* @return the key identifying the protection scheme for the sensitive property
*/
- public String getProtectionKey(String key) {
+ public static String getProtectionKey(String key) {
if (key == null || key.isEmpty()) {
throw new IllegalArgumentException("Cannot find protection key for null key");
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/2c371453/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/AESSensitivePropertyProviderTest.groovy
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/AESSensitivePropertyProviderTest.groovy b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/AESSensitivePropertyProviderTest.groovy
index 4c5a34d..7896afe 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/AESSensitivePropertyProviderTest.groovy
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/AESSensitivePropertyProviderTest.groovy
@@ -447,7 +447,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
@Test
public void testShouldEncryptArbitraryValues() {
// Arrange
- def values = ["thisIsABadPassword", "thisIsABadSensitiveKeyPassword", "thisIsABadKeystorePassword", "thisIsABadKeyPassword", "thisIsABadTruststorePassword", "This is an encrypted banner message"]
+ def values = ["thisIsABadPassword", "thisIsABadSensitiveKeyPassword", "thisIsABadKeystorePassword", "thisIsABadKeyPassword", "thisIsABadTruststorePassword", "This is an encrypted banner message", "nififtw!"]
String key = "2C576A9585DB862F5ECBEE5B4FFFCCA1" //getKeyOfSize(128)
// key = "0" * 64
http://git-wip-us.apache.org/repos/asf/nifi/blob/2c371453/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml b/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml
index f2ef7e9..22e83f4 100644
--- a/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml
+++ b/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml
@@ -58,6 +58,18 @@
<version>1.16.0</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-framework-core</artifactId>
+ <version>1.1.0-SNAPSHOT</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/nifi/blob/2c371453/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy
----------------------------------------------------------------------
diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy
index 0f69d1b..71166f0 100644
--- a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy
+++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy
@@ -25,6 +25,7 @@ import org.apache.commons.cli.HelpFormatter
import org.apache.commons.cli.Options
import org.apache.commons.cli.ParseException
import org.apache.commons.codec.binary.Hex
+import org.apache.commons.io.IOUtils
import org.apache.nifi.toolkit.tls.commandLine.CommandLineParseException
import org.apache.nifi.toolkit.tls.commandLine.ExitCode
import org.apache.nifi.util.NiFiProperties
@@ -37,9 +38,16 @@ import org.slf4j.LoggerFactory
import org.xml.sax.SAXException
import javax.crypto.Cipher
+import javax.crypto.SecretKey
+import javax.crypto.SecretKeyFactory
+import javax.crypto.spec.PBEKeySpec
+import javax.crypto.spec.PBEParameterSpec
import java.nio.charset.StandardCharsets
import java.security.KeyException
+import java.security.SecureRandom
import java.security.Security
+import java.util.zip.GZIPInputStream
+import java.util.zip.GZIPOutputStream
class ConfigEncryptionTool {
private static final Logger logger = LoggerFactory.getLogger(ConfigEncryptionTool.class)
@@ -49,14 +57,23 @@ class ConfigEncryptionTool {
public String outputNiFiPropertiesPath
public String loginIdentityProvidersPath
public String outputLoginIdentityProvidersPath
+ public String flowXmlPath
+ public String outputFlowXmlPath
private String keyHex
private String migrationKeyHex
private String password
private String migrationPassword
+ // This is the raw value used in nifi.sensitive.props.key
+ private String flowPropertiesPassword
+
+ private String newFlowAlgorithm
+ private String newFlowProvider
+
private NiFiProperties niFiProperties
private String loginIdentityProviders
+ private String flowXml
private boolean usingPassword = true
private boolean usingPasswordMigration = true
@@ -64,6 +81,8 @@ class ConfigEncryptionTool {
private boolean isVerbose = false
private boolean handlingNiFiProperties = false
private boolean handlingLoginIdentityProviders = false
+ private boolean handlingFlowXml = false
+ private boolean ignorePropertiesFiles = false
private static final String HELP_ARG = "help"
private static final String VERBOSE_ARG = "verbose"
@@ -72,13 +91,21 @@ class ConfigEncryptionTool {
private static final String LOGIN_IDENTITY_PROVIDERS_ARG = "loginIdentityProviders"
private static final String OUTPUT_NIFI_PROPERTIES_ARG = "outputNiFiProperties"
private static final String OUTPUT_LOGIN_IDENTITY_PROVIDERS_ARG = "outputLoginIdentityProviders"
+ private static final String FLOW_XML_ARG = "flowXml"
+ private static final String OUTPUT_FLOW_XML_ARG = "outputFlowXml"
private static final String KEY_ARG = "key"
private static final String PASSWORD_ARG = "password"
private static final String KEY_MIGRATION_ARG = "oldKey"
private static final String PASSWORD_MIGRATION_ARG = "oldPassword"
private static final String USE_KEY_ARG = "useRawKey"
private static final String MIGRATION_ARG = "migrate"
+ private static final String PROPS_KEY_ARG = "propsKey"
+ private static final String DO_NOT_ENCRYPT_NIFI_PROPERTIES_ARG = "encryptFlowXmlOnly"
+ private static final String NEW_FLOW_ALGORITHM_ARG = "newFlowAlgorithm"
+ private static final String NEW_FLOW_PROVIDER_ARG = "newFlowProvider"
+ // Hard-coded fallback value from {@link org.apache.nifi.encrypt.StringEncryptor}
+ private static final String DEFAULT_NIFI_SENSITIVE_PROPS_KEY = "nififtw!"
private static final int MIN_PASSWORD_LENGTH = 12
// Strong parameters as of 12 Aug 2016
@@ -86,6 +113,10 @@ class ConfigEncryptionTool {
private static final int SCRYPT_R = 8
private static final int SCRYPT_P = 1
+ // Hard-coded values from StandardPBEByteEncryptor which will be removed during refactor of all flow encryption code in NIFI-1465
+ private static final int DEFAULT_KDF_ITERATIONS = 1000
+ private static final int DEFAULT_SALT_SIZE_BYTES = 16
+
private static
final String BOOTSTRAP_KEY_COMMENT = "# Master key in hexadecimal format for encrypted sensitive configuration values"
private static final String BOOTSTRAP_KEY_PREFIX = "nifi.bootstrap.sensitive.key="
@@ -96,10 +127,20 @@ class ConfigEncryptionTool {
private static final String FOOTER = buildFooter()
private static
- final String DEFAULT_DESCRIPTION = "This tool reads from a nifi.properties and/or login-identity-providers.xml file with plain sensitive configuration values, prompts the user for a master key, and encrypts each value. It will replace the plain value with the protected value in the same file (or write to a new file if specified)."
+ final String DEFAULT_DESCRIPTION = "This tool reads from a nifi.properties and/or " +
+ "login-identity-providers.xml file with plain sensitive configuration values, " +
+ "prompts the user for a master key, and encrypts each value. It will replace the " +
+ "plain value with the protected value in the same file (or write to a new file if " +
+ "specified). It can also be used to migrate already-encrypted values in those " +
+ "files or in flow.xml.gz to be encrypted with a new key."
private static final String LDAP_PROVIDER_CLASS = "org.apache.nifi.ldap.LdapProvider"
- static private final String LDAP_PROVIDER_REGEX = /<provider>[\s\S]*?<class>\s*org\.apache\.nifi\.ldap\.LdapProvider[\s\S]*?<\/provider>/
- static private final String XML_DECLARATION_REGEX = /<\?xml version="1.0" encoding="UTF-8"\?>/
+ private static
+ final String LDAP_PROVIDER_REGEX = /<provider>[\s\S]*?<class>\s*org\.apache\.nifi\.ldap\.LdapProvider[\s\S]*?<\/provider>/
+ private static final String XML_DECLARATION_REGEX = /<\?xml version="1.0" encoding="UTF-8"\?>/
+ private static final String WRAPPED_FLOW_XML_CIPHER_TEXT_REGEX = /enc\{[a-fA-F0-9]+?\}/
+
+ private static final String DEFAULT_PROVIDER = BouncyCastleProvider.PROVIDER_NAME
+ private static final String DEFAULT_FLOW_ALGORITHM = "PBEWITHMD5AND256BITAES-CBC-OPENSSL"
private static String buildHeader(String description = DEFAULT_DESCRIPTION) {
"${SEP}${description}${SEP * 2}"
@@ -109,8 +150,8 @@ class ConfigEncryptionTool {
"${SEP}Java home: ${System.getenv(JAVA_HOME)}${SEP}NiFi Toolkit home: ${System.getenv(NIFI_TOOLKIT_HOME)}"
}
- private final Options options;
- private final String header;
+ private final Options options
+ private final String header
public ConfigEncryptionTool() {
@@ -124,15 +165,21 @@ class ConfigEncryptionTool {
options.addOption("v", VERBOSE_ARG, false, "Sets verbose mode (default false)")
options.addOption("n", NIFI_PROPERTIES_ARG, true, "The nifi.properties file containing unprotected config values (will be overwritten)")
options.addOption("l", LOGIN_IDENTITY_PROVIDERS_ARG, true, "The login-identity-providers.xml file containing unprotected config values (will be overwritten)")
+ options.addOption("f", FLOW_XML_ARG, true, "The flow.xml.gz file currently protected with old password (will be overwritten)")
options.addOption("b", BOOTSTRAP_CONF_ARG, true, "The bootstrap.conf file to persist master key")
options.addOption("o", OUTPUT_NIFI_PROPERTIES_ARG, true, "The destination nifi.properties file containing protected config values (will not modify input nifi.properties)")
options.addOption("i", OUTPUT_LOGIN_IDENTITY_PROVIDERS_ARG, true, "The destination login-identity-providers.xml file containing protected config values (will not modify input login-identity-providers.xml)")
+ options.addOption("g", OUTPUT_FLOW_XML_ARG, true, "The destination flow.xml.gz file containing protected config values (will not modify input flow.xml.gz)")
options.addOption("k", KEY_ARG, true, "The raw hexadecimal key to use to encrypt the sensitive properties")
options.addOption("e", KEY_MIGRATION_ARG, true, "The old raw hexadecimal key to use during key migration")
options.addOption("p", PASSWORD_ARG, true, "The password from which to derive the key to use to encrypt the sensitive properties")
options.addOption("w", PASSWORD_MIGRATION_ARG, true, "The old password from which to derive the key during migration")
options.addOption("r", USE_KEY_ARG, false, "If provided, the secure console will prompt for the raw key value in hexadecimal form")
- options.addOption("m", MIGRATION_ARG, false, "If provided, the sensitive properties will be re-encrypted with a new key")
+ options.addOption("m", MIGRATION_ARG, false, "If provided, the nifi.properties and/or login-identity-providers.xml sensitive properties will be re-encrypted with a new key")
+ options.addOption("x", DO_NOT_ENCRYPT_NIFI_PROPERTIES_ARG, false, "If provided, the properties in flow.xml.gz will be re-encrypted with a new key but the nifi.properties and/or login-identity-providers.xml files will not be modified")
+ options.addOption("s", PROPS_KEY_ARG, true, "The password or key to use to encrypt the sensitive processor properties in flow.xml.gz")
+ options.addOption("A", NEW_FLOW_ALGORITHM_ARG, true, "The algorithm to use to encrypt the sensitive processor properties in flow.xml.gz")
+ options.addOption("P", NEW_FLOW_PROVIDER_ARG, true, "The security provider to use to encrypt the sensitive processor properties in flow.xml.gz")
}
/**
@@ -169,13 +216,36 @@ class ConfigEncryptionTool {
bootstrapConfPath = commandLine.getOptionValue(BOOTSTRAP_CONF_ARG)
+ // If this flag is provided, the nifi.properties is necessary to read/write the flow encryption key, but the encryption process will not actually be applied to nifi.properties / login-identity-providers.xml
+ if (commandLine.hasOption(DO_NOT_ENCRYPT_NIFI_PROPERTIES_ARG)) {
+ handlingNiFiProperties = false
+ handlingLoginIdentityProviders = false
+ ignorePropertiesFiles = true
+ } else {
+ if (commandLine.hasOption(LOGIN_IDENTITY_PROVIDERS_ARG)) {
+ if (isVerbose) {
+ logger.info("Handling encryption of login-identity-providers.xml")
+ }
+ loginIdentityProvidersPath = commandLine.getOptionValue(LOGIN_IDENTITY_PROVIDERS_ARG)
+ outputLoginIdentityProvidersPath = commandLine.getOptionValue(OUTPUT_LOGIN_IDENTITY_PROVIDERS_ARG, loginIdentityProvidersPath)
+ handlingLoginIdentityProviders = true
+
+ if (loginIdentityProvidersPath == outputLoginIdentityProvidersPath) {
+ // TODO: Add confirmation pause and provide -y flag to offer no-interaction mode?
+ logger.warn("The source login-identity-providers.xml and destination login-identity-providers.xml are identical [${outputLoginIdentityProvidersPath}] so the original will be overwritten")
+ }
+ }
+ }
+
+ // This needs to occur even if the nifi.properties won't be encrypted
if (commandLine.hasOption(NIFI_PROPERTIES_ARG)) {
- if (isVerbose) {
+ boolean ignoreFlagPresent = commandLine.hasOption(DO_NOT_ENCRYPT_NIFI_PROPERTIES_ARG)
+ if (isVerbose && !ignoreFlagPresent) {
logger.info("Handling encryption of nifi.properties")
}
niFiPropertiesPath = commandLine.getOptionValue(NIFI_PROPERTIES_ARG)
outputNiFiPropertiesPath = commandLine.getOptionValue(OUTPUT_NIFI_PROPERTIES_ARG, niFiPropertiesPath)
- handlingNiFiProperties = true
+ handlingNiFiProperties = !ignoreFlagPresent
if (niFiPropertiesPath == outputNiFiPropertiesPath) {
// TODO: Add confirmation pause and provide -y flag to offer no-interaction mode?
@@ -183,17 +253,24 @@ class ConfigEncryptionTool {
}
}
- if (commandLine.hasOption(LOGIN_IDENTITY_PROVIDERS_ARG)) {
+ if (commandLine.hasOption(FLOW_XML_ARG)) {
if (isVerbose) {
- logger.info("Handling encryption of login-identity-providers.xml")
+ logger.info("Handling encryption of flow.xml.gz")
}
- loginIdentityProvidersPath = commandLine.getOptionValue(LOGIN_IDENTITY_PROVIDERS_ARG)
- outputLoginIdentityProvidersPath = commandLine.getOptionValue(OUTPUT_LOGIN_IDENTITY_PROVIDERS_ARG, loginIdentityProvidersPath)
- handlingLoginIdentityProviders = true
+ flowXmlPath = commandLine.getOptionValue(FLOW_XML_ARG)
+ outputFlowXmlPath = commandLine.getOptionValue(OUTPUT_FLOW_XML_ARG, flowXmlPath)
+ handlingFlowXml = true
+
+ newFlowAlgorithm = commandLine.getOptionValue(NEW_FLOW_ALGORITHM_ARG)
+ newFlowProvider = commandLine.getOptionValue(NEW_FLOW_PROVIDER_ARG)
- if (loginIdentityProvidersPath == outputLoginIdentityProvidersPath) {
+ if (flowXmlPath == outputFlowXmlPath) {
// TODO: Add confirmation pause and provide -y flag to offer no-interaction mode?
- logger.warn("The source login-identity-providers.xml and destination login-identity-providers.xml are identical [${outputLoginIdentityProvidersPath}] so the original will be overwritten")
+ logger.warn("The source flow.xml.gz and destination flow.xml.gz are identical [${outputFlowXmlPath}] so the original will be overwritten")
+ }
+
+ if (!commandLine.hasOption(NIFI_PROPERTIES_ARG)) {
+ printUsageAndThrow("In order to migrate a flow.xml.gz, a nifi.properties file must also be specified via '-n'/'--${NIFI_PROPERTIES_ARG}'.", ExitCode.INVALID_ARGS)
}
}
@@ -203,6 +280,8 @@ class ConfigEncryptionTool {
logger.info("(dest) nifi.properties: \t${outputNiFiPropertiesPath}")
logger.info("(src) login-identity-providers.xml: \t${loginIdentityProvidersPath}")
logger.info("(dest) login-identity-providers.xml: \t${outputLoginIdentityProvidersPath}")
+ logger.info("(src) flow.xml.gz: \t\t\t\t\t${flowXmlPath}")
+ logger.info("(dest) flow.xml.gz: \t\t\t\t\t${outputFlowXmlPath}")
}
// TODO: Implement in NIFI-2655
@@ -251,6 +330,10 @@ class ConfigEncryptionTool {
usingPassword = false
}
}
+
+ if (commandLine.hasOption(PROPS_KEY_ARG)) {
+ flowPropertiesPassword = commandLine.getOptionValue(PROPS_KEY_ARG)
+ }
} catch (ParseException e) {
if (isVerbose) {
logger.error("Encountered an error", e)
@@ -299,6 +382,10 @@ class ConfigEncryptionTool {
getKeyInternal(TextDevices.defaultTextDevice(), migrationKeyHex, migrationPassword, usingPasswordMigration)
}
+ private String getFlowPassword(TextDevice textDevice = TextDevices.defaultTextDevice()) {
+ readPasswordFromConsole(textDevice)
+ }
+
private static String readKeyFromConsole(TextDevice textDevice) {
textDevice.printf("Enter the master key in hexadecimal format (spaces acceptable): ")
new String(textDevice.readPassword())
@@ -387,6 +474,227 @@ class ConfigEncryptionTool {
}
}
+ /**
+ * Loads the flow definition from the provided file path, handling the GZIP file compression. Unlike {@link #loadLoginIdentityProviders()} this method does not decrypt the content (for performance and separation of concern reasons).
+ *
+ * @return the file content
+ * @throw IOException if the flow.xml.gz file cannot be read
+ */
+ private String loadFlowXml() throws IOException {
+ File flowXmlFile
+ if (flowXmlPath && (flowXmlFile = new File(flowXmlPath)).exists()) {
+ try {
+ new FileInputStream(flowXmlPath).withCloseable {
+ new GZIPInputStream(it).withCloseable {
+ String xmlContent = IOUtils.toString(it, StandardCharsets.UTF_8)
+ return xmlContent
+ }
+ }
+ } catch (RuntimeException e) {
+ if (isVerbose) {
+ logger.error("Encountered an error", e)
+ }
+ throw new IOException("Cannot load flow from [${flowXmlPath}]", e)
+ }
+ } else {
+ printUsageAndThrow("Cannot load flow from [${flowXmlPath}]", ExitCode.ERROR_READING_NIFI_PROPERTIES)
+ }
+ }
+
+ /**
+ * Decrypts a single element encrypted in the flow.xml.gz style (hex-encoded and wrapped with "enc{" and "}").
+ *
+ * Example:
+ * {@code enc{0123456789ABCDEF} } -> "some text"
+ *
+ * @param wrappedCipherText the wrapped and hex-encoded cipher text
+ * @param password the password used to encrypt the content (UTF-8 encoded)
+ * @param algorithm the encryption and KDF algorithm (defaults to PBEWITHMD5AND256BITAES-CBC-OPENSSL)
+ * @param provider the security provider (defaults to BC)
+ * @return the plaintext in UTF-8 encoding
+ */
+ private
+ static String decryptFlowElement(String wrappedCipherText, String password, String algorithm = DEFAULT_FLOW_ALGORITHM, String provider = DEFAULT_PROVIDER) {
+ // Drop the "enc{" and closing "}"
+ if (!(wrappedCipherText =~ WRAPPED_FLOW_XML_CIPHER_TEXT_REGEX)) {
+ throw new SensitivePropertyProtectionException("The provided cipher text does not match the expected format 'enc{0123456789ABCDEF...}'")
+ }
+ String unwrappedCipherText = wrappedCipherText.replaceAll(/enc\{/, "")[0..<-1]
+ if (unwrappedCipherText.length() % 2 == 1 || unwrappedCipherText.length() == 0) {
+ throw new SensitivePropertyProtectionException("The provided cipher text must have an even number of hex characters")
+ }
+
+ // Decode the hex
+ byte[] cipherBytes = Hex.decodeHex(unwrappedCipherText.chars)
+
+ /* The structure of each cipher text is 16 bytes of salt || actual cipher text,
+ * so extract the salt (32 bytes encoded as hex, 16 bytes raw) and combine that
+ * with the default (and unchanged) iteration count that is hardcoded in
+ * {@link StandardPBEByteEncryptor}. I am extracting
+ * these values to magic numbers here so when the refactoring is performed,
+ * stronger decisions can be implemented here
+ */
+ byte[] saltBytes = cipherBytes[0..<DEFAULT_SALT_SIZE_BYTES]
+ cipherBytes = cipherBytes[DEFAULT_SALT_SIZE_BYTES..-1]
+
+ Cipher decryptionCipher = generateFlowDecryptionCipher(password, saltBytes, algorithm, provider)
+
+ byte[] plainBytes = decryptionCipher.doFinal(cipherBytes)
+ new String(plainBytes, StandardCharsets.UTF_8)
+ }
+
+ /**
+ * Returns an initialized {@link javax.crypto.Cipher} instance with the extracted salt.
+ *
+ * @param password the password (UTF-8 encoding)
+ * @param saltBytes the salt (raw bytes)
+ * @param algorithm the KDF/encryption algorithm
+ * @param provider the security provider
+ * @return the initialized {@link javax.crypto.Cipher}
+ */
+ private
+ static Cipher generateFlowDecryptionCipher(String password, byte[] saltBytes, String algorithm = DEFAULT_FLOW_ALGORITHM, String provider = DEFAULT_PROVIDER) {
+ Cipher decryptCipher = Cipher.getInstance(algorithm, provider)
+ PBEKeySpec keySpec = new PBEKeySpec(password.chars)
+ SecretKeyFactory keyFactory = SecretKeyFactory.getInstance(algorithm, provider)
+ SecretKey pbeKey = keyFactory.generateSecret(keySpec)
+ PBEParameterSpec parameterSpec = new PBEParameterSpec(saltBytes, DEFAULT_KDF_ITERATIONS)
+ decryptCipher.init(Cipher.DECRYPT_MODE, pbeKey, parameterSpec)
+ decryptCipher
+ }
+
+ /**
+ * Encrypts a single element in the flow.xml.gz style (hex-encoded and wrapped with "enc{" and "}").
+ *
+ * Example:
+ * "some text" -> {@code enc{0123456789ABCDEF} }
+ *
+ * @param plaintext the plaintext in UTF-8 encoding
+ * @param saltBytes the salt to embed in the cipher text to allow key derivation and decryption later in raw format
+ * @param encryptCipher the configured Cipher instance
+ * @return the wrapped and hex-encoded cipher text
+ */
+ private static String encryptFlowElement(String plaintext, byte[] saltBytes, Cipher encryptCipher) {
+ byte[] plainBytes = plaintext?.getBytes(StandardCharsets.UTF_8) ?: new byte[0]
+
+ /* The structure of each cipher text is 16 bytes of salt || actual cipher text,
+ * so extract the salt (32 bytes encoded as hex, 16 bytes raw) and combine that
+ * with the default (and unchanged) iteration count that is hardcoded in
+ * {@link StandardPBEByteEncryptor}. I am extracting
+ * these values to magic numbers here so when the refactoring is performed,
+ * stronger decisions can be implemented here
+ */
+ if (saltBytes.length != DEFAULT_SALT_SIZE_BYTES) {
+ throw new SensitivePropertyProtectionException("The salt must be ${DEFAULT_SALT_SIZE_BYTES} bytes")
+ }
+
+ byte[] cipherBytes = encryptCipher.doFinal(plainBytes)
+ byte[] saltAndCipherBytes = concatByteArrays(saltBytes, cipherBytes)
+
+ // Encode the hex
+ String hexEncodedCipherText = Hex.encodeHexString(saltAndCipherBytes)
+ "enc{${hexEncodedCipherText}}"
+ }
+
+ /**
+ * Utility method to quickly concatenate an arbitrary number of byte[].
+ *
+ * @param arrays the byte[] arrays
+ * @returna single byte[] containing the values concatenated
+ */
+ private static byte[] concatByteArrays(byte[] ... arrays) {
+ ByteArrayOutputStream outputStream = new ByteArrayOutputStream()
+ arrays.each { byte[] it -> outputStream.write(it) }
+ outputStream.toByteArray()
+ }
+
+ /**
+ * Scans XML content and decrypts each encrypted element, then re-encrypts it with the new key, and returns the final XML content.
+ *
+ * @param flowXmlContent the original flow.xml.gz content
+ * @param existingFlowPassword the existing value of nifi.sensitive.props.key (not a raw key, but rather a password)
+ * @param newFlowPassword the password to use to for encryption (not a raw key, but rather a password)
+ * @param existingAlgorithm the KDF algorithm to use (defaults to PBEWITHMD5AND256BITAES-CBC-OPENSSL)
+ * @param existingProvider the {@link java.security.Provider} to use (defaults to BC)
+ * @return the encrypted XML content
+ */
+ private String migrateFlowXmlContent(String flowXmlContent, String existingFlowPassword, String newFlowPassword, String existingAlgorithm = DEFAULT_FLOW_ALGORITHM, String existingProvider = DEFAULT_PROVIDER, String newAlgorithm = DEFAULT_FLOW_ALGORITHM, String newProvider = DEFAULT_PROVIDER) {
+ /* For re-encryption, for performance reasons, we will use a fixed salt for all of
+ * the operations. These values are stored in the same file and the default key is in the
+ * source code (see NIFI-1465 and NIFI-1277), so the security trade-off is minimal
+ * but the performance hit is substantial. We can't make this decision for
+ * decryption because the FlowSerializer still uses StringEncryptor which does not
+ * follow this pattern
+ */
+ byte[] encryptionSalt = new byte[DEFAULT_SALT_SIZE_BYTES]
+ new SecureRandom().nextBytes(encryptionSalt)
+ Cipher encryptCipher = generateFlowEncryptionCipher(newFlowPassword, encryptionSalt, newAlgorithm, newProvider)
+
+ int elementCount = 0
+
+ // Scan the XML content and identify every encrypted element, decrypt it, and replace it with the re-encrypted value
+ String migratedFlowXmlContent = flowXmlContent.replaceAll(WRAPPED_FLOW_XML_CIPHER_TEXT_REGEX) { String wrappedCipherText ->
+ String plaintext = decryptFlowElement(wrappedCipherText, existingFlowPassword, existingAlgorithm, existingProvider)
+ byte[] cipherBytes = encryptCipher.doFinal(plaintext.bytes)
+ byte[] saltAndCipherBytes = concatByteArrays(encryptionSalt, cipherBytes)
+ elementCount++
+ "enc{${Hex.encodeHex(saltAndCipherBytes)}}"
+ }
+
+ if (isVerbose) {
+ logger.info("Decrypted and re-encrypted ${elementCount} elements for flow.xml.gz")
+ }
+
+ migratedFlowXmlContent
+ }
+
+ /**
+ * Returns an initialized encryption cipher for the flow.xml.gz content.
+ *
+ * @param newFlowPassword the new encryption password
+ * @param saltBytes the salt [16 bytes in raw format]
+ * @param algorithm the KDF/encryption algorithm
+ * @param provider the security provider
+ * @return the initialized cipher instance
+ */
+ private
+ static Cipher generateFlowEncryptionCipher(String newFlowPassword, byte[] saltBytes, String algorithm = DEFAULT_FLOW_ALGORITHM, String provider = DEFAULT_PROVIDER) {
+ /* The Jasypt StringEncryptor implementation is final and has some design decisions
+ * that will pollute this code (i.e. using a random salt on every encrypt operation
+ * rather than a unique IV, so the derived key for every encrypt/decrypt operation is
+ * different, which is very wasteful), so just use the standard JCE ciphers with the
+ * password derived using the prescribed algorithm
+ */
+ Cipher encryptCipher = Cipher.getInstance(algorithm, provider)
+
+ /* For re-encryption, for performance reasons, we will use a fixed salt for all of
+ * the operations. These values are stored in the same file and the default key is in the
+ * source code (see NIFI-1465 and NIFI-1277), so the security trade-off is minimal
+ * but the performance hit is substantial. We can't make this decision for
+ * decryption because the FlowSerializer still uses StringEncryptor which does not
+ * follow this pattern
+ */
+ PBEKeySpec keySpec = new PBEKeySpec(newFlowPassword.chars)
+ SecretKeyFactory keyFactory = SecretKeyFactory.getInstance(algorithm, provider)
+ SecretKey pbeKey = keyFactory.generateSecret(keySpec)
+ PBEParameterSpec parameterSpec = new PBEParameterSpec(saltBytes, DEFAULT_KDF_ITERATIONS)
+ encryptCipher.init(Cipher.ENCRYPT_MODE, pbeKey, parameterSpec)
+ encryptCipher
+ }
+
+ /**
+ * Writes the XML content to the {@link .outputFlowXmlPath} location, handling the GZIP file compression.
+ *
+ * @param flowXmlContent the XML content to write
+ */
+ private void writeFlowXmlToFile(String flowXmlContent) {
+ new FileOutputStream(outputFlowXmlPath).withCloseable {
+ new GZIPOutputStream(it).withCloseable {
+ IOUtils.write(flowXmlContent, it, StandardCharsets.UTF_8)
+ }
+ }
+ }
+
String decryptLoginIdentityProviders(String encryptedXml, String existingKeyHex = keyHex) {
AESSensitivePropertyProvider sensitivePropertyProvider = new AESSensitivePropertyProvider(existingKeyHex)
@@ -654,8 +962,11 @@ class ConfigEncryptionTool {
List<String> lines = originalPropertiesFile.readLines()
ProtectedNiFiProperties protectedNiFiProperties = new ProtectedNiFiProperties(niFiProperties)
- // Only need to replace the keys that have been protected
+ // Only need to replace the keys that have been protected AND nifi.sensitive.props.key
Map<String, String> protectedKeys = protectedNiFiProperties.getProtectedPropertyKeys()
+ if (!protectedKeys.containsKey(NiFiProperties.SENSITIVE_PROPS_KEY)) {
+ protectedKeys.put(NiFiProperties.SENSITIVE_PROPS_KEY, protectedNiFiProperties.getProperty(ProtectedNiFiProperties.getProtectionKey(NiFiProperties.SENSITIVE_PROPS_KEY)))
+ }
protectedKeys.each { String key, String protectionScheme ->
int l = lines.findIndexOf { it.startsWith(key) }
@@ -664,7 +975,7 @@ class ConfigEncryptionTool {
}
// Get the index of the following line (or cap at max)
int p = l + 1 > lines.size() ? lines.size() : l + 1
- String protectionLine = "${protectedNiFiProperties.getProtectionKey(key)}=${protectionScheme}"
+ String protectionLine = "${protectedNiFiProperties.getProtectionKey(key)}=${protectionScheme ?: ""}"
if (p < lines.size() && lines.get(p).startsWith("${protectedNiFiProperties.getProtectionKey(key)}=")) {
lines.set(p, protectionLine)
} else {
@@ -765,6 +1076,28 @@ class ConfigEncryptionTool {
"NIFI_SCRYPT_SALT".getBytes(StandardCharsets.UTF_8)
}
+ private String getExistingFlowPassword() {
+ return niFiProperties.getProperty(NiFiProperties.SENSITIVE_PROPS_KEY) as String ?: DEFAULT_NIFI_SENSITIVE_PROPS_KEY
+ }
+
+ /**
+ * Utility method which returns true if the {@link org.apache.nifi.util.NiFiProperties} instance has encrypted properties.
+ *
+ * @return true if the properties instance will require a key to access
+ */
+ boolean niFiPropertiesAreEncrypted() {
+ if (niFiPropertiesPath) {
+ try {
+ def nfp = NiFiPropertiesLoader.withKey(keyHex).readProtectedPropertiesFromDisk(new File(niFiPropertiesPath))
+ return nfp.hasProtectedKeys()
+ } catch (SensitivePropertyProtectionException | IOException e) {
+ return true
+ }
+ } else {
+ return false
+ }
+ }
+
/**
* Runs main tool logic (parsing arguments, reading files, protecting properties, and writing key and properties out to destination files).
*
@@ -779,48 +1112,56 @@ class ConfigEncryptionTool {
try {
tool.parse(args)
- tool.keyHex = tool.getKey()
-
- if (!tool.keyHex) {
- tool.printUsageAndThrow("Hex key must be provided", ExitCode.INVALID_ARGS)
- }
-
- try {
- // Validate the length and format
- tool.keyHex = parseKey(tool.keyHex)
- } catch (KeyException e) {
- if (tool.isVerbose) {
- logger.error("Encountered an error", e)
+ boolean existingNiFiPropertiesAreEncrypted = tool.niFiPropertiesAreEncrypted()
+ if (!tool.ignorePropertiesFiles || (tool.handlingFlowXml && existingNiFiPropertiesAreEncrypted)) {
+ // If we are handling the flow.xml.gz and nifi.properties is already encrypted, try getting the key from bootstrap.conf rather than the console
+ if (tool.ignorePropertiesFiles) {
+ tool.keyHex = NiFiPropertiesLoader.extractKeyFromBootstrapFile(tool.bootstrapConfPath)
+ } else {
+ tool.keyHex = tool.getKey()
}
- tool.printUsageAndThrow(e.getMessage(), ExitCode.INVALID_ARGS)
- }
- if (tool.migration) {
- String migrationKeyHex = tool.getMigrationKey()
-
- if (!migrationKeyHex) {
- tool.printUsageAndThrow("Original hex key must be provided for migration", ExitCode.INVALID_ARGS)
+ if (!tool.keyHex) {
+ tool.printUsageAndThrow("Hex key must be provided", ExitCode.INVALID_ARGS)
}
try {
// Validate the length and format
- tool.migrationKeyHex = parseKey(migrationKeyHex)
+ tool.keyHex = parseKey(tool.keyHex)
} catch (KeyException e) {
if (tool.isVerbose) {
logger.error("Encountered an error", e)
}
tool.printUsageAndThrow(e.getMessage(), ExitCode.INVALID_ARGS)
}
+
+ if (tool.migration) {
+ String migrationKeyHex = tool.getMigrationKey()
+
+ if (!migrationKeyHex) {
+ tool.printUsageAndThrow("Original hex key must be provided for migration", ExitCode.INVALID_ARGS)
+ }
+
+ try {
+ // Validate the length and format
+ tool.migrationKeyHex = parseKey(migrationKeyHex)
+ } catch (KeyException e) {
+ if (tool.isVerbose) {
+ logger.error("Encountered an error", e)
+ }
+ tool.printUsageAndThrow(e.getMessage(), ExitCode.INVALID_ARGS)
+ }
+ }
}
String existingKeyHex = tool.migrationKeyHex ?: tool.keyHex
- if (tool.handlingNiFiProperties) {
+ // Load NiFiProperties for either scenario; only encrypt if "handling" (see after flow XML)
+ if (tool.handlingNiFiProperties || tool.handlingFlowXml) {
try {
tool.niFiProperties = tool.loadNiFiProperties(existingKeyHex)
} catch (Exception e) {
tool.printUsageAndThrow("Cannot migrate key if no previous encryption occurred", ExitCode.ERROR_READING_NIFI_PROPERTIES)
}
- tool.niFiProperties = tool.encryptSensitiveProperties(tool.niFiProperties)
}
if (tool.handlingLoginIdentityProviders) {
@@ -831,6 +1172,61 @@ class ConfigEncryptionTool {
}
tool.loginIdentityProviders = tool.encryptLoginIdentityProviders(tool.loginIdentityProviders)
}
+
+ if (tool.handlingFlowXml) {
+ try {
+ tool.flowXml = tool.loadFlowXml()
+ } catch (Exception e) {
+ tool.printUsageAndThrow("Cannot load flow.xml.gz", ExitCode.ERROR_READING_NIFI_PROPERTIES)
+ }
+
+ // If the flow password was not set in nifi.properties, use the hard-coded default
+ String existingFlowPassword = tool.getExistingFlowPassword()
+
+ // If the new password was not provided in the arguments, read from the console. If that is empty, use the same value (essentially a copy no-op)
+ String newFlowPassword = tool.flowPropertiesPassword ?: tool.getFlowPassword()
+ if (!newFlowPassword) {
+ newFlowPassword = existingFlowPassword
+ }
+
+ // Get the algorithms and providers
+ NiFiProperties nfp = tool.niFiProperties
+ String existingAlgorithm = nfp?.getProperty(NiFiProperties.SENSITIVE_PROPS_ALGORITHM) ?: DEFAULT_FLOW_ALGORITHM
+ String existingProvider = nfp?.getProperty(NiFiProperties.SENSITIVE_PROPS_PROVIDER) ?: DEFAULT_PROVIDER
+
+ String newAlgorithm = tool.newFlowAlgorithm ?: existingAlgorithm
+ String newProvider = tool.newFlowProvider ?: existingProvider
+
+ tool.flowXml = tool.migrateFlowXmlContent(tool.flowXml, existingFlowPassword, newFlowPassword, existingAlgorithm, existingProvider, newAlgorithm, newProvider)
+
+ // If the new key is the hard-coded internal value, don't persist it to nifi.properties
+ if (newFlowPassword != DEFAULT_NIFI_SENSITIVE_PROPS_KEY && newFlowPassword != existingFlowPassword) {
+ // Update the NiFiProperties object with the new flow password before it gets encrypted (wasteful, but NiFiProperties instances are immutable)
+ Properties rawProperties = new Properties()
+ nfp.getPropertyKeys().each { String k ->
+ rawProperties.put(k, nfp.getProperty(k))
+ }
+
+ // If the tool is not going to encrypt NiFiProperties and the existing file is already encrypted, encrypt and update the new sensitive props key
+ if (!tool.handlingNiFiProperties && existingNiFiPropertiesAreEncrypted) {
+ AESSensitivePropertyProvider spp = new AESSensitivePropertyProvider(tool.keyHex)
+ String encryptedSPK = spp.protect(newFlowPassword)
+ rawProperties.put(NiFiProperties.SENSITIVE_PROPS_KEY, encryptedSPK)
+ // Manually update the protection scheme or it will be lost
+ rawProperties.put(ProtectedNiFiProperties.getProtectionKey(NiFiProperties.SENSITIVE_PROPS_KEY), spp.getIdentifierKey())
+ if (tool.isVerbose) {
+ logger.info("Tool is not configured to encrypt nifi.properties, but the existing nifi.properties is encrypted and flow.xml.gz was migrated, so manually persisting the new encrypted value to nifi.properties")
+ }
+ } else {
+ rawProperties.put(NiFiProperties.SENSITIVE_PROPS_KEY, newFlowPassword)
+ }
+ tool.niFiProperties = new StandardNiFiProperties(rawProperties)
+ }
+ }
+
+ if (tool.handlingNiFiProperties) {
+ tool.niFiProperties = tool.encryptSensitiveProperties(tool.niFiProperties)
+ }
} catch (CommandLineParseException e) {
if (e.exitCode == ExitCode.HELP) {
System.exit(ExitCode.HELP.ordinal())
@@ -846,8 +1242,13 @@ class ConfigEncryptionTool {
try {
// Do this as part of a transaction?
synchronized (this) {
- tool.writeKeyToBootstrapConf()
- if (tool.handlingNiFiProperties) {
+ if (!tool.ignorePropertiesFiles) {
+ tool.writeKeyToBootstrapConf()
+ }
+ if (tool.handlingFlowXml) {
+ tool.writeFlowXmlToFile(tool.flowXml)
+ }
+ if (tool.handlingNiFiProperties || tool.handlingFlowXml) {
tool.writeNiFiProperties()
}
if (tool.handlingLoginIdentityProviders) {