You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by jg...@apache.org on 2022/02/22 17:58:52 UTC
[nifi] branch main updated: NIFI-9711 Added support for flow.json.gz in SetSensitivePropertiesKey
This is an automated email from the ASF dual-hosted git repository.
jgresock pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 45f8795 NIFI-9711 Added support for flow.json.gz in SetSensitivePropertiesKey
45f8795 is described below
commit 45f87951778f31d3bee06da7cd03ca5bc9e82c0f
Author: exceptionfactory <ex...@apache.org>
AuthorDate: Fri Feb 18 18:57:12 2022 -0500
NIFI-9711 Added support for flow.json.gz in SetSensitivePropertiesKey
Signed-off-by: Joe Gresock <jg...@gmail.com>
This closes #5783.
---
.../command/SetSensitivePropertiesKey.java | 40 +++++++----
.../command/SetSensitivePropertiesKeyTest.java | 81 +++++++++++++++++-----
.../src/test/resources/blank.nifi.properties | 1 +
...ifi.properties => legacy-blank.nifi.properties} | 0
.../src/test/resources/populated.nifi.properties | 1 +
.../src/main/asciidoc/administration-guide.adoc | 11 ++-
6 files changed, 100 insertions(+), 34 deletions(-)
diff --git a/nifi-commons/nifi-flow-encryptor/src/main/java/org/apache/nifi/flow/encryptor/command/SetSensitivePropertiesKey.java b/nifi-commons/nifi-flow-encryptor/src/main/java/org/apache/nifi/flow/encryptor/command/SetSensitivePropertiesKey.java
index 7f61aae..1a4ea8f 100644
--- a/nifi-commons/nifi-flow-encryptor/src/main/java/org/apache/nifi/flow/encryptor/command/SetSensitivePropertiesKey.java
+++ b/nifi-commons/nifi-flow-encryptor/src/main/java/org/apache/nifi/flow/encryptor/command/SetSensitivePropertiesKey.java
@@ -33,6 +33,7 @@ import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
+import java.util.Arrays;
import java.util.List;
import java.util.Properties;
import java.util.stream.Collectors;
@@ -51,9 +52,13 @@ public class SetSensitivePropertiesKey {
protected static final String CONFIGURATION_FILE = "nifi.flow.configuration.file";
+ protected static final String CONFIGURATION_JSON_FILE = "nifi.flow.configuration.json.file";
+
+ private static final List<String> CONFIGURATION_FILES = Arrays.asList(CONFIGURATION_FILE, CONFIGURATION_JSON_FILE);
+
private static final int MINIMUM_REQUIRED_LENGTH = 12;
- private static final String FLOW_XML_PREFIX = "flow.xml.";
+ private static final String FLOW_PREFIX = "nifi.flow.";
private static final String GZ_EXTENSION = ".gz";
@@ -82,7 +87,6 @@ public class SetSensitivePropertiesKey {
final File propertiesFile = new File(propertiesFilePath);
final Properties properties = loadProperties(propertiesFile);
- final File flowConfigurationFile = getFlowConfigurationFile(properties);
try {
storeProperties(propertiesFile, outputPropertiesKey);
System.out.printf("NiFi Properties Processed [%s]%n", propertiesFilePath);
@@ -91,15 +95,27 @@ public class SetSensitivePropertiesKey {
throw new UncheckedIOException(message, e);
}
- if (flowConfigurationFile.exists()) {
- final String algorithm = getAlgorithm(properties);
- final PropertyEncryptor outputEncryptor = getPropertyEncryptor(outputPropertiesKey, algorithm);
- processFlowConfiguration(properties, outputEncryptor);
+ processFlowConfigurationFiles(properties, outputPropertiesKey);
+ }
+
+ private static void processFlowConfigurationFiles(final Properties properties, final String outputPropertiesKey) {
+ final String algorithm = getAlgorithm(properties);
+ final PropertyEncryptor outputEncryptor = getPropertyEncryptor(outputPropertiesKey, algorithm);
+
+ for (final String configurationFilePropertyName : CONFIGURATION_FILES) {
+ final String configurationFileProperty = properties.getProperty(configurationFilePropertyName);
+ if (configurationFileProperty == null || configurationFileProperty.isEmpty()) {
+ System.out.printf("Flow Configuration Property not specified [%s]%n", configurationFileProperty);
+ } else {
+ final File configurationFile = new File(configurationFileProperty);
+ if (configurationFile.exists()) {
+ processFlowConfiguration(configurationFile, properties, outputEncryptor);
+ }
+ }
}
}
- private static void processFlowConfiguration(final Properties properties, final PropertyEncryptor outputEncryptor) {
- final File flowConfigurationFile = getFlowConfigurationFile(properties);
+ private static void processFlowConfiguration(final File flowConfigurationFile, final Properties properties, final PropertyEncryptor outputEncryptor) {
try (final InputStream flowInputStream = new GZIPInputStream(new FileInputStream(flowConfigurationFile))) {
final File flowOutputFile = getFlowOutputFile();
final Path flowOutputPath = flowOutputFile.toPath();
@@ -115,7 +131,7 @@ public class SetSensitivePropertiesKey {
final Path flowConfigurationPath = flowConfigurationFile.toPath();
Files.move(flowOutputPath, flowConfigurationPath, StandardCopyOption.REPLACE_EXISTING);
System.out.printf("Flow Configuration Processed [%s]%n", flowConfigurationPath);
- } catch (final IOException|RuntimeException e) {
+ } catch (final IOException | RuntimeException e) {
System.err.printf("Failed to process Flow Configuration [%s]%n", flowConfigurationFile);
e.printStackTrace();
}
@@ -138,7 +154,7 @@ public class SetSensitivePropertiesKey {
}
private static File getFlowOutputFile() throws IOException {
- final File flowOutputFile = File.createTempFile(FLOW_XML_PREFIX, GZ_EXTENSION);
+ final File flowOutputFile = File.createTempFile(FLOW_PREFIX, GZ_EXTENSION);
flowOutputFile.deleteOnExit();
return flowOutputFile;
}
@@ -170,8 +186,4 @@ public class SetSensitivePropertiesKey {
private static PropertyEncryptor getPropertyEncryptor(final String propertiesKey, final String propertiesAlgorithm) {
return new PropertyEncryptorBuilder(propertiesKey).setAlgorithm(propertiesAlgorithm).build();
}
-
- private static File getFlowConfigurationFile(final Properties properties) {
- return new File(properties.getProperty(CONFIGURATION_FILE));
- }
}
diff --git a/nifi-commons/nifi-flow-encryptor/src/test/java/org/apache/nifi/flow/encryptor/command/SetSensitivePropertiesKeyTest.java b/nifi-commons/nifi-flow-encryptor/src/test/java/org/apache/nifi/flow/encryptor/command/SetSensitivePropertiesKeyTest.java
index 8a2a4d8..5675bb9 100644
--- a/nifi-commons/nifi-flow-encryptor/src/test/java/org/apache/nifi/flow/encryptor/command/SetSensitivePropertiesKeyTest.java
+++ b/nifi-commons/nifi-flow-encryptor/src/test/java/org/apache/nifi/flow/encryptor/command/SetSensitivePropertiesKeyTest.java
@@ -21,9 +21,11 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URISyntaxException;
+import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -33,11 +35,27 @@ import java.util.Optional;
import java.util.UUID;
import java.util.stream.Collectors;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
public class SetSensitivePropertiesKeyTest {
- private static final String FLOW_CONTENTS = "<property><value>PROPERTY</value></property>";
+ private static final String TEMP_FILE_PREFIX = SetSensitivePropertiesKeyTest.class.getSimpleName();
+
+ private static final String FLOW_CONTENTS_JSON = "{\"property\":\"value\"}";
+
+ private static final String FLOW_CONTENTS_XML = "<property><value>PROPERTY</value></property>";
+
+ private static final String JSON_GZ = ".json.gz";
+
+ private static final String XML_GZ = ".xml.gz";
+
+ private static final String PROPERTIES_EXTENSION = ".properties";
+
+ private static final String BLANK_PROPERTIES = "/blank.nifi.properties";
+
+ private static final String POPULATED_PROPERTIES = "/populated.nifi.properties";
+
+ private static final String LEGACY_BLANK_PROPERTIES = "/legacy-blank.nifi.properties";
@AfterEach
public void clearProperties() {
@@ -51,8 +69,9 @@ public class SetSensitivePropertiesKeyTest {
@Test
public void testMainBlankKeyAndAlgorithm() throws IOException, URISyntaxException {
- final Path flowConfiguration = getFlowConfiguration();
- final Path propertiesPath = getNiFiProperties(flowConfiguration, "/blank.nifi.properties");
+ final Path flowConfiguration = getFlowConfiguration(FLOW_CONTENTS_XML, XML_GZ);
+ final Path flowConfigurationJson = getFlowConfiguration(FLOW_CONTENTS_JSON, JSON_GZ);
+ final Path propertiesPath = getNiFiProperties(flowConfiguration, flowConfigurationJson, BLANK_PROPERTIES);
System.setProperty(SetSensitivePropertiesKey.PROPERTIES_FILE_PATH, propertiesPath.toString());
@@ -60,13 +79,26 @@ public class SetSensitivePropertiesKeyTest {
SetSensitivePropertiesKey.main(new String[]{sensitivePropertiesKey});
assertPropertiesKeyUpdated(propertiesPath, sensitivePropertiesKey);
- assertTrue("Flow Configuration not found", flowConfiguration.toFile().exists());
}
@Test
public void testMainPopulatedKeyAndAlgorithm() throws IOException, URISyntaxException {
- final Path flowConfiguration = getFlowConfiguration();
- final Path propertiesPath = getNiFiProperties(flowConfiguration, "/populated.nifi.properties");
+ final Path flowConfiguration = getFlowConfiguration(FLOW_CONTENTS_XML, XML_GZ);
+ final Path flowConfigurationJson = getFlowConfiguration(FLOW_CONTENTS_JSON, JSON_GZ);
+ final Path propertiesPath = getNiFiProperties(flowConfiguration, flowConfigurationJson, POPULATED_PROPERTIES);
+
+ System.setProperty(SetSensitivePropertiesKey.PROPERTIES_FILE_PATH, propertiesPath.toString());
+
+ final String sensitivePropertiesKey = UUID.randomUUID().toString();
+ SetSensitivePropertiesKey.main(new String[]{sensitivePropertiesKey});
+
+ assertPropertiesKeyUpdated(propertiesPath, sensitivePropertiesKey);
+ }
+
+ @Test
+ public void testMainLegacyBlankKeyAndAlgorithm() throws IOException, URISyntaxException {
+ final Path flowConfiguration = getFlowConfiguration(FLOW_CONTENTS_XML, XML_GZ);
+ final Path propertiesPath = getNiFiProperties(flowConfiguration, null, LEGACY_BLANK_PROPERTIES);
System.setProperty(SetSensitivePropertiesKey.PROPERTIES_FILE_PATH, propertiesPath.toString());
@@ -74,7 +106,6 @@ public class SetSensitivePropertiesKeyTest {
SetSensitivePropertiesKey.main(new String[]{sensitivePropertiesKey});
assertPropertiesKeyUpdated(propertiesPath, sensitivePropertiesKey);
- assertTrue("Flow Configuration not found", flowConfiguration.toFile().exists());
}
private void assertPropertiesKeyUpdated(final Path propertiesPath, final String sensitivePropertiesKey) throws IOException {
@@ -82,37 +113,51 @@ public class SetSensitivePropertiesKeyTest {
.stream()
.filter(line -> line.startsWith(SetSensitivePropertiesKey.PROPS_KEY))
.findFirst();
- assertTrue("Sensitive Key Property not found", keyProperty.isPresent());
+ assertTrue(keyProperty.isPresent(), "Sensitive Key Property not found");
final String expectedProperty = String.format("%s=%s", SetSensitivePropertiesKey.PROPS_KEY, sensitivePropertiesKey);
- assertEquals("Sensitive Key Property not updated", expectedProperty, keyProperty.get());
+ assertEquals(expectedProperty, keyProperty.get(), "Sensitive Key Property not updated");
}
- private Path getNiFiProperties(final Path flowConfigurationPath, String propertiesResource) throws IOException, URISyntaxException {
- final Path sourcePropertiesPath = Paths.get(SetSensitivePropertiesKey.class.getResource(propertiesResource).toURI());
+ private Path getNiFiProperties(
+ final Path flowConfigurationPath,
+ final Path flowConfigurationJsonPath,
+ String propertiesResource
+ ) throws IOException, URISyntaxException {
+ final Path sourcePropertiesPath = Paths.get(getResourceUrl(propertiesResource).toURI());
final List<String> sourceProperties = Files.readAllLines(sourcePropertiesPath);
final List<String> flowProperties = sourceProperties.stream().map(line -> {
if (line.startsWith(SetSensitivePropertiesKey.CONFIGURATION_FILE)) {
- return line + flowConfigurationPath.toString();
+ return line + flowConfigurationPath;
+ } else if (line.startsWith(SetSensitivePropertiesKey.CONFIGURATION_JSON_FILE)) {
+ return flowConfigurationJsonPath == null ? line : line + flowConfigurationJsonPath;
} else {
return line;
}
}).collect(Collectors.toList());
- final Path propertiesPath = Files.createTempFile(SetSensitivePropertiesKey.class.getSimpleName(), ".properties");
+ final Path propertiesPath = Files.createTempFile(TEMP_FILE_PREFIX, PROPERTIES_EXTENSION);
propertiesPath.toFile().deleteOnExit();
Files.write(propertiesPath, flowProperties);
return propertiesPath;
}
- private Path getFlowConfiguration() throws IOException {
- final Path flowConfigurationPath = Files.createTempFile(SetSensitivePropertiesKey.class.getSimpleName(), ".xml.gz");
+ private Path getFlowConfiguration(final String contents, final String extension) throws IOException {
+ final Path flowConfigurationPath = Files.createTempFile(TEMP_FILE_PREFIX, extension);
final File flowConfigurationFile = flowConfigurationPath.toFile();
flowConfigurationFile.deleteOnExit();
try (final GZIPOutputStream outputStream = new GZIPOutputStream(new FileOutputStream(flowConfigurationFile))) {
- outputStream.write(FLOW_CONTENTS.getBytes(StandardCharsets.UTF_8));
+ outputStream.write(contents.getBytes(StandardCharsets.UTF_8));
}
return flowConfigurationPath;
}
+
+ private URL getResourceUrl(String resource) throws FileNotFoundException {
+ final URL resourceUrl = SetSensitivePropertiesKey.class.getResource(resource);
+ if (resourceUrl == null) {
+ throw new FileNotFoundException(String.format("Resource [%s] not found", resource));
+ }
+ return resourceUrl;
+ }
}
diff --git a/nifi-commons/nifi-flow-encryptor/src/test/resources/blank.nifi.properties b/nifi-commons/nifi-flow-encryptor/src/test/resources/blank.nifi.properties
index 8c16c51..b6e24e7 100644
--- a/nifi-commons/nifi-flow-encryptor/src/test/resources/blank.nifi.properties
+++ b/nifi-commons/nifi-flow-encryptor/src/test/resources/blank.nifi.properties
@@ -15,3 +15,4 @@
nifi.sensitive.props.key=
nifi.sensitive.props.algorithm=
nifi.flow.configuration.file=
+nifi.flow.configuration.json.file=
diff --git a/nifi-commons/nifi-flow-encryptor/src/test/resources/blank.nifi.properties b/nifi-commons/nifi-flow-encryptor/src/test/resources/legacy-blank.nifi.properties
similarity index 100%
copy from nifi-commons/nifi-flow-encryptor/src/test/resources/blank.nifi.properties
copy to nifi-commons/nifi-flow-encryptor/src/test/resources/legacy-blank.nifi.properties
diff --git a/nifi-commons/nifi-flow-encryptor/src/test/resources/populated.nifi.properties b/nifi-commons/nifi-flow-encryptor/src/test/resources/populated.nifi.properties
index 36e2707..520019e 100644
--- a/nifi-commons/nifi-flow-encryptor/src/test/resources/populated.nifi.properties
+++ b/nifi-commons/nifi-flow-encryptor/src/test/resources/populated.nifi.properties
@@ -15,3 +15,4 @@
nifi.sensitive.props.key=D5E41AC1-EEF8-4A54-930D-593F749AE95C
nifi.sensitive.props.algorithm=NIFI_ARGON2_AES_GCM_256
nifi.flow.configuration.file=
+nifi.flow.configuration.json.file=
diff --git a/nifi-docs/src/main/asciidoc/administration-guide.adoc b/nifi-docs/src/main/asciidoc/administration-guide.adoc
index d0db77b..793cc70 100644
--- a/nifi-docs/src/main/asciidoc/administration-guide.adoc
+++ b/nifi-docs/src/main/asciidoc/administration-guide.adoc
@@ -4334,14 +4334,21 @@ For more information see the <<toolkit-guide.adoc#encrypt_config_tool,Encrypt-Co
==== Updating the Sensitive Properties Key
-Starting with version 1.14.0, NiFi requires a value for 'nifi.sensitive.props.key' in _nifi.properties_.
+Starting with version 1.14.0, NiFi requires a value for `nifi.sensitive.props.key` in _nifi.properties_.
-The following command can be used to read an existing _flow.json.gz_ configuration and set a new sensitive properties key in _nifi.properties_:
+The following command can be used to read an existing flow configuration and set a new sensitive properties key in _nifi.properties_:
```
$ ./bin/nifi.sh set-sensitive-properties-key <sensitivePropertiesKey>
```
+The command reads the following flow configuration file properties from _nifi.properties_:
+
+- `nifi.flow.configuration.file`
+- `nifi.flow.configuration.json.file`
+
+The command checks for the existence of each file and updates the sensitive property values found.
+
The minimum required length for a new sensitive properties key is 12 characters.
=== Start New NiFi