You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by jo...@apache.org on 2022/04/19 16:25:12 UTC
[nifi] branch support/nifi-1.16 updated (509a445ee5 -> 927f31129a)
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a change to branch support/nifi-1.16
in repository https://gitbox.apache.org/repos/asf/nifi.git
from 509a445ee5 NIFI-9897 This closes #5949. Refactored GRPC SSL Configuration
new f50b615a12 NIFI-9924 Corrected text encoding in PutEmail filenames and TestFTP
new fc2149f610 NIFI-9862 Updated JsonTreeReader to read Records from Nested Field
new d6176fea41 NIFI-9899 Corrected MongoDBLookupService attribute handling
new 5588f32bb7 NIFI-9928 Removed nifi-security-utils from nifi-prometheus-reporting-task
new 14c1470755 NIFI-9935 Upgraded Zip4j from 2.9.1 to 2.10.0
new b16aa36681 NIFI-9934 Remove unused Groovy dependency management from nifi-graph-test-clients
new 906dc6ca9b NIFI-9933 Upgraded Apache Ant to 1.10.12
new 0e17c49803 NIFI-9883 Refactor Property Protection using Isolated ClassLoader (#5972)
new 927f31129a NIFI-9868 fix version references to 1.16.1-SNAP after all cherry-picks
The 9 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
nifi-assembly/pom.xml | 15 +
nifi-assembly/src/main/assembly/common.xml | 14 +
nifi-assembly/src/main/assembly/dependencies.xml | 2 +
.../SensitivePropertyProviderFactoryAware.java | 55 --
.../StandardSensitivePropertyProviderFactory.java | 13 +-
...ifi.properties.SensitivePropertyProviderFactory | 1 +
...nifi.properties.scheme.ProtectionSchemeResolver | 1 +
.../nifi-property-protection-loader}/pom.xml | 40 +-
.../properties/ApplicationPropertiesProtector.java | 13 +-
.../loader/PropertyProtectionURLClassLoader.java | 65 ++
.../loader/PropertyProviderFactoryLoader.java | 45 +
.../loader/ProtectionSchemeResolverLoader.java | 45 +
nifi-commons/pom.xml | 1 +
.../nifi-framework/nifi-authorizer/pom.xml | 13 +
.../nifi/authorization/AuthorizerFactoryBean.java | 240 +++--
.../authorization/AuthorizerFactoryBeanSpec.groovy | 72 --
.../authorization/AuthorizerFactoryBeanTest.groovy | 105 --
.../authorization/AuthorizerFactoryBeanTest.java | 107 ++
.../mock/MockAccessPolicyProvider.java | 66 ++
.../nifi/authorization/mock/MockAuthorizer.java | 48 +
.../authorization/mock/MockUserGroupProvider.java | 76 ++
.../src/test/resources/authorizers.xml | 31 +
.../nifi/integration/versioned/ImportFlowIT.java | 4 -
.../nifi-framework/nifi-properties-loader/pom.xml | 51 +-
.../nifi/properties/NiFiPropertiesLoader.java | 115 ++-
.../properties/NiFiPropertiesGroovyTest.groovy | 1029 --------------------
.../NiFiPropertiesLoaderGroovyTest.groovy | 491 ----------
.../nifi/properties/NiFiPropertiesLoaderTest.java | 138 +++
.../resources/bootstrap_tests/conf/bootstrap.conf | 74 --
...h_sensitive_properties_protected_aes.properties | 126 ---
.../bootstrap_tests/missing_key/bootstrap.conf | 74 --
.../missing_key_line/bootstrap.conf | 71 --
.../unreadable_bootstrap/bootstrap.conf | 74 --
.../src/test/resources/conf/nifi.blank.properties | 121 ---
.../conf/nifi.cluster.without.key.properties | 94 --
.../test/resources/conf/nifi.missing.properties | 119 ---
.../src/test/resources/conf/nifi.properties | 121 ---
.../resources/conf/nifi.without.key.properties | 94 --
.../resources/conf/nifi_no_permissions.properties | 14 -
.../nifi_with_additional_sensitive_keys.properties | 122 ---
...l_sensitive_properties_protected_aes.properties | 126 ---
..._recursive_additional_sensitive_keys.properties | 122 ---
...h_sensitive_properties_protected_aes.properties | 125 ---
...nsitive_properties_protected_aes_128.properties | 125 ---
...roperties_protected_aes_128_password.properties | 125 ---
...otected_aes_improper_delimiter_value.properties | 125 ---
...ies_protected_aes_multiple_malformed.properties | 125 ---
...rties_protected_aes_single_malformed.properties | 125 ---
...nsitive_properties_protected_unknown.properties | 125 ---
...ith_sensitive_properties_unprotected.properties | 122 ---
...ve_properties_unprotected_extra_line.properties | 123 ---
.../resources/conf/nifi_with_whitespace.properties | 24 -
.../conf/empty.nifi.properties} | 0
.../conf/flow.nifi.properties} | 3 +
.../conf/protected.nifi.properties} | 6 +
.../nifi-framework/nifi-runtime/pom.xml | 15 +
.../resources/NiFiProperties/conf/bootstrap.conf} | 2 +
.../nifi-web/nifi-web-security/pom.xml | 13 +
.../spring/LoginIdentityProviderFactoryBean.java | 97 +-
.../LoginIdentityProviderFactoryBeanTest.groovy | 107 --
.../LoginIdentityProviderFactoryBeanTest.java | 93 ++
.../spring/mock/MockLoginIdentityProvider.java | 49 +
.../test/resources/login-identity-providers.xml | 22 +
nifi-nar-bundles/nifi-framework-bundle/pom.xml | 5 +
.../nifi-graph-test-clients/pom.xml | 22 +-
.../nifi-groovyx-bundle/nifi-groovyx-nar/pom.xml | 17 +
nifi-nar-bundles/nifi-hive-bundle/pom.xml | 6 +
.../apache/nifi/mongodb/MongoDBLookupService.java | 13 +-
.../nifi-prometheus-reporting-task/pom.xml | 11 +-
.../reporting/prometheus/PrometheusServer.java | 17 +-
.../nifi-scripting-nar/pom.xml | 17 +
.../nifi-standard-processors/pom.xml | 2 +-
.../apache/nifi/processors/standard/PutEmail.java | 2 +-
.../apache/nifi/processors/standard/TestFTP.java | 23 +-
.../nifi-record-serialization-services/pom.xml | 3 +
.../nifi/json/AbstractJsonRowRecordReader.java | 98 +-
.../org/apache/nifi/json/JsonRecordSource.java | 26 +-
.../java/org/apache/nifi/json/JsonTreeReader.java | 59 +-
.../apache/nifi/json/JsonTreeRowRecordReader.java | 12 +-
.../apache/nifi/json/StartingFieldStrategy.java | 38 +
.../additionalDetails.html | 398 +++++---
.../json/TestInferJsonSchemaAccessStrategy.java | 107 +-
.../nifi/json/TestJsonPathRowRecordReader.java | 74 +-
.../apache/nifi/json/TestJsonSchemaInference.java | 4 +-
.../nifi/json/TestJsonTreeRowRecordReader.java | 490 ++++++----
.../org/apache/nifi/json/TestWriteJsonResult.java | 38 +-
.../test/resources/json/multiple-nested-field.json | 22 +
.../json/nested-array-then-start-object.json | 20 +
.../json/single-element-nested-array-middle.json | 16 +
.../nifi-registry-properties-loader/pom.xml | 4 +
.../apache/nifi/registry/web/api/SecureLdapIT.java | 813 ----------------
nifi-registry/pom.xml | 5 +
nifi-toolkit/nifi-toolkit-admin/pom.xml | 4 +
nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml | 9 +
.../nifi/properties/ConfigEncryptionTool.groovy | 12 +-
.../properties/ConfigEncryptionToolTest.groovy | 30 +-
.../encryptconfig/EncryptConfigMainTest.groovy | 4 +-
97 files changed, 2230 insertions(+), 5990 deletions(-)
delete mode 100644 nifi-commons/nifi-property-protection-factory/src/main/java/org/apache/nifi/properties/SensitivePropertyProviderFactoryAware.java
rename nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key_line/nifi.properties => nifi-commons/nifi-property-protection-factory/src/main/resources/META-INF/services/org.apache.nifi.properties.SensitivePropertyProviderFactory (91%)
rename nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key/nifi.properties => nifi-commons/nifi-property-protection-factory/src/main/resources/META-INF/services/org.apache.nifi.properties.scheme.ProtectionSchemeResolver (92%)
copy {nifi-registry/nifi-registry-core/nifi-registry-properties-loader => nifi-commons/nifi-property-protection-loader}/pom.xml (53%)
rename nifi-commons/{nifi-property-protection-factory => nifi-property-protection-loader}/src/main/java/org/apache/nifi/properties/ApplicationPropertiesProtector.java (96%)
create mode 100644 nifi-commons/nifi-property-protection-loader/src/main/java/org/apache/nifi/property/protection/loader/PropertyProtectionURLClassLoader.java
create mode 100644 nifi-commons/nifi-property-protection-loader/src/main/java/org/apache/nifi/property/protection/loader/PropertyProviderFactoryLoader.java
create mode 100644 nifi-commons/nifi-property-protection-loader/src/main/java/org/apache/nifi/property/protection/loader/ProtectionSchemeResolverLoader.java
delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/groovy/org/apache/nifi/authorization/AuthorizerFactoryBeanSpec.groovy
delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/groovy/org/apache/nifi/authorization/AuthorizerFactoryBeanTest.groovy
create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/java/org/apache/nifi/authorization/AuthorizerFactoryBeanTest.java
create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/java/org/apache/nifi/authorization/mock/MockAccessPolicyProvider.java
create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/java/org/apache/nifi/authorization/mock/MockAuthorizer.java
create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/java/org/apache/nifi/authorization/mock/MockUserGroupProvider.java
create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/resources/authorizers.xml
delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/NiFiPropertiesGroovyTest.groovy
delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/NiFiPropertiesLoaderGroovyTest.groovy
create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/java/org/apache/nifi/properties/NiFiPropertiesLoaderTest.java
delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/conf/bootstrap.conf
delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/conf/nifi_with_sensitive_properties_protected_aes.properties
delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key/bootstrap.conf
delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key_line/bootstrap.conf
delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/unreadable_bootstrap/bootstrap.conf
delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.blank.properties
delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.cluster.without.key.properties
delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.missing.properties
delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.properties
delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.without.key.properties
delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_no_permissions.properties
delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_additional_sensitive_keys.properties
delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_all_sensitive_properties_protected_aes.properties
delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_recursive_additional_sensitive_keys.properties
delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes.properties
delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_128.properties
delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_128_password.properties
delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_improper_delimiter_value.properties
delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_multiple_malformed.properties
delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_single_malformed.properties
delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_unknown.properties
delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_unprotected.properties
delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_unprotected_extra_line.properties
delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_whitespace.properties
copy nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/{bootstrap_tests/missing_bootstrap/nifi.properties => properties/conf/empty.nifi.properties} (100%)
copy nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/{bootstrap_tests/missing_bootstrap/nifi.properties => properties/conf/flow.nifi.properties} (88%)
rename nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/{bootstrap_tests/missing_bootstrap/nifi.properties => properties/conf/protected.nifi.properties} (76%)
rename nifi-nar-bundles/nifi-framework-bundle/nifi-framework/{nifi-properties-loader/src/test/resources/bootstrap_tests/unreadable_bootstrap/nifi.properties => nifi-runtime/src/test/resources/NiFiProperties/conf/bootstrap.conf} (89%)
delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/test/groovy/org/apache/nifi/web/security/spring/LoginIdentityProviderFactoryBeanTest.groovy
create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/test/java/org/apache/nifi/web/security/spring/LoginIdentityProviderFactoryBeanTest.java
create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/test/java/org/apache/nifi/web/security/spring/mock/MockLoginIdentityProvider.java
create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/test/resources/login-identity-providers.xml
create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/StartingFieldStrategy.java
create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/multiple-nested-field.json
create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/nested-array-then-start-object.json
create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/single-element-nested-array-middle.json
delete mode 100644 nifi-registry/nifi-registry-core/nifi-registry-web-api/src/test/java/org/apache/nifi/registry/web/api/SecureLdapIT.java
[nifi] 08/09: NIFI-9883 Refactor Property Protection using Isolated ClassLoader (#5972)
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a commit to branch support/nifi-1.16
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit 0e17c49803cb424d74b32a3196d5b4670ac06446
Author: exceptionfactory <ex...@apache.org>
AuthorDate: Tue Apr 19 10:08:04 2022 -0500
NIFI-9883 Refactor Property Protection using Isolated ClassLoader (#5972)
* NIFI-9883 Refactored property protection to isolated ClassLoader
- Added nifi-property-protection-loader for abstracting access to implementation classes using ServiceLoader
- Updated Authorizer and Login Identity Provider configuration using isolated ClassLoader
- Updated NiFi Properties Loader using isolated ClassLoader
- Updated nifi-assembly to place property protection dependencies in lib/properties directory
- Updated and refactored unit tests
- Corrected LoginIdentityProviderFactoryBean getObject() Type
---
nifi-assembly/pom.xml | 15 +
nifi-assembly/src/main/assembly/common.xml | 14 +
nifi-assembly/src/main/assembly/dependencies.xml | 2 +
.../SensitivePropertyProviderFactoryAware.java | 55 --
.../StandardSensitivePropertyProviderFactory.java | 13 +-
...ifi.properties.SensitivePropertyProviderFactory | 1 +
...nifi.properties.scheme.ProtectionSchemeResolver | 1 +
.../nifi-property-protection-loader}/pom.xml | 42 +-
.../properties/ApplicationPropertiesProtector.java | 13 +-
.../loader/PropertyProtectionURLClassLoader.java | 65 ++
.../loader/PropertyProviderFactoryLoader.java | 45 +
.../loader/ProtectionSchemeResolverLoader.java | 45 +
nifi-commons/pom.xml | 1 +
.../nifi-framework/nifi-authorizer/pom.xml | 13 +
.../nifi/authorization/AuthorizerFactoryBean.java | 240 +++--
.../authorization/AuthorizerFactoryBeanSpec.groovy | 72 --
.../authorization/AuthorizerFactoryBeanTest.groovy | 105 --
.../authorization/AuthorizerFactoryBeanTest.java | 107 ++
.../mock/MockAccessPolicyProvider.java | 66 ++
.../nifi/authorization/mock/MockAuthorizer.java | 48 +
.../authorization/mock/MockUserGroupProvider.java | 76 ++
.../src/test/resources/authorizers.xml | 31 +
.../nifi/integration/versioned/ImportFlowIT.java | 4 -
.../nifi-framework/nifi-properties-loader/pom.xml | 51 +-
.../nifi/properties/NiFiPropertiesLoader.java | 115 ++-
.../properties/NiFiPropertiesGroovyTest.groovy | 1029 --------------------
.../NiFiPropertiesLoaderGroovyTest.groovy | 491 ----------
.../nifi/properties/NiFiPropertiesLoaderTest.java | 138 +++
.../resources/bootstrap_tests/conf/bootstrap.conf | 74 --
...h_sensitive_properties_protected_aes.properties | 126 ---
.../bootstrap_tests/missing_key/bootstrap.conf | 74 --
.../missing_key_line/bootstrap.conf | 71 --
.../unreadable_bootstrap/bootstrap.conf | 74 --
.../src/test/resources/conf/nifi.blank.properties | 121 ---
.../conf/nifi.cluster.without.key.properties | 94 --
.../test/resources/conf/nifi.missing.properties | 119 ---
.../src/test/resources/conf/nifi.properties | 121 ---
.../resources/conf/nifi.without.key.properties | 94 --
.../resources/conf/nifi_no_permissions.properties | 14 -
.../nifi_with_additional_sensitive_keys.properties | 122 ---
...l_sensitive_properties_protected_aes.properties | 126 ---
..._recursive_additional_sensitive_keys.properties | 122 ---
...h_sensitive_properties_protected_aes.properties | 125 ---
...nsitive_properties_protected_aes_128.properties | 125 ---
...roperties_protected_aes_128_password.properties | 125 ---
...otected_aes_improper_delimiter_value.properties | 125 ---
...ies_protected_aes_multiple_malformed.properties | 125 ---
...rties_protected_aes_single_malformed.properties | 125 ---
...nsitive_properties_protected_unknown.properties | 125 ---
...ith_sensitive_properties_unprotected.properties | 122 ---
...ve_properties_unprotected_extra_line.properties | 123 ---
.../resources/conf/nifi_with_whitespace.properties | 24 -
.../conf/empty.nifi.properties} | 0
.../conf/flow.nifi.properties} | 3 +
.../conf/protected.nifi.properties} | 6 +
.../nifi-framework/nifi-runtime/pom.xml | 15 +
.../resources/NiFiProperties/conf/bootstrap.conf} | 2 +
.../nifi-web/nifi-web-security/pom.xml | 13 +
.../spring/LoginIdentityProviderFactoryBean.java | 97 +-
.../LoginIdentityProviderFactoryBeanTest.groovy | 107 --
.../LoginIdentityProviderFactoryBeanTest.java | 93 ++
.../spring/mock/MockLoginIdentityProvider.java | 49 +
.../test/resources/login-identity-providers.xml | 22 +
nifi-nar-bundles/nifi-framework-bundle/pom.xml | 5 +
.../nifi-registry-properties-loader/pom.xml | 4 +
.../apache/nifi/registry/web/api/SecureLdapIT.java | 813 ----------------
nifi-registry/pom.xml | 5 +
nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml | 5 +
.../nifi/properties/ConfigEncryptionTool.groovy | 12 +-
.../properties/ConfigEncryptionToolTest.groovy | 30 +-
.../encryptconfig/EncryptConfigMainTest.groovy | 4 +-
71 files changed, 1224 insertions(+), 5455 deletions(-)
diff --git a/nifi-assembly/pom.xml b/nifi-assembly/pom.xml
index 0408e9009b..c3f4d87362 100644
--- a/nifi-assembly/pom.xml
+++ b/nifi-assembly/pom.xml
@@ -167,6 +167,21 @@ language governing permissions and limitations under the License. -->
<artifactId>nifi-bootstrap-utils</artifactId>
<version>1.16.1-SNAPSHOT</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-property-utils</artifactId>
+ <version>1.17.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-property-protection-api</artifactId>
+ <version>1.17.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-property-protection-factory</artifactId>
+ <version>1.17.0-SNAPSHOT</version>
+ </dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-resources</artifactId>
diff --git a/nifi-assembly/src/main/assembly/common.xml b/nifi-assembly/src/main/assembly/common.xml
index 58f6923410..cec28e5b32 100644
--- a/nifi-assembly/src/main/assembly/common.xml
+++ b/nifi-assembly/src/main/assembly/common.xml
@@ -29,6 +29,7 @@
<include>slf4j-api</include>
<include>logback-classic</include>
<include>nifi-api</include>
+ <include>nifi-property-protection-api</include>
</includes>
</dependencySet>
@@ -62,6 +63,19 @@
</includes>
</dependencySet>
+ <!-- Write Property Protection implementation libraries to isolated directory-->
+ <dependencySet>
+ <scope>runtime</scope>
+ <useProjectArtifact>false</useProjectArtifact>
+ <outputDirectory>lib/properties</outputDirectory>
+ <directoryMode>0770</directoryMode>
+ <fileMode>0664</fileMode>
+ <useTransitiveFiltering>true</useTransitiveFiltering>
+ <includes>
+ <include>nifi-property-protection-factory</include>
+ </includes>
+ </dependencySet>
+
<!-- Write out the conf directory contents -->
<dependencySet>
<scope>runtime</scope>
diff --git a/nifi-assembly/src/main/assembly/dependencies.xml b/nifi-assembly/src/main/assembly/dependencies.xml
index ed6ef0a327..2e5a141974 100644
--- a/nifi-assembly/src/main/assembly/dependencies.xml
+++ b/nifi-assembly/src/main/assembly/dependencies.xml
@@ -34,6 +34,8 @@
<excludes>
<exclude>nifi-bootstrap-utils</exclude>
<exclude>nifi-bootstrap</exclude>
+ <exclude>nifi-property-protection-api</exclude>
+ <exclude>nifi-property-protection-factory</exclude>
<exclude>nifi-resources</exclude>
<exclude>nifi-docs</exclude>
diff --git a/nifi-commons/nifi-property-protection-factory/src/main/java/org/apache/nifi/properties/SensitivePropertyProviderFactoryAware.java b/nifi-commons/nifi-property-protection-factory/src/main/java/org/apache/nifi/properties/SensitivePropertyProviderFactoryAware.java
deleted file mode 100644
index 46bfbffe7d..0000000000
--- a/nifi-commons/nifi-property-protection-factory/src/main/java/org/apache/nifi/properties/SensitivePropertyProviderFactoryAware.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.properties;
-
-import org.apache.nifi.properties.scheme.StandardProtectionScheme;
-
-import java.util.function.Supplier;
-
-/**
- * Provides a default SensitivePropertyProviderFactory to subclasses.
- */
-public class SensitivePropertyProviderFactoryAware {
-
- private SensitivePropertyProviderFactory sensitivePropertyProviderFactory;
-
- protected SensitivePropertyProviderFactory getSensitivePropertyProviderFactory() throws SensitivePropertyProtectionException {
- if (sensitivePropertyProviderFactory == null) {
- sensitivePropertyProviderFactory = StandardSensitivePropertyProviderFactory.withDefaults();
- }
- return sensitivePropertyProviderFactory;
- }
-
- protected String decryptValue(final String cipherText, final String protectionScheme, final String propertyName, final String groupIdentifier) throws SensitivePropertyProtectionException {
- final SensitivePropertyProviderFactory sensitivePropertyProviderFactory = getSensitivePropertyProviderFactory();
- final ProtectedPropertyContext protectedPropertyContext = sensitivePropertyProviderFactory.getPropertyContext(groupIdentifier, propertyName);
- return sensitivePropertyProviderFactory.getProvider(new StandardProtectionScheme(protectionScheme))
- .unprotect(cipherText, protectedPropertyContext);
- }
-
- /**
- * Configures and sets the SensitivePropertyProviderFactory.
- * @param keyHex An key in hex format, which some providers may use for encryption
- * @param bootstrapPropertiesSupplier The bootstrap.conf properties supplier
- * @return The configured SensitivePropertyProviderFactory
- */
- public SensitivePropertyProviderFactory configureSensitivePropertyProviderFactory(final String keyHex,
- final Supplier<BootstrapProperties> bootstrapPropertiesSupplier) {
- sensitivePropertyProviderFactory = StandardSensitivePropertyProviderFactory.withKeyAndBootstrapSupplier(keyHex, bootstrapPropertiesSupplier);
- return sensitivePropertyProviderFactory;
- }
-}
diff --git a/nifi-commons/nifi-property-protection-factory/src/main/java/org/apache/nifi/properties/StandardSensitivePropertyProviderFactory.java b/nifi-commons/nifi-property-protection-factory/src/main/java/org/apache/nifi/properties/StandardSensitivePropertyProviderFactory.java
index 74ca100529..feedb8e5d9 100644
--- a/nifi-commons/nifi-property-protection-factory/src/main/java/org/apache/nifi/properties/StandardSensitivePropertyProviderFactory.java
+++ b/nifi-commons/nifi-property-protection-factory/src/main/java/org/apache/nifi/properties/StandardSensitivePropertyProviderFactory.java
@@ -66,11 +66,22 @@ public class StandardSensitivePropertyProviderFactory implements SensitiveProper
HashiCorpVaultTransitSensitivePropertyProvider.class
);
- private final Optional<String> keyHex;
+ private Optional<String> keyHex;
private final Supplier<BootstrapProperties> bootstrapPropertiesSupplier;
private final Map<Class<? extends SensitivePropertyProvider>, SensitivePropertyProvider> providers;
private Map<String, Pattern> customPropertyContextMap;
+ /**
+ * Factory default constructor to support java.util.ServiceLoader
+ */
+ public StandardSensitivePropertyProviderFactory() {
+ this(null, null);
+ }
+
+ public void setKeyHex(final String hexadecimalKey) {
+ this.keyHex = Optional.ofNullable(hexadecimalKey);
+ }
+
/**
* Creates a StandardSensitivePropertyProviderFactory using the default bootstrap.conf location and
* the keyHex extracted from this bootstrap.conf.
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key_line/nifi.properties b/nifi-commons/nifi-property-protection-factory/src/main/resources/META-INF/services/org.apache.nifi.properties.SensitivePropertyProviderFactory
similarity index 91%
rename from nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key_line/nifi.properties
rename to nifi-commons/nifi-property-protection-factory/src/main/resources/META-INF/services/org.apache.nifi.properties.SensitivePropertyProviderFactory
index ae1e83eeb3..f318d70c15 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key_line/nifi.properties
+++ b/nifi-commons/nifi-property-protection-factory/src/main/resources/META-INF/services/org.apache.nifi.properties.SensitivePropertyProviderFactory
@@ -12,3 +12,4 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+org.apache.nifi.properties.StandardSensitivePropertyProviderFactory
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key/nifi.properties b/nifi-commons/nifi-property-protection-factory/src/main/resources/META-INF/services/org.apache.nifi.properties.scheme.ProtectionSchemeResolver
similarity index 92%
rename from nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key/nifi.properties
rename to nifi-commons/nifi-property-protection-factory/src/main/resources/META-INF/services/org.apache.nifi.properties.scheme.ProtectionSchemeResolver
index ae1e83eeb3..28d1fcc31c 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key/nifi.properties
+++ b/nifi-commons/nifi-property-protection-factory/src/main/resources/META-INF/services/org.apache.nifi.properties.scheme.ProtectionSchemeResolver
@@ -12,3 +12,4 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+org.apache.nifi.properties.scheme.StandardProtectionSchemeResolver
diff --git a/nifi-registry/nifi-registry-core/nifi-registry-properties-loader/pom.xml b/nifi-commons/nifi-property-protection-loader/pom.xml
similarity index 51%
copy from nifi-registry/nifi-registry-core/nifi-registry-properties-loader/pom.xml
copy to nifi-commons/nifi-property-protection-loader/pom.xml
index 21127df36f..f98dbc5d71 100644
--- a/nifi-registry/nifi-registry-core/nifi-registry-properties-loader/pom.xml
+++ b/nifi-commons/nifi-property-protection-loader/pom.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8"?>
+<?xml version="1.0"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
@@ -16,44 +16,16 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>org.apache.nifi.registry</groupId>
- <artifactId>nifi-registry-core</artifactId>
- <version>1.16.1-SNAPSHOT</version>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-commons</artifactId>
+ <version>1.17.0-SNAPSHOT</version>
</parent>
- <artifactId>nifi-registry-properties-loader</artifactId>
- <packaging>jar</packaging>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.codehaus.gmavenplus</groupId>
- <artifactId>gmavenplus-plugin</artifactId>
- <version>1.5</version>
- <executions>
- <execution>
- <goals>
- <goal>addTestSources</goal>
- <goal>testCompile</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
+ <artifactId>nifi-property-protection-loader</artifactId>
<dependencies>
- <dependency>
- <groupId>org.apache.nifi.registry</groupId>
- <artifactId>nifi-registry-properties</artifactId>
- </dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
- <artifactId>nifi-property-protection-factory</artifactId>
+ <artifactId>nifi-property-protection-api</artifactId>
+ <version>1.17.0-SNAPSHOT</version>
</dependency>
</dependencies>
-
-
- <scm>
- <tag>nifi-1.16.0-RC3</tag>
- </scm>
</project>
diff --git a/nifi-commons/nifi-property-protection-factory/src/main/java/org/apache/nifi/properties/ApplicationPropertiesProtector.java b/nifi-commons/nifi-property-protection-loader/src/main/java/org/apache/nifi/properties/ApplicationPropertiesProtector.java
similarity index 96%
rename from nifi-commons/nifi-property-protection-factory/src/main/java/org/apache/nifi/properties/ApplicationPropertiesProtector.java
rename to nifi-commons/nifi-property-protection-loader/src/main/java/org/apache/nifi/properties/ApplicationPropertiesProtector.java
index 4500ff1c21..d485c2e978 100644
--- a/nifi-commons/nifi-property-protection-factory/src/main/java/org/apache/nifi/properties/ApplicationPropertiesProtector.java
+++ b/nifi-commons/nifi-property-protection-loader/src/main/java/org/apache/nifi/properties/ApplicationPropertiesProtector.java
@@ -16,7 +16,6 @@
*/
package org.apache.nifi.properties;
-import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -158,7 +157,7 @@ public class ApplicationPropertiesProtector<T extends ProtectedProperties<U>, U
@Override
public List<String> getPopulatedSensitivePropertyKeys() {
List<String> allSensitiveKeys = getSensitivePropertyKeys();
- return allSensitiveKeys.stream().filter(k -> StringUtils.isNotBlank(getProperty(k))).collect(Collectors.toList());
+ return allSensitiveKeys.stream().filter(k -> isNotBlank(getProperty(k))).collect(Collectors.toList());
}
@Override
@@ -179,7 +178,7 @@ public class ApplicationPropertiesProtector<T extends ProtectedProperties<U>, U
final Map<String, String> traditionalProtectedProperties = new HashMap<>();
for (final String key : sensitiveKeys) {
final String protection = getProperty(getProtectionKey(key));
- if (StringUtils.isNotBlank(protection) && StringUtils.isNotBlank(getProperty(key))) {
+ if (isNotBlank(protection) && isNotBlank(getProperty(key))) {
traditionalProtectedProperties.put(key, protection);
}
}
@@ -203,7 +202,7 @@ public class ApplicationPropertiesProtector<T extends ProtectedProperties<U>, U
*/
@Override
public boolean isPropertyProtected(final String key) {
- return key != null && isPropertySensitive(key) && !StringUtils.isBlank(getProperty(getProtectionKey(key)));
+ return key != null && isPropertySensitive(key) && isNotBlank(getProperty(getProtectionKey(key)));
}
@Override
@@ -238,7 +237,7 @@ public class ApplicationPropertiesProtector<T extends ProtectedProperties<U>, U
}
if (!failedKeys.isEmpty()) {
- final String failed = failedKeys.size() == 1 ? failedKeys.iterator().next() : StringUtils.join(failedKeys, ", ");
+ final String failed = failedKeys.size() == 1 ? failedKeys.iterator().next() : String.join(", ", failedKeys);
throw new SensitivePropertyProtectionException(String.format("Failed unprotected properties: %s", failed));
}
@@ -297,4 +296,8 @@ public class ApplicationPropertiesProtector<T extends ProtectedProperties<U>, U
.map(Map.Entry::getValue)
.orElseThrow(() -> new UnsupportedOperationException(String.format("Protection Scheme Path [%s] Provider not found", protectionSchemePath)));
}
+
+ private boolean isNotBlank(final String string) {
+ return string != null && string.length() > 0;
+ }
}
diff --git a/nifi-commons/nifi-property-protection-loader/src/main/java/org/apache/nifi/property/protection/loader/PropertyProtectionURLClassLoader.java b/nifi-commons/nifi-property-protection-loader/src/main/java/org/apache/nifi/property/protection/loader/PropertyProtectionURLClassLoader.java
new file mode 100644
index 0000000000..93a8381212
--- /dev/null
+++ b/nifi-commons/nifi-property-protection-loader/src/main/java/org/apache/nifi/property/protection/loader/PropertyProtectionURLClassLoader.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.property.protection.loader;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.stream.Stream;
+
+/**
+ * Property Protection URL Class Loader uses the Current Thread Class Loader as the parent and loads libraries from a standard directory
+ */
+public class PropertyProtectionURLClassLoader extends URLClassLoader {
+ private static final String STANDARD_DIRECTORY = "lib/properties";
+
+ private static final Logger logger = LoggerFactory.getLogger(PropertyProtectionURLClassLoader.class);
+
+ public PropertyProtectionURLClassLoader(final ClassLoader parentClassLoader) {
+ super(getPropertyProtectionUrls(), parentClassLoader);
+ }
+
+ private static URL[] getPropertyProtectionUrls() {
+ final Path standardDirectory = Paths.get(STANDARD_DIRECTORY);
+ if (Files.exists(standardDirectory)) {
+ try (final Stream<Path> files = Files.list(standardDirectory)) {
+ return files.map(Path::toUri)
+ .map(uri -> {
+ try {
+ return uri.toURL();
+ } catch (final MalformedURLException e) {
+ throw new UncheckedIOException(String.format("Processing Property Protection libraries failed [%s]", standardDirectory), e);
+ }
+ })
+ .toArray(URL[]::new);
+ } catch (final IOException e) {
+ throw new UncheckedIOException(String.format("Loading Property Protection libraries failed [%s]", standardDirectory), e);
+ }
+ } else {
+ logger.warn("Property Protection libraries directory [{}] not found", standardDirectory);
+ return new URL[0];
+ }
+ }
+}
diff --git a/nifi-commons/nifi-property-protection-loader/src/main/java/org/apache/nifi/property/protection/loader/PropertyProviderFactoryLoader.java b/nifi-commons/nifi-property-protection-loader/src/main/java/org/apache/nifi/property/protection/loader/PropertyProviderFactoryLoader.java
new file mode 100644
index 0000000000..2505d9a98b
--- /dev/null
+++ b/nifi-commons/nifi-property-protection-loader/src/main/java/org/apache/nifi/property/protection/loader/PropertyProviderFactoryLoader.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.property.protection.loader;
+
+import org.apache.nifi.properties.SensitivePropertyProtectionException;
+import org.apache.nifi.properties.SensitivePropertyProviderFactory;
+
+import java.util.Iterator;
+import java.util.ServiceLoader;
+
+/**
+ * Loader for Sensitive Property Provider Factory
+ */
+public class PropertyProviderFactoryLoader {
+ /**
+ * Get Sensitive Property Provider Factory using ServiceLoader to find available implementations from META-INF directories
+ *
+ * @return Sensitive Property Provider Factory
+ * @throws SensitivePropertyProtectionException Thrown when no implementations found
+ */
+ public SensitivePropertyProviderFactory getPropertyProviderFactory() {
+ final ServiceLoader<SensitivePropertyProviderFactory> serviceLoader = ServiceLoader.load(SensitivePropertyProviderFactory.class);
+ final Iterator<SensitivePropertyProviderFactory> factories = serviceLoader.iterator();
+
+ if (factories.hasNext()) {
+ return factories.next();
+ } else {
+ throw new SensitivePropertyProtectionException(String.format("No implementations found [%s]", SensitivePropertyProviderFactory.class.getName()));
+ }
+ }
+}
diff --git a/nifi-commons/nifi-property-protection-loader/src/main/java/org/apache/nifi/property/protection/loader/ProtectionSchemeResolverLoader.java b/nifi-commons/nifi-property-protection-loader/src/main/java/org/apache/nifi/property/protection/loader/ProtectionSchemeResolverLoader.java
new file mode 100644
index 0000000000..5608299e25
--- /dev/null
+++ b/nifi-commons/nifi-property-protection-loader/src/main/java/org/apache/nifi/property/protection/loader/ProtectionSchemeResolverLoader.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.property.protection.loader;
+
+import org.apache.nifi.properties.SensitivePropertyProtectionException;
+import org.apache.nifi.properties.scheme.ProtectionSchemeResolver;
+
+import java.util.Iterator;
+import java.util.ServiceLoader;
+
+/**
+ * Loader for Protection Scheme Resolver
+ */
+public class ProtectionSchemeResolverLoader {
+ /**
+ * Get Protection Scheme Resolver using ServiceLoader to find available implementations from META-INF directories
+ *
+ * @return Protection Scheme Resolver
+ * @throws SensitivePropertyProtectionException Thrown when no implementations found
+ */
+ public ProtectionSchemeResolver getProtectionSchemeResolver() {
+ final ServiceLoader<ProtectionSchemeResolver> serviceLoader = ServiceLoader.load(ProtectionSchemeResolver.class);
+ final Iterator<ProtectionSchemeResolver> factories = serviceLoader.iterator();
+
+ if (factories.hasNext()) {
+ return factories.next();
+ } else {
+ throw new SensitivePropertyProtectionException(String.format("No implementations found [%s]", ProtectionSchemeResolver.class.getName()));
+ }
+ }
+}
diff --git a/nifi-commons/pom.xml b/nifi-commons/pom.xml
index 7c6ce72ece..1864f7a183 100644
--- a/nifi-commons/pom.xml
+++ b/nifi-commons/pom.xml
@@ -44,6 +44,7 @@
<module>nifi-property-protection-factory</module>
<module>nifi-property-protection-gcp</module>
<module>nifi-property-protection-hashicorp</module>
+ <module>nifi-property-protection-loader</module>
<module>nifi-property-protection-shared</module>
<module>nifi-record</module>
<module>nifi-record-path</module>
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/pom.xml
index 5ecfc17469..8e7230e7a2 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/pom.xml
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/pom.xml
@@ -148,6 +148,14 @@
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-properties-loader</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-property-protection-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-property-protection-loader</artifactId>
+ </dependency>
<!-- Test dependencies -->
<dependency>
@@ -156,6 +164,11 @@
<version>1.16.1-SNAPSHOT</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-property-protection-factory</artifactId>
+ <scope>test</scope>
+ </dependency>
<!-- Spock testing dependencies-->
<dependency>
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizerFactoryBean.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizerFactoryBean.java
index 804c7f2f40..eec86bf080 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizerFactoryBean.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizerFactoryBean.java
@@ -25,7 +25,14 @@ import org.apache.nifi.authorization.generated.Authorizers;
import org.apache.nifi.authorization.generated.Property;
import org.apache.nifi.bundle.Bundle;
import org.apache.nifi.nar.ExtensionManager;
-import org.apache.nifi.properties.SensitivePropertyProviderFactoryAware;
+import org.apache.nifi.properties.ProtectedPropertyContext;
+import org.apache.nifi.properties.SensitivePropertyProvider;
+import org.apache.nifi.properties.SensitivePropertyProviderFactory;
+import org.apache.nifi.properties.scheme.ProtectionScheme;
+import org.apache.nifi.properties.scheme.ProtectionSchemeResolver;
+import org.apache.nifi.property.protection.loader.PropertyProtectionURLClassLoader;
+import org.apache.nifi.property.protection.loader.PropertyProviderFactoryLoader;
+import org.apache.nifi.property.protection.loader.ProtectionSchemeResolverLoader;
import org.apache.nifi.util.NiFiProperties;
import org.apache.nifi.util.file.classloader.ClassLoaderUtils;
import org.apache.nifi.xml.processing.ProcessingException;
@@ -62,8 +69,7 @@ import java.util.stream.Collectors;
/**
* Factory bean for loading the configured authorizer.
*/
-public class AuthorizerFactoryBean extends SensitivePropertyProviderFactoryAware
- implements FactoryBean, DisposableBean, UserGroupProviderLookup, AccessPolicyProviderLookup, AuthorizerLookup {
+public class AuthorizerFactoryBean implements FactoryBean<Authorizer>, DisposableBean, UserGroupProviderLookup, AccessPolicyProviderLookup, AuthorizerLookup {
private static final Logger logger = LoggerFactory.getLogger(AuthorizerFactoryBean.class);
private static final String AUTHORIZERS_XSD = "/authorizers.xsd";
@@ -109,7 +115,7 @@ public class AuthorizerFactoryBean extends SensitivePropertyProviderFactoryAware
}
@Override
- public Object getObject() throws Exception {
+ public Authorizer getObject() throws Exception {
if (authorizer == null) {
if (properties.getSslPort() == null) {
// use a default authorizer... only allowable when running not securely
@@ -132,12 +138,6 @@ public class AuthorizerFactoryBean extends SensitivePropertyProviderFactoryAware
userGroupProviders.put(userGroupProvider.getIdentifier(), createUserGroupProvider(userGroupProvider.getIdentifier(), userGroupProvider.getClazz()));
}
- // configure each user group provider
- for (final org.apache.nifi.authorization.generated.UserGroupProvider provider : authorizerConfiguration.getUserGroupProvider()) {
- final UserGroupProvider instance = userGroupProviders.get(provider.getIdentifier());
- instance.onConfigured(loadAuthorizerConfiguration(provider.getIdentifier(), provider.getProperty()));
- }
-
// create each access policy provider
for (final org.apache.nifi.authorization.generated.AccessPolicyProvider accessPolicyProvider : authorizerConfiguration.getAccessPolicyProvider()) {
if (accessPolicyProviders.containsKey(accessPolicyProvider.getIdentifier())) {
@@ -146,12 +146,6 @@ public class AuthorizerFactoryBean extends SensitivePropertyProviderFactoryAware
accessPolicyProviders.put(accessPolicyProvider.getIdentifier(), createAccessPolicyProvider(accessPolicyProvider.getIdentifier(), accessPolicyProvider.getClazz()));
}
- // configure each access policy provider
- for (final org.apache.nifi.authorization.generated.AccessPolicyProvider provider : authorizerConfiguration.getAccessPolicyProvider()) {
- final AccessPolicyProvider instance = accessPolicyProviders.get(provider.getIdentifier());
- instance.onConfigured(loadAuthorizerConfiguration(provider.getIdentifier(), provider.getProperty()));
- }
-
// create each authorizer
for (final org.apache.nifi.authorization.generated.Authorizer authorizer : authorizerConfiguration.getAuthorizer()) {
if (authorizers.containsKey(authorizer.getIdentifier())) {
@@ -160,15 +154,6 @@ public class AuthorizerFactoryBean extends SensitivePropertyProviderFactoryAware
authorizers.put(authorizer.getIdentifier(), createAuthorizer(authorizer.getIdentifier(), authorizer.getClazz(), authorizer.getClasspath()));
}
- // configure each authorizer, except the authorizer that is selected in nifi.properties
- for (final org.apache.nifi.authorization.generated.Authorizer provider : authorizerConfiguration.getAuthorizer()) {
- if (provider.getIdentifier().equals(authorizerIdentifier)) {
- continue;
- }
- final Authorizer instance = authorizers.get(provider.getIdentifier());
- instance.onConfigured(loadAuthorizerConfiguration(provider.getIdentifier(), provider.getProperty()));
- }
-
// get the authorizer instance
authorizer = getAuthorizer(authorizerIdentifier);
@@ -180,19 +165,7 @@ public class AuthorizerFactoryBean extends SensitivePropertyProviderFactoryAware
authorizer = AuthorizerFactory.installIntegrityChecks(authorizer);
// configure authorizer after integrity checks are installed
- AuthorizerConfigurationContext authorizerConfigurationContext = null;
- for (final org.apache.nifi.authorization.generated.Authorizer provider : authorizerConfiguration.getAuthorizer()) {
- if (provider.getIdentifier().equals(authorizerIdentifier)) {
- authorizerConfigurationContext = loadAuthorizerConfiguration(provider.getIdentifier(), provider.getProperty());
- break;
- }
- }
-
- if (authorizerConfigurationContext == null) {
- throw new IllegalStateException("Unable to load configuration for authorizer with id: " + authorizerIdentifier);
- }
-
- authorizer.onConfigured(authorizerConfigurationContext);
+ loadProviderProperties(authorizerConfiguration, authorizerIdentifier);
}
}
}
@@ -201,6 +174,75 @@ public class AuthorizerFactoryBean extends SensitivePropertyProviderFactoryAware
return authorizer;
}
+ private void loadProviderProperties(final Authorizers authorizerConfiguration, final String authorizerIdentifier) {
+ final ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
+
+ try {
+ final PropertyProtectionURLClassLoader protectionClassLoader = new PropertyProtectionURLClassLoader(contextClassLoader);
+ Thread.currentThread().setContextClassLoader(protectionClassLoader);
+
+ final ProtectionSchemeResolverLoader resolverLoader = new ProtectionSchemeResolverLoader();
+ final ProtectionSchemeResolver protectionSchemeResolver = resolverLoader.getProtectionSchemeResolver();
+
+ final PropertyProviderFactoryLoader factoryLoader = new PropertyProviderFactoryLoader();
+ final SensitivePropertyProviderFactory sensitivePropertyProviderFactory = factoryLoader.getPropertyProviderFactory();
+
+ // configure each user group provider
+ for (final org.apache.nifi.authorization.generated.UserGroupProvider provider : authorizerConfiguration.getUserGroupProvider()) {
+ final UserGroupProvider instance = userGroupProviders.get(provider.getIdentifier());
+ final AuthorizerConfigurationContext configurationContext = getConfigurationContext(
+ provider.getIdentifier(),
+ provider.getProperty(),
+ sensitivePropertyProviderFactory,
+ protectionSchemeResolver
+ );
+ instance.onConfigured(configurationContext);
+ }
+
+ // configure each access policy provider
+ for (final org.apache.nifi.authorization.generated.AccessPolicyProvider provider : authorizerConfiguration.getAccessPolicyProvider()) {
+ final AccessPolicyProvider instance = accessPolicyProviders.get(provider.getIdentifier());
+ final AuthorizerConfigurationContext configurationContext = getConfigurationContext(
+ provider.getIdentifier(),
+ provider.getProperty(),
+ sensitivePropertyProviderFactory,
+ protectionSchemeResolver
+ );
+ instance.onConfigured(configurationContext);
+ }
+
+ // configure each authorizer, except the authorizer that is selected in nifi.properties
+ AuthorizerConfigurationContext authorizerConfigurationContext = null;
+ for (final org.apache.nifi.authorization.generated.Authorizer provider : authorizerConfiguration.getAuthorizer()) {
+ if (provider.getIdentifier().equals(authorizerIdentifier)) {
+ authorizerConfigurationContext = getConfigurationContext(
+ provider.getIdentifier(),
+ provider.getProperty(),
+ sensitivePropertyProviderFactory,
+ protectionSchemeResolver
+ );
+ continue;
+ }
+ final Authorizer instance = authorizers.get(provider.getIdentifier());
+ final AuthorizerConfigurationContext configurationContext = getConfigurationContext(
+ provider.getIdentifier(),
+ provider.getProperty(),
+ sensitivePropertyProviderFactory,
+ protectionSchemeResolver
+ );
+ instance.onConfigured(configurationContext);
+ }
+
+ if (authorizerConfigurationContext == null) {
+ throw new IllegalStateException("Unable to load configuration for authorizer with id: " + authorizerIdentifier);
+ }
+
+ authorizer.onConfigured(authorizerConfigurationContext);
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
private Authorizers loadAuthorizersConfiguration() throws Exception {
final File authorizersConfigurationFile = properties.getAuthorizerConfigurationFile();
@@ -252,15 +294,10 @@ public class AuthorizerFactoryBean extends SensitivePropertyProviderFactoryAware
// attempt to load the class
Class<?> rawUserGroupProviderClass = Class.forName(userGroupProviderClassName, true, userGroupProviderClassLoader);
Class<? extends UserGroupProvider> userGroupProviderClass = rawUserGroupProviderClass.asSubclass(UserGroupProvider.class);
+ Constructor<? extends UserGroupProvider> constructor = userGroupProviderClass.getConstructor();
+ instance = constructor.newInstance();
- // otherwise create a new instance
- Constructor constructor = userGroupProviderClass.getConstructor();
- instance = (UserGroupProvider) constructor.newInstance();
-
- // method injection
performMethodInjection(instance, userGroupProviderClass);
-
- // field injection
performFieldInjection(instance, userGroupProviderClass);
// call post construction lifecycle event
@@ -300,15 +337,10 @@ public class AuthorizerFactoryBean extends SensitivePropertyProviderFactoryAware
// attempt to load the class
Class<?> rawAccessPolicyProviderClass = Class.forName(accessPolicyProviderClassName, true, accessPolicyProviderClassLoader);
Class<? extends AccessPolicyProvider> accessPolicyClass = rawAccessPolicyProviderClass.asSubclass(AccessPolicyProvider.class);
+ Constructor<? extends AccessPolicyProvider> constructor = accessPolicyClass.getConstructor();
+ instance = constructor.newInstance();
- // otherwise create a new instance
- Constructor constructor = accessPolicyClass.getConstructor();
- instance = (AccessPolicyProvider) constructor.newInstance();
-
- // method injection
performMethodInjection(instance, accessPolicyClass);
-
- // field injection
performFieldInjection(instance, accessPolicyClass);
// call post construction lifecycle event
@@ -356,15 +388,10 @@ public class AuthorizerFactoryBean extends SensitivePropertyProviderFactoryAware
// attempt to load the class
Class<?> rawAuthorizerClass = Class.forName(authorizerClassName, true, authorizerClassLoader);
Class<? extends Authorizer> authorizerClass = rawAuthorizerClass.asSubclass(Authorizer.class);
+ Constructor<? extends Authorizer> constructor = authorizerClass.getConstructor();
+ instance = constructor.newInstance();
- // otherwise create a new instance
- Constructor constructor = authorizerClass.getConstructor();
- instance = (Authorizer) constructor.newInstance();
-
- // method injection
performMethodInjection(instance, authorizerClass);
-
- // field injection
performFieldInjection(instance, authorizerClass);
// call post construction lifecycle event
@@ -378,22 +405,43 @@ public class AuthorizerFactoryBean extends SensitivePropertyProviderFactoryAware
return AuthorizerFactory.withNarLoader(instance, authorizerClassLoader);
}
- private AuthorizerConfigurationContext loadAuthorizerConfiguration(final String identifier, final List<Property> properties) {
+ private AuthorizerConfigurationContext getConfigurationContext(
+ final String identifier,
+ final List<Property> properties,
+ final SensitivePropertyProviderFactory sensitivePropertyProviderFactory,
+ final ProtectionSchemeResolver protectionSchemeResolver
+ ) {
final Map<String, String> authorizerProperties = new HashMap<>();
for (final Property property : properties) {
- if (!StringUtils.isBlank(property.getEncryption())) {
- String decryptedValue = decryptValue(property.getValue(), property.getEncryption(), property.getName(), identifier);
- authorizerProperties.put(property.getName(), decryptedValue);
- } else {
+ final String encryption = property.getEncryption();
+
+ if (StringUtils.isBlank(encryption)) {
authorizerProperties.put(property.getName(), property.getValue());
+ } else {
+ final String propertyDecrypted = getPropertyDecrypted(identifier, property, sensitivePropertyProviderFactory, protectionSchemeResolver);
+ authorizerProperties.put(property.getName(), propertyDecrypted);
}
}
return new StandardAuthorizerConfigurationContext(identifier, authorizerProperties);
}
- private void performMethodInjection(final Object instance, final Class authorizerClass) throws IllegalAccessException, IllegalArgumentException, InvocationTargetException {
+ private String getPropertyDecrypted(
+ final String providerIdentifier,
+ final Property property,
+ final SensitivePropertyProviderFactory sensitivePropertyProviderFactory,
+ final ProtectionSchemeResolver protectionSchemeResolver
+ ) {
+ final String scheme = property.getEncryption();
+ final ProtectionScheme protectionScheme = protectionSchemeResolver.getProtectionScheme(scheme);
+ final SensitivePropertyProvider propertyProvider = sensitivePropertyProviderFactory.getProvider(protectionScheme);
+ final ProtectedPropertyContext protectedPropertyContext = sensitivePropertyProviderFactory.getPropertyContext(providerIdentifier, property.getName());
+ final String protectedProperty = property.getValue();
+ return propertyProvider.unprotect(protectedProperty, protectedPropertyContext);
+ }
+
+ private void performMethodInjection(final Object instance, final Class<?> authorizerClass) throws IllegalAccessException, IllegalArgumentException, InvocationTargetException {
for (final Method method : authorizerClass.getMethods()) {
if (method.isAnnotationPresent(AuthorizerContext.class)) {
// make the method accessible
@@ -419,13 +467,13 @@ public class AuthorizerFactoryBean extends SensitivePropertyProviderFactoryAware
}
}
- final Class parentClass = authorizerClass.getSuperclass();
+ final Class<?> parentClass = authorizerClass.getSuperclass();
if (parentClass != null && Authorizer.class.isAssignableFrom(parentClass)) {
performMethodInjection(instance, parentClass);
}
}
- private void performFieldInjection(final Object instance, final Class authorizerClass) throws IllegalArgumentException, IllegalAccessException {
+ private void performFieldInjection(final Object instance, final Class<?> authorizerClass) throws IllegalArgumentException, IllegalAccessException {
for (final Field field : authorizerClass.getDeclaredFields()) {
if (field.isAnnotationPresent(AuthorizerContext.class)) {
// make the method accessible
@@ -451,7 +499,7 @@ public class AuthorizerFactoryBean extends SensitivePropertyProviderFactoryAware
}
}
- final Class parentClass = authorizerClass.getSuperclass();
+ final Class<?> parentClass = authorizerClass.getSuperclass();
if (parentClass != null && Authorizer.class.isAssignableFrom(parentClass)) {
performFieldInjection(instance, parentClass);
}
@@ -482,7 +530,7 @@ public class AuthorizerFactoryBean extends SensitivePropertyProviderFactoryAware
}
@Override
- public Class getObjectType() {
+ public Class<Authorizer> getObjectType() {
return Authorizer.class;
}
@@ -495,38 +543,32 @@ public class AuthorizerFactoryBean extends SensitivePropertyProviderFactoryAware
public void destroy() throws Exception {
List<Exception> errors = new ArrayList<>();
- if (authorizers != null) {
- authorizers.forEach((identifier, object) -> {
- try {
- object.preDestruction();
- } catch (Exception e) {
- errors.add(e);
- logger.error("Error pre-destructing {}: {}", identifier, e);
- }
- });
- }
+ authorizers.forEach((identifier, object) -> {
+ try {
+ object.preDestruction();
+ } catch (Exception e) {
+ errors.add(e);
+ logger.error("Authorizer [{}] destruction failed", identifier, e);
+ }
+ });
- if (accessPolicyProviders != null) {
- accessPolicyProviders.forEach((identifier, object) -> {
- try {
- object.preDestruction();
- } catch (Exception e) {
- errors.add(e);
- logger.error("Error pre-destructing {}: {}", identifier, e);
- }
- });
- }
+ accessPolicyProviders.forEach((identifier, object) -> {
+ try {
+ object.preDestruction();
+ } catch (Exception e) {
+ errors.add(e);
+ logger.error("Access Policy Provider [{}] destruction failed", identifier, e);
+ }
+ });
- if (userGroupProviders != null) {
- userGroupProviders.forEach((identifier, object) -> {
- try {
- object.preDestruction();
- } catch (Exception e) {
- errors.add(e);
- logger.error("Error pre-destructing {}: {}", identifier, e);
- }
- });
- }
+ userGroupProviders.forEach((identifier, object) -> {
+ try {
+ object.preDestruction();
+ } catch (Exception e) {
+ errors.add(e);
+ logger.error("User Group Provider [{}] destruction failed", identifier, e);
+ }
+ });
if (!errors.isEmpty()) {
List<String> errorMessages = errors.stream().map(Throwable::toString).collect(Collectors.toList());
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/groovy/org/apache/nifi/authorization/AuthorizerFactoryBeanSpec.groovy b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/groovy/org/apache/nifi/authorization/AuthorizerFactoryBeanSpec.groovy
deleted file mode 100644
index c16e018f19..0000000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/groovy/org/apache/nifi/authorization/AuthorizerFactoryBeanSpec.groovy
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.authorization
-
-import org.apache.nifi.authorization.resource.ResourceFactory
-import org.apache.nifi.util.NiFiProperties
-import spock.lang.Specification
-
-class AuthorizerFactoryBeanSpec extends Specification {
-
- def mockProperties = Mock(NiFiProperties)
-
- AuthorizerFactoryBean authorizerFactoryBean
-
- // runs before every feature method
- def setup() {
- authorizerFactoryBean = new AuthorizerFactoryBean()
- authorizerFactoryBean.setProperties(mockProperties)
- }
-
- // runs after every feature method
- def cleanup() {}
-
- // runs before the first feature method
- def setupSpec() {}
-
- // runs after the last feature method
- def cleanupSpec() {}
-
- def "create default authorizer"() {
-
- setup: "properties indicate nifi-registry is unsecured"
- mockProperties.getProperty(NiFiProperties.WEB_HTTPS_PORT) >> ""
-
- when: "getAuthorizer() is first called"
- def authorizer = (Authorizer) authorizerFactoryBean.getObject()
-
- then: "the default authorizer is returned"
- authorizer != null
-
- and: "any authorization request made to that authorizer is approved"
- def authorizationResult = authorizer.authorize(getTestAuthorizationRequest())
- authorizationResult.result == AuthorizationResult.Result.Approved
-
- }
-
- // Helper methods
-
- private static AuthorizationRequest getTestAuthorizationRequest() {
- return new AuthorizationRequest.Builder()
- .resource(ResourceFactory.getFlowResource())
- .action(RequestAction.WRITE)
- .accessAttempt(false)
- .anonymous(true)
- .build()
- }
-
-}
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/groovy/org/apache/nifi/authorization/AuthorizerFactoryBeanTest.groovy b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/groovy/org/apache/nifi/authorization/AuthorizerFactoryBeanTest.groovy
deleted file mode 100644
index 85f85ef1e6..0000000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/groovy/org/apache/nifi/authorization/AuthorizerFactoryBeanTest.groovy
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.authorization
-
-import org.apache.nifi.authorization.generated.Property
-import org.bouncycastle.jce.provider.BouncyCastleProvider
-import org.junit.Before
-import org.junit.BeforeClass
-import org.junit.Test
-import org.junit.runner.RunWith
-import org.junit.runners.JUnit4
-import org.slf4j.Logger
-import org.slf4j.LoggerFactory
-
-import javax.crypto.Cipher
-import java.security.Security
-
-@RunWith(JUnit4.class)
-class AuthorizerFactoryBeanTest extends GroovyTestCase {
- private static final Logger logger = LoggerFactory.getLogger(AuthorizerFactoryBeanTest.class)
-
- // These blocks configure the constant values depending on JCE policies of the machine running the tests
- private static final String KEY_HEX_128 = "0123456789ABCDEFFEDCBA9876543210"
- private static final String KEY_HEX_256 = KEY_HEX_128 * 2
- public static final String KEY_HEX = isUnlimitedStrengthCryptoAvailable() ? KEY_HEX_256 : KEY_HEX_128
-
- private static final String CIPHER_TEXT_128 = "6pqdM1urBEPHtj+L||ds0Z7RpqOA2321c/+7iPMfxDrqmH5Qx6UwQG0eIYB//3Ng"
- private static final String CIPHER_TEXT_256 = "TepMCD7v3LAMF0KX||ydSRWPRl1/JXgTsZtfzCnDXu7a0lTLysjPL2I06EPUCHzw"
- public static final String CIPHER_TEXT = isUnlimitedStrengthCryptoAvailable() ? CIPHER_TEXT_256 : CIPHER_TEXT_128
-
- private static final String ENCRYPTION_SCHEME_128 = "aes/gcm/128"
- private static final String ENCRYPTION_SCHEME_256 = "aes/gcm/256"
- public static
- final String ENCRYPTION_SCHEME = isUnlimitedStrengthCryptoAvailable() ? ENCRYPTION_SCHEME_256 : ENCRYPTION_SCHEME_128
-
- private static final String PASSWORD = "thisIsABadPassword"
-
- private AuthorizerFactoryBean bean
-
- @BeforeClass
- static void setUpOnce() throws Exception {
- Security.addProvider(new BouncyCastleProvider())
-
- logger.metaClass.methodMissing = { String name, args ->
- logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}")
- }
- }
-
- @Before
- void setUp() throws Exception {
- bean = new AuthorizerFactoryBean()
- bean.configureSensitivePropertyProviderFactory(KEY_HEX, null)
- }
-
- private static boolean isUnlimitedStrengthCryptoAvailable() {
- Cipher.getMaxAllowedKeyLength("AES") > 128
- }
-
- @Test
- void testShouldDecryptValue() {
- // Arrange
- logger.info("Encryption scheme: ${ENCRYPTION_SCHEME}")
- logger.info("Cipher text: ${CIPHER_TEXT}")
-
- // Act
- String decrypted = bean.decryptValue(CIPHER_TEXT, ENCRYPTION_SCHEME, "propertyName", "ldap-user-group-provider")
- logger.info("Decrypted ${CIPHER_TEXT} -> ${decrypted}")
-
- // Assert
- assert decrypted == PASSWORD
- }
-
- @Test
- void testShouldLoadEncryptedAuthorizersConfiguration() {
- // Arrange
- def identifier = "ldap-user-group-provider"
- def managerPasswordName = "Manager Password"
- Property managerPasswordProperty = new Property(name: managerPasswordName, value: CIPHER_TEXT, encryption: ENCRYPTION_SCHEME)
- List<Property> properties = [managerPasswordProperty]
-
- logger.info("Manager Password property: ${managerPasswordProperty.dump()}")
-
- // Act
- def context = bean.loadAuthorizerConfiguration(identifier, properties)
- logger.info("Loaded context: ${context.dump()}")
-
- // Assert
- String decryptedPropertyValue = context.getProperty(managerPasswordName)
- assert decryptedPropertyValue == PASSWORD
- }
-}
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/java/org/apache/nifi/authorization/AuthorizerFactoryBeanTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/java/org/apache/nifi/authorization/AuthorizerFactoryBeanTest.java
new file mode 100644
index 0000000000..4a959b6062
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/java/org/apache/nifi/authorization/AuthorizerFactoryBeanTest.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.authorization;
+
+import org.apache.nifi.authorization.mock.MockAccessPolicyProvider;
+import org.apache.nifi.authorization.mock.MockAuthorizer;
+import org.apache.nifi.authorization.mock.MockUserGroupProvider;
+import org.apache.nifi.authorization.resource.ResourceFactory;
+import org.apache.nifi.bundle.Bundle;
+import org.apache.nifi.nar.ExtensionManager;
+import org.apache.nifi.util.NiFiProperties;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import java.io.File;
+import java.net.URL;
+import java.util.Collections;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.when;
+
+@ExtendWith(MockitoExtension.class)
+class AuthorizerFactoryBeanTest {
+ private static final String AUTHORIZER_ID = "authorizer";
+
+ private static final String AUTHORIZERS_PATH = "/authorizers.xml";
+
+ private static final AuthorizationRequest REQUEST = new AuthorizationRequest.Builder()
+ .resource(ResourceFactory.getFlowResource())
+ .action(RequestAction.READ)
+ .accessAttempt(true)
+ .anonymous(true)
+ .build();
+
+ @Mock
+ NiFiProperties properties;
+
+ @Mock
+ ExtensionManager extensionManager;
+
+ @Mock
+ Bundle bundle;
+
+ @Test
+ void testGetObjectDefaultAuthorizerRequestApproved() throws Exception {
+ when(properties.getSslPort()).thenReturn(null);
+
+ final AuthorizerFactoryBean bean = new AuthorizerFactoryBean();
+ bean.setProperties(properties);
+
+ final Authorizer authorizer = bean.getObject();
+
+ assertNotNull(authorizer);
+
+ final AuthorizationResult authorizationResult = authorizer.authorize(REQUEST);
+ assertEquals(AuthorizationResult.approved(), authorizationResult);
+ }
+
+ @Test
+ void testGetObject() throws Exception {
+ when(properties.getSslPort()).thenReturn(8443);
+ when(properties.getProperty(eq(NiFiProperties.SECURITY_USER_AUTHORIZER))).thenReturn(AUTHORIZER_ID);
+ when(properties.getAuthorizerConfigurationFile()).thenReturn(getAuthorizersConfigurationFile());
+
+ when(bundle.getClassLoader()).thenReturn(getClass().getClassLoader());
+ final List<Bundle> bundles = Collections.singletonList(bundle);
+
+ when(extensionManager.getBundles(eq(MockUserGroupProvider.class.getName()))).thenReturn(bundles);
+ when(extensionManager.getBundles(eq(MockAccessPolicyProvider.class.getName()))).thenReturn(bundles);
+ when(extensionManager.getBundles(eq(MockAuthorizer.class.getName()))).thenReturn(bundles);
+
+ final AuthorizerFactoryBean bean = new AuthorizerFactoryBean();
+ bean.setProperties(properties);
+ bean.setExtensionManager(extensionManager);
+
+ final Authorizer authorizer = bean.getObject();
+
+ assertNotNull(authorizer);
+ }
+
+ private File getAuthorizersConfigurationFile() {
+ final URL url = AuthorizerFactoryBeanTest.class.getResource(AUTHORIZERS_PATH);
+ if (url == null) {
+ throw new IllegalStateException(String.format("Authorizers [%s] not found", AUTHORIZERS_PATH));
+ }
+ return new File(url.getPath());
+ }
+}
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/java/org/apache/nifi/authorization/mock/MockAccessPolicyProvider.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/java/org/apache/nifi/authorization/mock/MockAccessPolicyProvider.java
new file mode 100644
index 0000000000..1174fbf72d
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/java/org/apache/nifi/authorization/mock/MockAccessPolicyProvider.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.authorization.mock;
+
+import org.apache.nifi.authorization.AccessPolicy;
+import org.apache.nifi.authorization.AccessPolicyProvider;
+import org.apache.nifi.authorization.AccessPolicyProviderInitializationContext;
+import org.apache.nifi.authorization.AuthorizerConfigurationContext;
+import org.apache.nifi.authorization.RequestAction;
+import org.apache.nifi.authorization.UserGroupProvider;
+import org.apache.nifi.authorization.exception.AuthorizationAccessException;
+import org.apache.nifi.authorization.exception.AuthorizerCreationException;
+import org.apache.nifi.authorization.exception.AuthorizerDestructionException;
+
+import java.util.Set;
+
+public class MockAccessPolicyProvider implements AccessPolicyProvider {
+ @Override
+ public Set<AccessPolicy> getAccessPolicies() throws AuthorizationAccessException {
+ return null;
+ }
+
+ @Override
+ public AccessPolicy getAccessPolicy(String identifier) throws AuthorizationAccessException {
+ return null;
+ }
+
+ @Override
+ public AccessPolicy getAccessPolicy(String resourceIdentifier, RequestAction action) throws AuthorizationAccessException {
+ return null;
+ }
+
+ @Override
+ public UserGroupProvider getUserGroupProvider() {
+ return null;
+ }
+
+ @Override
+ public void initialize(AccessPolicyProviderInitializationContext initializationContext) throws AuthorizerCreationException {
+
+ }
+
+ @Override
+ public void onConfigured(AuthorizerConfigurationContext configurationContext) throws AuthorizerCreationException {
+
+ }
+
+ @Override
+ public void preDestruction() throws AuthorizerDestructionException {
+
+ }
+}
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/java/org/apache/nifi/authorization/mock/MockAuthorizer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/java/org/apache/nifi/authorization/mock/MockAuthorizer.java
new file mode 100644
index 0000000000..88bd61e7de
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/java/org/apache/nifi/authorization/mock/MockAuthorizer.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.authorization.mock;
+
+import org.apache.nifi.authorization.AuthorizationRequest;
+import org.apache.nifi.authorization.AuthorizationResult;
+import org.apache.nifi.authorization.Authorizer;
+import org.apache.nifi.authorization.AuthorizerConfigurationContext;
+import org.apache.nifi.authorization.AuthorizerInitializationContext;
+import org.apache.nifi.authorization.exception.AuthorizationAccessException;
+import org.apache.nifi.authorization.exception.AuthorizerCreationException;
+import org.apache.nifi.authorization.exception.AuthorizerDestructionException;
+
+public class MockAuthorizer implements Authorizer {
+ @Override
+ public AuthorizationResult authorize(final AuthorizationRequest request) throws AuthorizationAccessException {
+ return AuthorizationResult.resourceNotFound();
+ }
+
+ @Override
+ public void initialize(final AuthorizerInitializationContext initializationContext) throws AuthorizerCreationException {
+
+ }
+
+ @Override
+ public void onConfigured(final AuthorizerConfigurationContext configurationContext) throws AuthorizerCreationException {
+
+ }
+
+ @Override
+ public void preDestruction() throws AuthorizerDestructionException {
+
+ }
+}
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/java/org/apache/nifi/authorization/mock/MockUserGroupProvider.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/java/org/apache/nifi/authorization/mock/MockUserGroupProvider.java
new file mode 100644
index 0000000000..5867c586c2
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/java/org/apache/nifi/authorization/mock/MockUserGroupProvider.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.authorization.mock;
+
+import org.apache.nifi.authorization.AuthorizerConfigurationContext;
+import org.apache.nifi.authorization.Group;
+import org.apache.nifi.authorization.User;
+import org.apache.nifi.authorization.UserAndGroups;
+import org.apache.nifi.authorization.UserGroupProvider;
+import org.apache.nifi.authorization.UserGroupProviderInitializationContext;
+import org.apache.nifi.authorization.exception.AuthorizationAccessException;
+import org.apache.nifi.authorization.exception.AuthorizerCreationException;
+import org.apache.nifi.authorization.exception.AuthorizerDestructionException;
+
+import java.util.Set;
+
+public class MockUserGroupProvider implements UserGroupProvider {
+ @Override
+ public Set<User> getUsers() throws AuthorizationAccessException {
+ return null;
+ }
+
+ @Override
+ public User getUser(String identifier) throws AuthorizationAccessException {
+ return null;
+ }
+
+ @Override
+ public User getUserByIdentity(String identity) throws AuthorizationAccessException {
+ return null;
+ }
+
+ @Override
+ public Set<Group> getGroups() throws AuthorizationAccessException {
+ return null;
+ }
+
+ @Override
+ public Group getGroup(String identifier) throws AuthorizationAccessException {
+ return null;
+ }
+
+ @Override
+ public UserAndGroups getUserAndGroups(String identity) throws AuthorizationAccessException {
+ return null;
+ }
+
+ @Override
+ public void initialize(UserGroupProviderInitializationContext initializationContext) throws AuthorizerCreationException {
+
+ }
+
+ @Override
+ public void onConfigured(AuthorizerConfigurationContext configurationContext) throws AuthorizerCreationException {
+
+ }
+
+ @Override
+ public void preDestruction() throws AuthorizerDestructionException {
+
+ }
+}
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/resources/authorizers.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/resources/authorizers.xml
new file mode 100644
index 0000000000..6000e66d41
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/resources/authorizers.xml
@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<authorizers>
+ <userGroupProvider>
+ <identifier>user-group-provider</identifier>
+ <class>org.apache.nifi.authorization.mock.MockUserGroupProvider</class>
+ </userGroupProvider>
+
+ <accessPolicyProvider>
+ <identifier>access-policy-provider</identifier>
+ <class>org.apache.nifi.authorization.mock.MockAccessPolicyProvider</class>
+ </accessPolicyProvider>
+
+ <authorizer>
+ <identifier>authorizer</identifier>
+ <class>org.apache.nifi.authorization.mock.MockAuthorizer</class>
+ </authorizer>
+</authorizers>
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/integration/versioned/ImportFlowIT.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/integration/versioned/ImportFlowIT.java
index 2c82165e67..27d6e4a2b1 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/integration/versioned/ImportFlowIT.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/integration/versioned/ImportFlowIT.java
@@ -57,7 +57,6 @@ import org.apache.nifi.registry.flow.diff.StandardFlowComparator;
import org.apache.nifi.registry.flow.mapping.InstantiatedVersionedProcessGroup;
import org.apache.nifi.registry.flow.mapping.NiFiRegistryFlowMapper;
import org.apache.nifi.util.FlowDifferenceFilters;
-import org.jetbrains.annotations.NotNull;
import org.junit.Test;
import java.io.IOException;
@@ -747,7 +746,6 @@ public class ImportFlowIT extends FrameworkIntegrationTest {
return flow;
}
- @NotNull
private VersionedProcessGroup createFlowContents() {
final VersionedProcessGroup flowContents = new VersionedProcessGroup();
flowContents.setIdentifier("unit-test-flow-contents");
@@ -763,7 +761,6 @@ public class ImportFlowIT extends FrameworkIntegrationTest {
bundle.setVersion(coordinate.getVersion());
}
- @NotNull
private VersionedFlow createVersionedFlow() {
final VersionedFlow flow = new VersionedFlow();
flow.setBucketIdentifier("unit-test-bucket");
@@ -774,7 +771,6 @@ public class ImportFlowIT extends FrameworkIntegrationTest {
return flow;
}
- @NotNull
private Bucket createBucket() {
final Bucket bucket = new Bucket();
bucket.setCreatedTimestamp(System.currentTimeMillis());
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/pom.xml
index c765034f73..1ec9d72056 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/pom.xml
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/pom.xml
@@ -21,66 +21,25 @@
</parent>
<artifactId>nifi-properties-loader</artifactId>
<name>nifi-properties-loader</name>
- <description>Handles the loading of the nifi.properties file to an instance of NiFiProperties, and transparently
- performs any decryption/retrieval of sensitive configuration properties.
- </description>
<packaging>jar</packaging>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-properties</artifactId>
</dependency>
- <dependency>
- <groupId>org.bouncycastle</groupId>
- <artifactId>bcprov-jdk15on</artifactId>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jul-to-slf4j</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
- <artifactId>nifi-security-utils</artifactId>
+ <artifactId>nifi-property-protection-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
- <artifactId>nifi-property-protection-api</artifactId>
+ <artifactId>nifi-property-protection-loader</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-property-protection-factory</artifactId>
+ <scope>provided</scope>
+ <!-- Provided through isolated ClassLoader to avoid unnecessary runtime dependencies -->
</dependency>
</dependencies>
- <build>
- <!-- Required to run Groovy tests without any Java tests -->
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <version>1.5</version>
- <executions>
- <execution>
- <id>add-test-source</id>
- <phase>generate-test-sources</phase>
- <goals>
- <goal>add-test-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>src/test/groovy</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
- <scm>
- <tag>nifi-1.16.0-RC3</tag>
- </scm>
-</project>
\ No newline at end of file
+</project>
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/NiFiPropertiesLoader.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/NiFiPropertiesLoader.java
index 77db7fb49d..c2958b4acb 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/NiFiPropertiesLoader.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/NiFiPropertiesLoader.java
@@ -16,9 +16,10 @@
*/
package org.apache.nifi.properties;
+import org.apache.nifi.property.protection.loader.PropertyProtectionURLClassLoader;
+import org.apache.nifi.property.protection.loader.PropertyProviderFactoryLoader;
import org.apache.nifi.util.NiFiBootstrapUtils;
import org.apache.nifi.util.NiFiProperties;
-import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -28,11 +29,12 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.UncheckedIOException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.SecureRandom;
-import java.security.Security;
import java.util.Base64;
import java.util.List;
import java.util.Properties;
@@ -48,13 +50,12 @@ public class NiFiPropertiesLoader {
private static final String MIGRATION_INSTRUCTIONS = "See Admin Guide section [Updating the Sensitive Properties Key]";
private static final String PROPERTIES_KEY_MESSAGE = String.format("Sensitive Properties Key [%s] not found: %s", NiFiProperties.SENSITIVE_PROPS_KEY, MIGRATION_INSTRUCTIONS);
+ private static final String SET_KEY_METHOD = "setKeyHex";
+
private final String defaultPropertiesFilePath = NiFiBootstrapUtils.getDefaultApplicationPropertiesFilePath();
private NiFiProperties instance;
private String keyHex;
- // Future enhancement: allow for external registration of new providers
- private SensitivePropertyProviderFactory sensitivePropertyProviderFactory;
-
public NiFiPropertiesLoader() {
}
@@ -96,13 +97,9 @@ public class NiFiPropertiesLoader {
* startup.
*
* @return the populated and decrypted NiFiProperties instance
- * @throws IOException if there is a problem reading from the bootstrap.conf
- * or nifi.properties files
*/
- public static NiFiProperties loadDefaultWithKeyFromBootstrap() throws IOException {
+ public static NiFiProperties loadDefaultWithKeyFromBootstrap() {
try {
- // The default behavior of StandardSensitivePropertiesFactory is to use the key
- // from bootstrap.conf if no key is provided
return new NiFiPropertiesLoader().loadDefault();
} catch (Exception e) {
logger.warn("Encountered an error naively loading the nifi.properties file because one or more properties are protected: {}", e.getLocalizedMessage());
@@ -110,17 +107,6 @@ public class NiFiPropertiesLoader {
}
}
- private NiFiProperties loadDefault() {
- return load(defaultPropertiesFilePath);
- }
-
- private SensitivePropertyProviderFactory getSensitivePropertyProviderFactory() {
- if (sensitivePropertyProviderFactory == null) {
- sensitivePropertyProviderFactory = StandardSensitivePropertyProviderFactory.withKey(keyHex);
- }
- return sensitivePropertyProviderFactory;
- }
-
/**
* Returns a {@link ProtectedNiFiProperties} instance loaded from the
* serialized form in the file. Responsible for actually reading from disk
@@ -130,29 +116,25 @@ public class NiFiPropertiesLoader {
* @param file the file containing serialized properties
* @return the ProtectedNiFiProperties instance
*/
- ProtectedNiFiProperties readProtectedPropertiesFromDisk(File file) {
+ ProtectedNiFiProperties loadProtectedProperties(final File file) {
if (file == null || !file.exists() || !file.canRead()) {
- String path = (file == null ? "missing file" : file.getAbsolutePath());
- logger.error("Cannot read from '{}' -- file is missing or not readable", path);
- throw new IllegalArgumentException("NiFi properties file missing or unreadable");
+ throw new IllegalArgumentException(String.format("Application Properties [%s] not found", file));
}
- final Properties rawProperties = new Properties();
+ logger.info("Loading Application Properties [{}]", file);
+ final Properties properties = new Properties();
try (final InputStream inputStream = new BufferedInputStream(new FileInputStream(file))) {
- rawProperties.load(inputStream);
- logger.info("Loaded {} properties from {}", rawProperties.size(), file.getAbsolutePath());
+ properties.load(inputStream);
- final Set<String> keys = rawProperties.stringPropertyNames();
+ final Set<String> keys = properties.stringPropertyNames();
for (final String key : keys) {
- final String property = rawProperties.getProperty(key);
- rawProperties.setProperty(key, property.trim());
+ final String property = properties.getProperty(key);
+ properties.setProperty(key, property.trim());
}
- return new ProtectedNiFiProperties(rawProperties);
- } catch (final Exception ex) {
- logger.error("Cannot load properties file due to {}", ex.getLocalizedMessage());
- throw new RuntimeException("Cannot load properties file due to "
- + ex.getLocalizedMessage(), ex);
+ return new ProtectedNiFiProperties(properties);
+ } catch (final Exception e) {
+ throw new RuntimeException(String.format("Loading Application Properties [%s] failed", file), e);
}
}
@@ -166,20 +148,32 @@ public class NiFiPropertiesLoader {
* @return the NiFiProperties instance
*/
public NiFiProperties load(final File file) {
- final ProtectedNiFiProperties protectedNiFiProperties = readProtectedPropertiesFromDisk(file);
- if (protectedNiFiProperties.hasProtectedKeys()) {
- Security.addProvider(new BouncyCastleProvider());
- getSensitivePropertyProviderFactory()
- .getSupportedProviders()
- .forEach(protectedNiFiProperties::addSensitivePropertyProvider);
- }
- NiFiProperties props = protectedNiFiProperties.getUnprotectedProperties();
- if (protectedNiFiProperties.hasProtectedKeys()) {
- getSensitivePropertyProviderFactory()
- .getSupportedProviders()
- .forEach(SensitivePropertyProvider::cleanUp);
+ final ProtectedNiFiProperties protectedProperties = loadProtectedProperties(file);
+ final NiFiProperties properties;
+
+ if (protectedProperties.hasProtectedKeys()) {
+ final ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
+
+ try {
+ final PropertyProtectionURLClassLoader protectionClassLoader = new PropertyProtectionURLClassLoader(contextClassLoader);
+ Thread.currentThread().setContextClassLoader(protectionClassLoader);
+
+ final PropertyProviderFactoryLoader factoryLoader = new PropertyProviderFactoryLoader();
+ final SensitivePropertyProviderFactory providerFactory = factoryLoader.getPropertyProviderFactory();
+ setBootstrapKey(providerFactory);
+ providerFactory.getSupportedProviders().forEach(protectedProperties::addSensitivePropertyProvider);
+
+ properties = protectedProperties.getUnprotectedProperties();
+
+ providerFactory.getSupportedProviders().forEach(SensitivePropertyProvider::cleanUp);
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ } else {
+ properties = protectedProperties.getUnprotectedProperties();
}
- return props;
+
+ return properties;
}
/**
@@ -217,6 +211,10 @@ public class NiFiPropertiesLoader {
return instance;
}
+ private NiFiProperties loadDefault() {
+ return load(defaultPropertiesFilePath);
+ }
+
private NiFiProperties getDefaultProperties() {
NiFiProperties defaultProperties = loadDefault();
if (isKeyGenerationRequired(defaultProperties)) {
@@ -272,4 +270,23 @@ public class NiFiPropertiesLoader {
final String configuredSensitivePropertiesKey = properties.getProperty(NiFiProperties.SENSITIVE_PROPS_KEY);
return (configuredSensitivePropertiesKey == null || configuredSensitivePropertiesKey.length() == 0);
}
+
+ private void setBootstrapKey(final SensitivePropertyProviderFactory providerFactory) {
+ if (keyHex == null) {
+ logger.debug("Bootstrap Sensitive Key not configured");
+ } else {
+ final Class<? extends SensitivePropertyProviderFactory> factoryClass = providerFactory.getClass();
+ try {
+ // Set Bootstrap Key using reflection to preserve ClassLoader isolation
+ final Method setMethod = factoryClass.getMethod(SET_KEY_METHOD, String.class);
+ setMethod.invoke(providerFactory, keyHex);
+ } catch (final NoSuchMethodException e) {
+ logger.warn("Method [{}] on Class [{}] not found", SET_KEY_METHOD, factoryClass.getName());
+ } catch (final IllegalAccessException e) {
+ logger.warn("Method [{}] on Class [{}] access not allowed", SET_KEY_METHOD, factoryClass.getName());
+ } catch (final InvocationTargetException e) {
+ throw new SensitivePropertyProtectionException("Set Bootstrap Key on Provider Factory failed", e);
+ }
+ }
+ }
}
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/NiFiPropertiesGroovyTest.groovy b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/NiFiPropertiesGroovyTest.groovy
deleted file mode 100644
index 485050bd1b..0000000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/NiFiPropertiesGroovyTest.groovy
+++ /dev/null
@@ -1,1029 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.properties
-
-import org.apache.nifi.util.NiFiProperties
-import org.junit.After
-import org.junit.AfterClass
-import org.junit.Before
-import org.junit.BeforeClass
-import org.junit.Test
-import org.junit.runner.RunWith
-import org.junit.runners.JUnit4
-import org.slf4j.Logger
-import org.slf4j.LoggerFactory
-
-@RunWith(JUnit4.class)
-class NiFiPropertiesGroovyTest extends GroovyTestCase {
- private static final Logger logger = LoggerFactory.getLogger(NiFiPropertiesGroovyTest.class)
-
- private static String originalPropertiesPath = System.getProperty(NiFiProperties.PROPERTIES_FILE_PATH)
- private static final String PREK = NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY
- private static final String PREKID = NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_ID
-
- private static final String CREK = NiFiProperties.CONTENT_REPOSITORY_ENCRYPTION_KEY
- private static final String CREKID = NiFiProperties.CONTENT_REPOSITORY_ENCRYPTION_KEY_ID
-
- private static final String FFREK = NiFiProperties.FLOWFILE_REPOSITORY_ENCRYPTION_KEY
- private static final String FFREKID = NiFiProperties.FLOWFILE_REPOSITORY_ENCRYPTION_KEY_ID
-
- @BeforeClass
- static void setUpOnce() throws Exception {
- logger.metaClass.methodMissing = { String name, args ->
- logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}")
- }
- }
-
- @Before
- void setUp() throws Exception {
- }
-
- @After
- void tearDown() throws Exception {
- }
-
- @AfterClass
- static void tearDownOnce() {
- if (originalPropertiesPath) {
- System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, originalPropertiesPath)
- }
- }
-
- private static NiFiProperties loadFromFile(String propertiesFilePath) {
- String filePath
- try {
- filePath = NiFiPropertiesGroovyTest.class.getResource(propertiesFilePath).toURI().getPath()
- } catch (URISyntaxException ex) {
- throw new RuntimeException("Cannot load properties file due to " +
- ex.getLocalizedMessage(), ex)
- }
-
- System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, filePath)
-
- NiFiProperties properties = new NiFiProperties()
-
- // clear out existing properties
- for (String prop : properties.stringPropertyNames()) {
- properties.remove(prop)
- }
-
- InputStream inStream = null
- try {
- inStream = new BufferedInputStream(new FileInputStream(filePath))
- properties.load(inStream)
- } catch (final Exception ex) {
- throw new RuntimeException("Cannot load properties file due to " +
- ex.getLocalizedMessage(), ex)
- } finally {
- if (null != inStream) {
- try {
- inStream.close()
- } catch (Exception ex) {
- /**
- * do nothing *
- */
- }
- }
- }
-
- return properties
- }
-
- @Test
- void testConstructorShouldCreateNewInstance() throws Exception {
- // Arrange
-
- // Act
- NiFiProperties niFiProperties = new NiFiProperties()
- logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
-
- // Assert
- assert niFiProperties.size() == 0
- assert niFiProperties.getPropertyKeys() == [] as Set
- }
-
- @Test
- void testConstructorShouldAcceptRawProperties() throws Exception {
- // Arrange
- Properties rawProperties = new Properties()
- rawProperties.setProperty("key", "value")
- logger.info("rawProperties has ${rawProperties.size()} properties: ${rawProperties.stringPropertyNames()}")
- assert rawProperties.size() == 1
-
- // Act
- NiFiProperties niFiProperties = new NiFiProperties(rawProperties)
- logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
-
- // Assert
- assert niFiProperties.size() == 1
- assert niFiProperties.getPropertyKeys() == ["key"] as Set
- }
-
- @Test
- void testShouldAllowMultipleInstances() throws Exception {
- // Arrange
- Properties rawProperties = new Properties()
- rawProperties.setProperty("key", "value")
- logger.info("rawProperties has ${rawProperties.size()} properties: ${rawProperties.stringPropertyNames()}")
- assert rawProperties.size() == 1
-
- // Act
- NiFiProperties niFiProperties = new NiFiProperties(rawProperties)
- logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
- NiFiProperties emptyProperties = new NiFiProperties()
- logger.info("emptyProperties has ${emptyProperties.size()} properties: ${emptyProperties.getPropertyKeys()}")
-
- // Assert
- assert niFiProperties.size() == 1
- assert niFiProperties.getPropertyKeys() == ["key"] as Set
-
- assert emptyProperties.size() == 0
- assert emptyProperties.getPropertyKeys() == [] as Set
- }
-
- @Test
- void testShouldGetProvenanceRepoEncryptionKeyFromDefaultProperty() throws Exception {
- // Arrange
- Properties rawProperties = new Properties()
- final String KEY_ID = "arbitraryKeyId"
- final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210"
- rawProperties.setProperty(PREKID, KEY_ID)
- rawProperties.setProperty(PREK, KEY_HEX)
- NiFiProperties niFiProperties = new NiFiProperties(rawProperties)
- logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
-
- // Act
- def keyId = niFiProperties.getProvenanceRepoEncryptionKeyId()
- def key = niFiProperties.getProvenanceRepoEncryptionKey()
- def keys = niFiProperties.getProvenanceRepoEncryptionKeys()
-
- logger.info("Retrieved key ID: ${keyId}")
- logger.info("Retrieved key: ${key}")
- logger.info("Retrieved keys: ${keys}")
-
- // Assert
- assert keyId == KEY_ID
- assert key == KEY_HEX
- assert keys == [(KEY_ID): KEY_HEX]
- }
-
- @Test
- void testShouldGetProvenanceRepoEncryptionKeysFromMultipleProperties() throws Exception {
- // Arrange
- Properties rawProperties = new Properties()
- final String KEY_ID = "arbitraryKeyId"
- final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210"
- final String KEY_ID_2 = "arbitraryKeyId2"
- final String KEY_HEX_2 = "AAAABBBBCCCCDDDDEEEEFFFF00001111"
- final String KEY_ID_3 = "arbitraryKeyId3"
- final String KEY_HEX_3 = "01010101010101010101010101010101"
-
- rawProperties.setProperty(PREKID, KEY_ID)
- rawProperties.setProperty(PREK, KEY_HEX)
- rawProperties.setProperty("${PREK}.id.${KEY_ID_2}", KEY_HEX_2)
- rawProperties.setProperty("${PREK}.id.${KEY_ID_3}", KEY_HEX_3)
- NiFiProperties niFiProperties = new NiFiProperties(rawProperties)
- logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
-
- // Act
- def keyId = niFiProperties.getProvenanceRepoEncryptionKeyId()
- def key = niFiProperties.getProvenanceRepoEncryptionKey()
- def keys = niFiProperties.getProvenanceRepoEncryptionKeys()
-
- logger.info("Retrieved key ID: ${keyId}")
- logger.info("Retrieved key: ${key}")
- logger.info("Retrieved keys: ${keys}")
-
- // Assert
- assert keyId == KEY_ID
- assert key == KEY_HEX
- assert keys == [(KEY_ID): KEY_HEX, (KEY_ID_2): KEY_HEX_2, (KEY_ID_3): KEY_HEX_3]
- }
-
- @Test
- void testShouldGetFlowFileRepoEncryptionKeysFromMultiplePropertiesAfterMigration() throws Exception {
- // Arrange
- Properties rawProperties = new Properties()
- final String KEY_ID = "K1"
- final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210"
- final String KEY_ID_2 = "K2"
- final String KEY_HEX_2 = "AAAABBBBCCCCDDDDEEEEFFFF00001111"
-
- /**
- * This simulates after the initial key migration (K1 -> K2). The {@code nifi.properties} will look like:
- *
- * nifi.flowfile.repository.encryption.key.id=K2
- * nifi.flowfile.repository.encryption.key=
- * nifi.flowfile.repository.encryption.key.id.K1=0123456789ABCDEFFEDCBA9876543210
- * nifi.flowfile.repository.encryption.key.id.K2=00000000000000000000000000000000
- */
-
- rawProperties.setProperty(FFREKID, KEY_ID_2)
- rawProperties.setProperty(FFREK, "")
- rawProperties.setProperty("${FFREK}.id.${KEY_ID}", KEY_HEX)
- rawProperties.setProperty("${FFREK}.id.${KEY_ID_2}", KEY_HEX_2)
- NiFiProperties niFiProperties = new NiFiProperties(rawProperties)
- logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
-
- // Act
- def keyId = niFiProperties.getFlowFileRepoEncryptionKeyId()
- def key = niFiProperties.getFlowFileRepoEncryptionKey()
- def keys = niFiProperties.getFlowFileRepoEncryptionKeys()
-
- logger.info("Retrieved key ID: ${keyId}")
- logger.info("Retrieved key: ${key}")
- logger.info("Retrieved keys: ${keys}")
-
- // Assert
- assert keyId == KEY_ID_2
- assert key == KEY_HEX_2
- assert keys == [(KEY_ID): KEY_HEX, (KEY_ID_2): KEY_HEX_2]
- }
-
- @Test
- void testShouldGetFlowFileRepoEncryptionKeysFromMultiplePropertiesWithoutExplicitKey() throws Exception {
- // Arrange
- Properties rawProperties = new Properties()
- final String KEY_ID = "K1"
- final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210"
- final String KEY_ID_2 = "K2"
- final String KEY_HEX_2 = "AAAABBBBCCCCDDDDEEEEFFFF00001111"
-
- /**
- * This simulates after the initial key migration (K1 -> K2). The {@code nifi.properties} will look like:
- *
- * (note no nifi.flowfile.repository.encryption.key=)
- * nifi.flowfile.repository.encryption.key.id=K2
- * nifi.flowfile.repository.encryption.key.id.K1=0123456789ABCDEFFEDCBA9876543210
- * nifi.flowfile.repository.encryption.key.id.K2=00000000000000000000000000000000
- */
-
- rawProperties.setProperty(FFREKID, KEY_ID_2)
-// rawProperties.setProperty(FFREK, "")
- rawProperties.setProperty("${FFREK}.id.${KEY_ID}", KEY_HEX)
- rawProperties.setProperty("${FFREK}.id.${KEY_ID_2}", KEY_HEX_2)
- NiFiProperties niFiProperties = new NiFiProperties(rawProperties)
- logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
-
- // Act
- def keyId = niFiProperties.getFlowFileRepoEncryptionKeyId()
- def key = niFiProperties.getFlowFileRepoEncryptionKey()
- def keys = niFiProperties.getFlowFileRepoEncryptionKeys()
-
- logger.info("Retrieved key ID: ${keyId}")
- logger.info("Retrieved key: ${key}")
- logger.info("Retrieved keys: ${keys}")
-
- // Assert
- assert keyId == KEY_ID_2
- assert key == KEY_HEX_2
- assert keys == [(KEY_ID): KEY_HEX, (KEY_ID_2): KEY_HEX_2]
- }
-
- @Test
- void testGetFlowFileRepoEncryptionKeysShouldWarnOnMisformattedProperties() throws Exception {
- // Arrange
- Properties rawProperties = new Properties()
- final String KEY_ID = "K1"
- final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210"
- final String KEY_ID_2 = "K2"
- final String KEY_HEX_2 = "AAAABBBBCCCCDDDDEEEEFFFF00001111"
-
- /**
- * This simulates after the initial key migration (K1 -> K2) when the admin has mistyped. The {@code nifi.properties} will look like:
- *
- * (note no nifi.flowfile.repository.encryption.key=)
- * nifi.flowfile.repository.encryption.key.id=K2
- * nifi.flowfile.repository.encryption.key.K1=0123456789ABCDEFFEDCBA9876543210
- * nifi.flowfile.repository.encryption.key.K2=00000000000000000000000000000000
- *
- * The above properties should have ...key.id.K1= but they are missing the "id" segment
- */
-
- rawProperties.setProperty(FFREKID, KEY_ID_2)
-// rawProperties.setProperty(FFREK, "")
- rawProperties.setProperty("${FFREK}.${KEY_ID}", KEY_HEX)
- rawProperties.setProperty("${FFREK}.${KEY_ID_2}", KEY_HEX_2)
- NiFiProperties niFiProperties = new NiFiProperties(rawProperties)
- logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
-
- // Act
- def keyId = niFiProperties.getFlowFileRepoEncryptionKeyId()
- def key = niFiProperties.getFlowFileRepoEncryptionKey()
- def keys = niFiProperties.getFlowFileRepoEncryptionKeys()
-
- logger.info("Retrieved key ID: ${keyId}")
- logger.info("Retrieved key: ${key}")
- logger.info("Retrieved keys: ${keys}")
-
- // Assert
- assert keyId == KEY_ID_2
- assert !key
- assert keys == [:]
- }
-
- @Test
- void testShouldGetFlowFileRepoEncryptionKeysFromMultiplePropertiesWithDuplicates() throws Exception {
- // Arrange
- Properties rawProperties = new Properties()
- final String KEY_ID = "K1"
- final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210"
- final String KEY_ID_2 = "K2"
- final String KEY_HEX_2 = "AAAABBBBCCCCDDDDEEEEFFFF00001111"
- final String KEY_HEX_DUP = "AA" * 16
-
- /**
- * This simulates after the initial key migration (K1 -> K2) with a mistaken duplication. The
- * {@code nifi.properties} will look like:
- *
- * nifi.flowfile.repository.encryption.key.id=K2
- * nifi.flowfile.repository.encryption.key=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
- * nifi.flowfile.repository.encryption.key.id.K1=0123456789ABCDEFFEDCBA9876543210
- * nifi.flowfile.repository.encryption.key.id.K2=00000000000000000000000000000000
- *
- * The value of K2 wil be AAAA..., overriding key.K2 as .key will always win
- */
-
- /**
- * The properties loading code will print a warning if it detects duplicates, but will not stop execution
- */
- rawProperties.setProperty(FFREKID, KEY_ID_2)
- rawProperties.setProperty(FFREK, KEY_HEX_DUP)
- rawProperties.setProperty("${FFREK}.id.${KEY_ID}", KEY_HEX)
- rawProperties.setProperty("${FFREK}.id.${KEY_ID_2}", KEY_HEX_2)
- NiFiProperties niFiProperties = new NiFiProperties(rawProperties)
- logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
-
- // Act
- def keyId = niFiProperties.getFlowFileRepoEncryptionKeyId()
- def key = niFiProperties.getFlowFileRepoEncryptionKey()
- def keys = niFiProperties.getFlowFileRepoEncryptionKeys()
-
- logger.info("Retrieved key ID: ${keyId}")
- logger.info("Retrieved key: ${key}")
- logger.info("Retrieved keys: ${keys}")
-
- // Assert
- assert keyId == KEY_ID_2
- assert key == KEY_HEX_2
- assert keys == [(KEY_ID): KEY_HEX, (KEY_ID_2): KEY_HEX_DUP]
- }
-
- @Test
- void testShouldGetFlowFileRepoEncryptionKeysFromMultiplePropertiesWithDuplicatesInReverseOrder() throws Exception {
- // Arrange
- Properties rawProperties = new Properties()
- final String KEY_ID = "K1"
- final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210"
- final String KEY_ID_2 = "K2"
- final String KEY_HEX_2 = "AAAABBBBCCCCDDDDEEEEFFFF00001111"
- final String KEY_HEX_DUP = "AA" * 16
-
- /**
- * This simulates after the initial key migration (K1 -> K2) with a mistaken duplication. The
- * {@code nifi.properties} will look like:
- *
- * nifi.flowfile.repository.encryption.key.id.K1=0123456789ABCDEFFEDCBA9876543210
- * nifi.flowfile.repository.encryption.key.id.K2=00000000000000000000000000000000
- * nifi.flowfile.repository.encryption.key=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
- * nifi.flowfile.repository.encryption.key.id=K2
- *
- * The value of K2 wil be AAAA..., overriding key.K2 as .key will always win
- */
-
- /**
- * The properties loading code will print a warning if it detects duplicates, but will not stop execution
- */
- rawProperties.setProperty("${FFREK}.id.${KEY_ID_2}", KEY_HEX_2)
- rawProperties.setProperty("${FFREK}.id.${KEY_ID}", KEY_HEX)
- rawProperties.setProperty(FFREK, KEY_HEX_DUP)
- rawProperties.setProperty(FFREKID, KEY_ID_2)
- NiFiProperties niFiProperties = new NiFiProperties(rawProperties)
- logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
-
- // Act
- def keyId = niFiProperties.getFlowFileRepoEncryptionKeyId()
- def key = niFiProperties.getFlowFileRepoEncryptionKey()
- def keys = niFiProperties.getFlowFileRepoEncryptionKeys()
-
- logger.info("Retrieved key ID: ${keyId}")
- logger.info("Retrieved key: ${key}")
- logger.info("Retrieved keys: ${keys}")
-
- // Assert
- assert keyId == KEY_ID_2
- assert key == KEY_HEX_2
- assert keys == [(KEY_ID): KEY_HEX, (KEY_ID_2): KEY_HEX_DUP]
- }
-
- @Test
- void testShouldGetProvenanceRepoEncryptionKeysWithNoDefaultDefined() throws Exception {
- // Arrange
- Properties rawProperties = new Properties()
- final String KEY_ID = "arbitraryKeyId"
- final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210"
- final String KEY_ID_2 = "arbitraryKeyId2"
- final String KEY_HEX_2 = "AAAABBBBCCCCDDDDEEEEFFFF00001111"
- final String KEY_ID_3 = "arbitraryKeyId3"
- final String KEY_HEX_3 = "01010101010101010101010101010101"
-
- rawProperties.setProperty(PREKID, KEY_ID)
- rawProperties.setProperty("${PREK}.id.${KEY_ID}", KEY_HEX)
- rawProperties.setProperty("${PREK}.id.${KEY_ID_2}", KEY_HEX_2)
- rawProperties.setProperty("${PREK}.id.${KEY_ID_3}", KEY_HEX_3)
- NiFiProperties niFiProperties = new NiFiProperties(rawProperties)
- logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
-
- // Act
- def keyId = niFiProperties.getProvenanceRepoEncryptionKeyId()
- def key = niFiProperties.getProvenanceRepoEncryptionKey()
- def keys = niFiProperties.getProvenanceRepoEncryptionKeys()
-
- logger.info("Retrieved key ID: ${keyId}")
- logger.info("Retrieved key: ${key}")
- logger.info("Retrieved keys: ${keys}")
-
- // Assert
- assert keyId == KEY_ID
- assert key == KEY_HEX
- assert keys == [(KEY_ID): KEY_HEX, (KEY_ID_2): KEY_HEX_2, (KEY_ID_3): KEY_HEX_3]
- }
-
- @Test
- void testShouldGetProvenanceRepoEncryptionKeysWithNoneDefined() throws Exception {
- // Arrange
- Properties rawProperties = new Properties()
- NiFiProperties niFiProperties = new NiFiProperties(rawProperties)
- logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
-
- // Act
- def keyId = niFiProperties.getProvenanceRepoEncryptionKeyId()
- def key = niFiProperties.getProvenanceRepoEncryptionKey()
- def keys = niFiProperties.getProvenanceRepoEncryptionKeys()
-
- logger.info("Retrieved key ID: ${keyId}")
- logger.info("Retrieved key: ${key}")
- logger.info("Retrieved keys: ${keys}")
-
- // Assert
- assert keyId == null
- assert key == null
- assert keys == [:]
- }
-
- @Test
- void testShouldNotGetProvenanceRepoEncryptionKeysIfFileBasedKeyProvider() throws Exception {
- // Arrange
- Properties rawProperties = new Properties()
- final String KEY_ID = "arbitraryKeyId"
-
- rawProperties.setProperty(PREKID, KEY_ID)
- NiFiProperties niFiProperties = new NiFiProperties(rawProperties)
- logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
-
- // Act
- def keyId = niFiProperties.getProvenanceRepoEncryptionKeyId()
- def key = niFiProperties.getProvenanceRepoEncryptionKey()
- def keys = niFiProperties.getProvenanceRepoEncryptionKeys()
-
- logger.info("Retrieved key ID: ${keyId}")
- logger.info("Retrieved key: ${key}")
- logger.info("Retrieved keys: ${keys}")
-
- // Assert
- assert keyId == KEY_ID
- assert key == null
- assert keys == [:]
- }
-
- @Test
- void testGetProvenanceRepoEncryptionKeysShouldFilterOtherProperties() throws Exception {
- // Arrange
- Properties rawProperties = new Properties()
- final String KEY_ID = "arbitraryKeyId"
- final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210"
- final String KEY_ID_2 = "arbitraryKeyId2"
- final String KEY_HEX_2 = "AAAABBBBCCCCDDDDEEEEFFFF00001111"
- final String KEY_ID_3 = "arbitraryKeyId3"
- final String KEY_HEX_3 = "01010101010101010101010101010101"
-
- rawProperties.setProperty(PREKID, KEY_ID)
- rawProperties.setProperty("${PREK}.id.${KEY_ID}", KEY_HEX)
- rawProperties.setProperty("${PREK}.id.${KEY_ID_2}", KEY_HEX_2)
- rawProperties.setProperty("${PREK}.id.${KEY_ID_3}", KEY_HEX_3)
- rawProperties.setProperty(NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_IMPLEMENTATION_CLASS, "some.class.provider")
- rawProperties.setProperty(NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_LOCATION, "some://url")
- NiFiProperties niFiProperties = new NiFiProperties(rawProperties)
- logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
-
- // Act
- def keyId = niFiProperties.getProvenanceRepoEncryptionKeyId()
- def key = niFiProperties.getProvenanceRepoEncryptionKey()
- def keys = niFiProperties.getProvenanceRepoEncryptionKeys()
-
- logger.info("Retrieved key ID: ${keyId}")
- logger.info("Retrieved key: ${key}")
- logger.info("Retrieved keys: ${keys}")
-
- // Assert
- assert keyId == KEY_ID
- assert key == KEY_HEX
- assert keys == [(KEY_ID): KEY_HEX, (KEY_ID_2): KEY_HEX_2, (KEY_ID_3): KEY_HEX_3]
- }
-
- @Test
- void testShouldGetContentRepositoryEncryptionKeyFromDefaultProperty() throws Exception {
- // Arrange
- Properties rawProperties = new Properties()
- final String KEY_ID = "arbitraryKeyId"
- final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210"
- rawProperties.setProperty(CREKID, KEY_ID)
- rawProperties.setProperty(CREK, KEY_HEX)
- NiFiProperties niFiProperties = new NiFiProperties(rawProperties)
- logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
-
- // Act
- def keyId = niFiProperties.getContentRepositoryEncryptionKeyId()
- def key = niFiProperties.getContentRepositoryEncryptionKey()
- def keys = niFiProperties.getContentRepositoryEncryptionKeys()
-
- logger.info("Retrieved key ID: ${keyId}")
- logger.info("Retrieved key: ${key}")
- logger.info("Retrieved keys: ${keys}")
-
- // Assert
- assert keyId == KEY_ID
- assert key == KEY_HEX
- assert keys == [(KEY_ID): KEY_HEX]
- }
-
- @Test
- void testShouldGetContentRepositoryEncryptionKeysFromMultipleProperties() throws Exception {
- // Arrange
- Properties rawProperties = new Properties()
- final String KEY_ID = "arbitraryKeyId"
- final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210"
- final String KEY_ID_2 = "arbitraryKeyId2"
- final String KEY_HEX_2 = "AAAABBBBCCCCDDDDEEEEFFFF00001111"
- final String KEY_ID_3 = "arbitraryKeyId3"
- final String KEY_HEX_3 = "01010101010101010101010101010101"
-
- rawProperties.setProperty(CREKID, KEY_ID)
- rawProperties.setProperty(CREK, KEY_HEX)
- rawProperties.setProperty("${CREK}.id.${KEY_ID_2}", KEY_HEX_2)
- rawProperties.setProperty("${CREK}.id.${KEY_ID_3}", KEY_HEX_3)
- NiFiProperties niFiProperties = new NiFiProperties(rawProperties)
- logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
-
- // Act
- def keyId = niFiProperties.getContentRepositoryEncryptionKeyId()
- def key = niFiProperties.getContentRepositoryEncryptionKey()
- def keys = niFiProperties.getContentRepositoryEncryptionKeys()
-
- logger.info("Retrieved key ID: ${keyId}")
- logger.info("Retrieved key: ${key}")
- logger.info("Retrieved keys: ${keys}")
-
- // Assert
- assert keyId == KEY_ID
- assert key == KEY_HEX
- assert keys == [(KEY_ID): KEY_HEX, (KEY_ID_2): KEY_HEX_2, (KEY_ID_3): KEY_HEX_3]
- }
-
- @Test
- void testShouldGetContentRepositoryEncryptionKeysWithNoDefaultDefined() throws Exception {
- // Arrange
- Properties rawProperties = new Properties()
- final String KEY_ID = "arbitraryKeyId"
- final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210"
- final String KEY_ID_2 = "arbitraryKeyId2"
- final String KEY_HEX_2 = "AAAABBBBCCCCDDDDEEEEFFFF00001111"
- final String KEY_ID_3 = "arbitraryKeyId3"
- final String KEY_HEX_3 = "01010101010101010101010101010101"
-
- rawProperties.setProperty(CREKID, KEY_ID)
- rawProperties.setProperty("${CREK}.id.${KEY_ID}", KEY_HEX)
- rawProperties.setProperty("${CREK}.id.${KEY_ID_2}", KEY_HEX_2)
- rawProperties.setProperty("${CREK}.id.${KEY_ID_3}", KEY_HEX_3)
- NiFiProperties niFiProperties = new NiFiProperties(rawProperties)
- logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
-
- // Act
- def keyId = niFiProperties.getContentRepositoryEncryptionKeyId()
- def key = niFiProperties.getContentRepositoryEncryptionKey()
- def keys = niFiProperties.getContentRepositoryEncryptionKeys()
-
- logger.info("Retrieved key ID: ${keyId}")
- logger.info("Retrieved key: ${key}")
- logger.info("Retrieved keys: ${keys}")
-
- // Assert
- assert keyId == KEY_ID
- assert key == KEY_HEX
- assert keys == [(KEY_ID): KEY_HEX, (KEY_ID_2): KEY_HEX_2, (KEY_ID_3): KEY_HEX_3]
- }
-
- @Test
- void testShouldGetContentRepositoryEncryptionKeysWithNoneDefined() throws Exception {
- // Arrange
- Properties rawProperties = new Properties()
- NiFiProperties niFiProperties = new NiFiProperties(rawProperties)
- logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
-
- // Act
- def keyId = niFiProperties.getContentRepositoryEncryptionKeyId()
- def key = niFiProperties.getContentRepositoryEncryptionKey()
- def keys = niFiProperties.getContentRepositoryEncryptionKeys()
-
- logger.info("Retrieved key ID: ${keyId}")
- logger.info("Retrieved key: ${key}")
- logger.info("Retrieved keys: ${keys}")
-
- // Assert
- assert keyId == null
- assert key == null
- assert keys == [:]
- }
-
- @Test
- void testShouldNotGetContentRepositoryEncryptionKeysIfFileBasedKeyProvider() throws Exception {
- // Arrange
- Properties rawProperties = new Properties()
- final String KEY_ID = "arbitraryKeyId"
-
- rawProperties.setProperty(CREKID, KEY_ID)
- NiFiProperties niFiProperties = new NiFiProperties(rawProperties)
- logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
-
- // Act
- def keyId = niFiProperties.getContentRepositoryEncryptionKeyId()
- def key = niFiProperties.getContentRepositoryEncryptionKey()
- def keys = niFiProperties.getContentRepositoryEncryptionKeys()
-
- logger.info("Retrieved key ID: ${keyId}")
- logger.info("Retrieved key: ${key}")
- logger.info("Retrieved keys: ${keys}")
-
- // Assert
- assert keyId == KEY_ID
- assert key == null
- assert keys == [:]
- }
-
- @Test
- void testGetContentRepoEncryptionKeysShouldFilterOtherProperties() throws Exception {
- // Arrange
- Properties rawProperties = new Properties()
- final String KEY_ID = "arbitraryKeyId"
- final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210"
- final String KEY_ID_2 = "arbitraryKeyId2"
- final String KEY_HEX_2 = "AAAABBBBCCCCDDDDEEEEFFFF00001111"
- final String KEY_ID_3 = "arbitraryKeyId3"
- final String KEY_HEX_3 = "01010101010101010101010101010101"
-
- rawProperties.setProperty(CREKID, KEY_ID)
- rawProperties.setProperty("${CREK}.id.${KEY_ID}", KEY_HEX)
- rawProperties.setProperty("${CREK}.id.${KEY_ID_2}", KEY_HEX_2)
- rawProperties.setProperty("${CREK}.id.${KEY_ID_3}", KEY_HEX_3)
- rawProperties.setProperty(NiFiProperties.CONTENT_REPOSITORY_ENCRYPTION_KEY_PROVIDER_IMPLEMENTATION_CLASS, "some.class.provider")
- rawProperties.setProperty(NiFiProperties.CONTENT_REPOSITORY_ENCRYPTION_KEY_PROVIDER_LOCATION, "some://url")
- NiFiProperties niFiProperties = new NiFiProperties(rawProperties)
- logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
-
- // Act
- def keyId = niFiProperties.getContentRepositoryEncryptionKeyId()
- def key = niFiProperties.getContentRepositoryEncryptionKey()
- def keys = niFiProperties.getContentRepositoryEncryptionKeys()
-
- logger.info("Retrieved key ID: ${keyId}")
- logger.info("Retrieved key: ${key}")
- logger.info("Retrieved keys: ${keys}")
-
- // Assert
- assert keyId == KEY_ID
- assert key == KEY_HEX
- assert keys == [(KEY_ID): KEY_HEX, (KEY_ID_2): KEY_HEX_2, (KEY_ID_3): KEY_HEX_3]
- }
-
- @Test
- void testShouldNormalizeContextPathProperty() {
- // Arrange
- String noLeadingSlash = "some/context/path"
- Properties rawProps = new Properties(["nifi.web.proxy.context.path": noLeadingSlash])
- NiFiProperties props = new NiFiProperties(rawProps)
- logger.info("Created a NiFiProperties instance with raw context path property [${noLeadingSlash}]")
-
- // Act
- String normalizedContextPath = props.getAllowedContextPaths()
- logger.info("Read from NiFiProperties instance: ${normalizedContextPath}")
-
- // Assert
- assert normalizedContextPath == "/" + noLeadingSlash
- }
-
- @Test
- void testShouldHandleNormalizedContextPathProperty() {
- // Arrange
- String leadingSlash = "/some/context/path"
- Properties rawProps = new Properties(["nifi.web.proxy.context.path": leadingSlash])
- NiFiProperties props = new NiFiProperties(rawProps)
- logger.info("Created a NiFiProperties instance with raw context path property [${leadingSlash}]")
-
- // Act
- String normalizedContextPath = props.getAllowedContextPaths()
- logger.info("Read from NiFiProperties instance: ${normalizedContextPath}")
-
- // Assert
- assert normalizedContextPath == leadingSlash
- }
-
- @Test
- void testShouldNormalizeMultipleContextPathsInProperty() {
- // Arrange
- String noLeadingSlash = "some/context/path"
- String leadingSlash = "some/other/path"
- String leadingAndTrailingSlash = "/a/third/path/"
- List<String> paths = [noLeadingSlash, leadingSlash, leadingAndTrailingSlash]
- String combinedPaths = paths.join(",")
- Properties rawProps = new Properties(["nifi.web.proxy.context.path": combinedPaths])
- NiFiProperties props = new NiFiProperties(rawProps)
- logger.info("Created a NiFiProperties instance with raw context path property [${noLeadingSlash}]")
-
- // Act
- String normalizedContextPath = props.getAllowedContextPaths()
- logger.info("Read from NiFiProperties instance: ${normalizedContextPath}")
-
- // Assert
- def splitPaths = normalizedContextPath.split(",")
- splitPaths.every {
- assert it.startsWith("/")
- assert !it.endsWith("/")
- }
- }
-
- @Test
- void testShouldHandleNormalizedContextPathPropertyAsList() {
- // Arrange
- String leadingSlash = "/some/context/path"
- Properties rawProps = new Properties(["nifi.web.proxy.context.path": leadingSlash])
- NiFiProperties props = new NiFiProperties(rawProps)
- logger.info("Created a NiFiProperties instance with raw context path property [${leadingSlash}]")
-
- // Act
- def normalizedContextPaths = props.getAllowedContextPathsAsList()
- logger.info("Read from NiFiProperties instance: ${normalizedContextPaths}")
-
- // Assert
- assert normalizedContextPaths.size() == 1
- assert normalizedContextPaths.contains(leadingSlash)
- }
-
- @Test
- void testShouldNormalizeMultipleContextPathsInPropertyAsList() {
- // Arrange
- String noLeadingSlash = "some/context/path"
- String leadingSlash = "/some/other/path"
- String leadingAndTrailingSlash = "/a/third/path/"
- List<String> paths = [noLeadingSlash, leadingSlash, leadingAndTrailingSlash]
- String combinedPaths = paths.join(",")
- Properties rawProps = new Properties(["nifi.web.proxy.context.path": combinedPaths])
- NiFiProperties props = new NiFiProperties(rawProps)
- logger.info("Created a NiFiProperties instance with raw context path property [${noLeadingSlash}]")
-
- // Act
- def normalizedContextPaths = props.getAllowedContextPathsAsList()
- logger.info("Read from NiFiProperties instance: ${normalizedContextPaths}")
-
- // Assert
- assert normalizedContextPaths.size() == 3
- assert normalizedContextPaths.containsAll([leadingSlash, "/" + noLeadingSlash, leadingAndTrailingSlash[0..-2]])
- }
-
- @Test
- void testShouldHandleNormalizingEmptyContextPathProperty() {
- // Arrange
- String empty = ""
- Properties rawProps = new Properties(["nifi.web.proxy.context.path": empty])
- NiFiProperties props = new NiFiProperties(rawProps)
- logger.info("Created a NiFiProperties instance with raw context path property [${empty}]")
-
- // Act
- String normalizedContextPath = props.getAllowedContextPaths()
- logger.info("Read from NiFiProperties instance: ${normalizedContextPath}")
-
- // Assert
- assert normalizedContextPath == empty
- }
-
- @Test
- void testShouldNormalizeProxyHostProperty() {
- // Arrange
- String extraSpaceHostname = "somehost.com "
- Properties rawProps = new Properties(["nifi.web.proxy.host": extraSpaceHostname])
- NiFiProperties props = new NiFiProperties(rawProps)
- logger.info("Created a NiFiProperties instance with raw proxy host property [${extraSpaceHostname}]")
-
- // Act
- String normalizedHostname = props.getAllowedHosts()
- logger.info("Read from NiFiProperties instance: ${normalizedHostname}")
-
- // Assert
- assert extraSpaceHostname.startsWith(normalizedHostname)
- assert extraSpaceHostname.length() == normalizedHostname.length() + 2
- }
-
- @Test
- void testShouldHandleNormalizedProxyHostProperty() {
- // Arrange
- String hostname = "somehost.com"
- Properties rawProps = new Properties(["nifi.web.proxy.host": hostname])
- NiFiProperties props = new NiFiProperties(rawProps)
- logger.info("Created a NiFiProperties instance with raw proxy host property [${hostname}]")
-
- // Act
- String normalizedHostname = props.getAllowedHosts()
- logger.info("Read from NiFiProperties instance: ${normalizedHostname}")
-
- // Assert
- assert hostname == normalizedHostname
- }
-
- @Test
- void testShouldNormalizeMultipleProxyHostsInProperty() {
- // Arrange
- String extraSpaceHostname = "somehost.com "
- String normalHostname = "someotherhost.com"
- String hostnameWithPort = "otherhost.com:1234"
- String extraSpaceHostnameWithPort = " anotherhost.com:9999"
- List<String> hosts = [extraSpaceHostname, normalHostname, hostnameWithPort, extraSpaceHostnameWithPort]
- String combinedHosts = hosts.join(",")
- Properties rawProps = new Properties(["nifi.web.proxy.host": combinedHosts])
- NiFiProperties props = new NiFiProperties(rawProps)
- logger.info("Created a NiFiProperties instance with raw proxy host property [${combinedHosts}]")
-
- // Act
- String normalizedHostname = props.getAllowedHosts()
- logger.info("Read from NiFiProperties instance: ${normalizedHostname}")
-
- // Assert
- def splitHosts = normalizedHostname.split(",")
- def expectedValues = hosts*.trim()
- splitHosts.every {
- assert it.trim() == it
- assert expectedValues.contains(it)
- }
- }
-
- @Test
- void testShouldHandleNormalizedProxyHostPropertyAsList() {
- // Arrange
- String normalHostname = "someotherhost.com"
- Properties rawProps = new Properties(["nifi.web.proxy.host": normalHostname])
- NiFiProperties props = new NiFiProperties(rawProps)
- logger.info("Created a NiFiProperties instance with raw proxy host property [${normalHostname}]")
-
- // Act
- def listOfHosts = props.getAllowedHostsAsList()
- logger.info("Read from NiFiProperties instance: ${listOfHosts}")
-
- // Assert
- assert listOfHosts.size() == 1
- assert listOfHosts.contains(normalHostname)
- }
-
- @Test
- void testShouldNormalizeMultipleProxyHostsInPropertyAsList() {
- // Arrange
- String extraSpaceHostname = "somehost.com "
- String normalHostname = "someotherhost.com"
- String hostnameWithPort = "otherhost.com:1234"
- String extraSpaceHostnameWithPort = " anotherhost.com:9999"
- List<String> hosts = [extraSpaceHostname, normalHostname, hostnameWithPort, extraSpaceHostnameWithPort]
- String combinedHosts = hosts.join(",")
- Properties rawProps = new Properties(["nifi.web.proxy.host": combinedHosts])
- NiFiProperties props = new NiFiProperties(rawProps)
- logger.info("Created a NiFiProperties instance with raw proxy host property [${combinedHosts}]")
-
- // Act
- def listOfHosts = props.getAllowedHostsAsList()
- logger.info("Read from NiFiProperties instance: ${listOfHosts}")
-
- // Assert
- assert listOfHosts.size() == 4
- assert listOfHosts.containsAll([extraSpaceHostname[0..-3], normalHostname, hostnameWithPort, extraSpaceHostnameWithPort[2..-1]])
- }
-
- @Test
- void testShouldHandleNormalizingEmptyProxyHostProperty() {
- // Arrange
- String empty = ""
- Properties rawProps = new Properties(["nifi.web.proxy.host": empty])
- NiFiProperties props = new NiFiProperties(rawProps)
- logger.info("Created a NiFiProperties instance with raw proxy host property [${empty}]")
-
- // Act
- String normalizedHost = props.getAllowedHosts()
- logger.info("Read from NiFiProperties instance: ${normalizedHost}")
-
- // Assert
- assert normalizedHost == empty
- }
-
- @Test
- void testShouldReturnEmptyProxyHostPropertyAsList() {
- // Arrange
- String empty = ""
- Properties rawProps = new Properties(["nifi.web.proxy.host": empty])
- NiFiProperties props = new NiFiProperties(rawProps)
- logger.info("Created a NiFiProperties instance with raw proxy host property [${empty}]")
-
- // Act
- def hosts = props.getAllowedHostsAsList()
- logger.info("Read from NiFiProperties instance: ${hosts}")
-
- // Assert
- assert hosts.size() == 0
- }
-
- @Test
- void testStaticFactoryMethodShouldAcceptRawProperties() throws Exception {
- // Arrange
- Properties rawProperties = new Properties()
- rawProperties.setProperty("key", "value")
- logger.info("rawProperties has ${rawProperties.size()} properties: ${rawProperties.stringPropertyNames()}")
- assert rawProperties.size() == 1
-
- // Act
- NiFiProperties niFiProperties = NiFiProperties.createBasicNiFiProperties("", rawProperties)
- logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
-
- // Assert
- assert niFiProperties.size() == 1
- assert niFiProperties.getPropertyKeys() == ["key"] as Set
- }
-
- @Test
- void testStaticFactoryMethodShouldAcceptMap() throws Exception {
- // Arrange
- def mapProperties = ["key": "value"]
- logger.info("rawProperties has ${mapProperties.size()} properties: ${mapProperties.keySet()}")
- assert mapProperties.size() == 1
-
- // Act
- NiFiProperties niFiProperties = NiFiProperties.createBasicNiFiProperties("", mapProperties)
- logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
-
- // Assert
- assert niFiProperties.size() == 1
- assert niFiProperties.getPropertyKeys() == ["key"] as Set
- }
-
-
- @Test
- void testWebMaxContentSizeShouldDefaultToEmpty() {
- // Arrange
- Properties rawProps = new Properties(["nifi.web.max.content.size": ""])
- NiFiProperties props = new NiFiProperties(rawProps)
- logger.info("Created a NiFiProperties instance with empty web max content size property")
-
- // Act
- String webMaxContentSize = props.getWebMaxContentSize()
- logger.info("Read from NiFiProperties instance: ${webMaxContentSize}")
-
- // Assert
- assert webMaxContentSize == ""
- }
-
-
- @Test
- void testShouldStripWhitespace() throws Exception {
- // Arrange
- File unprotectedFile = new File("src/test/resources/conf/nifi_with_whitespace.properties")
- NiFiPropertiesLoader niFiPropertiesLoader = new NiFiPropertiesLoader()
-
- // Act
- NiFiProperties niFiProperties = niFiPropertiesLoader.load(unprotectedFile.path)
-
- // Assert
- assert niFiProperties.getProperty("nifi.whitespace.propWithNoSpace") == "foo"
- assert niFiProperties.getProperty("nifi.whitespace.propWithLeadingSpace") == "foo"
- assert niFiProperties.getProperty("nifi.whitespace.propWithTrailingSpace") == "foo"
- assert niFiProperties.getProperty("nifi.whitespace.propWithLeadingAndTrailingSpace") == "foo"
- assert niFiProperties.getProperty("nifi.whitespace.propWithTrailingTab") == "foo"
- assert niFiProperties.getProperty("nifi.whitespace.propWithMultipleLines") == "foobarbaz"
- }
-}
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/NiFiPropertiesLoaderGroovyTest.groovy b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/NiFiPropertiesLoaderGroovyTest.groovy
deleted file mode 100644
index 4809923f05..0000000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/NiFiPropertiesLoaderGroovyTest.groovy
+++ /dev/null
@@ -1,491 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.properties
-
-import org.apache.commons.lang3.SystemUtils
-import org.apache.nifi.util.NiFiBootstrapUtils
-import org.apache.nifi.util.NiFiProperties
-import org.apache.nifi.util.file.FileUtils
-import org.bouncycastle.jce.provider.BouncyCastleProvider
-import org.junit.After
-import org.junit.AfterClass
-import org.junit.Assume
-import org.junit.Before
-import org.junit.BeforeClass
-import org.junit.Ignore
-import org.junit.Test
-import org.junit.runner.RunWith
-import org.junit.runners.JUnit4
-import org.slf4j.Logger
-import org.slf4j.LoggerFactory
-
-import javax.crypto.Cipher
-import java.nio.file.Files
-import java.nio.file.attribute.PosixFilePermission
-import java.security.Security
-
-@RunWith(JUnit4.class)
-class NiFiPropertiesLoaderGroovyTest extends GroovyTestCase {
- private static final Logger logger = LoggerFactory.getLogger(NiFiPropertiesLoaderGroovyTest.class)
-
- final def DEFAULT_SENSITIVE_PROPERTIES = [
- "nifi.sensitive.props.key",
- "nifi.security.keystorePasswd",
- "nifi.security.keyPasswd",
- "nifi.security.truststorePasswd"
- ]
-
- final def COMMON_ADDITIONAL_SENSITIVE_PROPERTIES = [
- "nifi.sensitive.props.algorithm",
- "nifi.kerberos.service.principal",
- "nifi.kerberos.krb5.file",
- "nifi.kerberos.keytab.location"
- ]
-
- private static final String KEY_HEX_128 = "0123456789ABCDEFFEDCBA9876543210"
- private static final String KEY_HEX_256 = KEY_HEX_128 * 2
- public static final String KEY_HEX = isUnlimitedStrengthCryptoAvailable() ? KEY_HEX_256 : KEY_HEX_128
-
- private static final String PASSWORD_KEY_HEX_128 = "2C576A9585DB862F5ECBEE5B4FFFCCA1"
-
- private static String originalPropertiesPath = System.getProperty(NiFiProperties.PROPERTIES_FILE_PATH)
- private
- final Set<PosixFilePermission> ownerReadWrite = [PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_READ]
-
- private static boolean isUnlimitedStrengthCryptoAvailable() {
- Cipher.getMaxAllowedKeyLength("AES") > 128
- }
-
- @BeforeClass
- static void setUpOnce() throws Exception {
- Assume.assumeTrue("Test only runs on *nix", !SystemUtils.IS_OS_WINDOWS)
- Security.addProvider(new BouncyCastleProvider())
-
- logger.metaClass.methodMissing = { String name, args ->
- logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}")
- }
- }
-
- @Before
- void setUp() throws Exception {
- }
-
- @After
- void tearDown() throws Exception {
- // Clear the sensitive property providers between runs
-// if (ProtectedNiFiProperties.@localProviderCache) {
-// ProtectedNiFiProperties.@localProviderCache = [:]
-// }
- }
-
- @AfterClass
- static void tearDownOnce() {
- if (originalPropertiesPath) {
- System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, originalPropertiesPath)
- }
- }
-
- @Test
- void testConstructorShouldCreateNewInstance() throws Exception {
- // Arrange
-
- // Act
- NiFiPropertiesLoader niFiPropertiesLoader = new NiFiPropertiesLoader()
-
- // Assert
- assert !niFiPropertiesLoader.@instance
- assert !niFiPropertiesLoader.@keyHex
- }
-
- @Test
- void testShouldCreateInstanceWithKey() throws Exception {
- // Arrange
-
- // Act
- NiFiPropertiesLoader niFiPropertiesLoader = NiFiPropertiesLoader.withKey(KEY_HEX)
-
- // Assert
- assert !niFiPropertiesLoader.@instance
- assert niFiPropertiesLoader.@keyHex == KEY_HEX
- }
-
- @Test
- void testShouldLoadUnprotectedPropertiesFromFile() throws Exception {
- // Arrange
- File unprotectedFile = new File("src/test/resources/conf/nifi.properties")
- NiFiPropertiesLoader niFiPropertiesLoader = new NiFiPropertiesLoader()
-
- // Act
- NiFiProperties niFiProperties = niFiPropertiesLoader.load(unprotectedFile)
-
- // Assert
- assert niFiProperties.size() > 0
-
- // Ensure it is not a ProtectedNiFiProperties
- assert !(niFiProperties instanceof ProtectedNiFiProperties)
- }
-
- @Test
- void testShouldLoadUnprotectedPropertiesFromFileWithoutBootstrap() throws Exception {
- // Arrange
- File unprotectedFile = new File("src/test/resources/conf/nifi.properties")
-
- // Set the system property to the test file
- System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, unprotectedFile.absolutePath)
- logger.info("Set ${NiFiProperties.PROPERTIES_FILE_PATH} to ${unprotectedFile.absolutePath}")
-
- // Act
- NiFiProperties niFiProperties = NiFiPropertiesLoader.loadDefaultWithKeyFromBootstrap()
-
- // Assert
- assert niFiProperties.size() > 0
-
- // Ensure it is not a ProtectedNiFiProperties
- assert !(niFiProperties instanceof ProtectedNiFiProperties)
- }
-
- @Test
- void testShouldNotLoadProtectedPropertiesFromFileWithoutBootstrap() throws Exception {
- // Arrange
- File protectedFile = new File("src/test/resources/conf/nifi_with_sensitive_properties_protected_aes.properties")
-
- // Set the system property to the test file
- System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, protectedFile.absolutePath)
- logger.info("Set ${NiFiProperties.PROPERTIES_FILE_PATH} to ${protectedFile.absolutePath}")
-
- // Act
- def msg = shouldFail(SensitivePropertyProtectionException) {
- NiFiProperties niFiProperties = NiFiPropertiesLoader.loadDefaultWithKeyFromBootstrap()
- }
- logger.expected(msg)
-
- // Assert
- assert msg =~ "Could not read root key from bootstrap.conf"
- }
-
- @Test
- void testShouldLoadUnprotectedPropertiesFromPathWithGeneratedSensitivePropertiesKey() throws Exception {
- // Arrange
- final File propertiesFile = File.createTempFile("nifi.without.key", ".properties")
- propertiesFile.deleteOnExit()
- final OutputStream outputStream = new FileOutputStream(propertiesFile)
- final InputStream inputStream = getClass().getResourceAsStream("/conf/nifi.without.key.properties")
- FileUtils.copy(inputStream, outputStream)
-
- System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, propertiesFile.absolutePath);
- NiFiPropertiesLoader niFiPropertiesLoader = new NiFiPropertiesLoader()
-
- // Act
- NiFiProperties niFiProperties = niFiPropertiesLoader.get()
-
- // Assert
- final String sensitivePropertiesKey = niFiProperties.getProperty(NiFiProperties.SENSITIVE_PROPS_KEY)
- assert sensitivePropertiesKey.length() == 32
- }
-
- @Test
- void testShouldNotLoadUnprotectedPropertiesFromPathWithBlankKeyForClusterNode() throws Exception {
- // Arrange
- final File propertiesFile = File.createTempFile("nifi.without.key", ".properties")
- propertiesFile.deleteOnExit()
- final OutputStream outputStream = new FileOutputStream(propertiesFile)
- final InputStream inputStream = getClass().getResourceAsStream("/conf/nifi.cluster.without.key.properties")
- FileUtils.copy(inputStream, outputStream)
-
- System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, propertiesFile.absolutePath);
- NiFiPropertiesLoader niFiPropertiesLoader = new NiFiPropertiesLoader()
-
- // Act
- shouldFail(SensitivePropertyProtectionException) {
- niFiPropertiesLoader.get()
- }
- }
-
- @Test
- void testShouldNotLoadUnprotectedPropertiesFromNullFile() throws Exception {
- // Arrange
- NiFiPropertiesLoader niFiPropertiesLoader = new NiFiPropertiesLoader()
-
- // Act
- def msg = shouldFail(IllegalArgumentException) {
- NiFiProperties niFiProperties = niFiPropertiesLoader.load(null as File)
- }
- logger.expected(msg)
-
- // Assert
- assert msg == "NiFi properties file missing or unreadable"
- }
-
- @Test
- void testShouldNotLoadUnprotectedPropertiesFromMissingFile() throws Exception {
- // Arrange
- File missingFile = new File("src/test/resources/conf/nifi_missing.properties")
- assert !missingFile.exists()
-
- NiFiPropertiesLoader niFiPropertiesLoader = new NiFiPropertiesLoader()
-
- // Act
- def msg = shouldFail(IllegalArgumentException) {
- NiFiProperties niFiProperties = niFiPropertiesLoader.load(missingFile)
- }
- logger.expected(msg)
-
- // Assert
- assert msg == "NiFi properties file missing or unreadable"
- }
-
- @Test
- void testShouldNotLoadUnprotectedPropertiesFromUnreadableFile() throws Exception {
- // Arrange
- File unreadableFile = new File("src/test/resources/conf/nifi_no_permissions.properties")
- Files.setPosixFilePermissions(unreadableFile.toPath(), [] as Set)
- assert !unreadableFile.canRead()
-
- NiFiPropertiesLoader niFiPropertiesLoader = new NiFiPropertiesLoader()
-
- // Act
- def msg = shouldFail(IllegalArgumentException) {
- NiFiProperties niFiProperties = niFiPropertiesLoader.load(unreadableFile)
- }
- logger.expected(msg)
-
- // Assert
- assert msg == "NiFi properties file missing or unreadable"
-
- // Clean up to allow for indexing, etc.
- Files.setPosixFilePermissions(unreadableFile.toPath(), ownerReadWrite)
- }
-
- @Test
- void testShouldLoadUnprotectedPropertiesFromPath() throws Exception {
- // Arrange
- File unprotectedFile = new File("src/test/resources/conf/nifi.properties")
- NiFiPropertiesLoader niFiPropertiesLoader = new NiFiPropertiesLoader()
-
- // Act
- NiFiProperties niFiProperties = niFiPropertiesLoader.load(unprotectedFile.path)
-
- // Assert
- assert niFiProperties.size() > 0
-
- // Ensure it is not a ProtectedNiFiProperties
- assert !(niFiProperties instanceof ProtectedNiFiProperties)
- }
-
- @Test
- void testShouldLoadUnprotectedPropertiesFromProtectedFile() throws Exception {
- // Arrange
- File protectedFile = new File("src/test/resources/conf/nifi_with_sensitive_properties_protected_aes.properties")
-
- System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, protectedFile.path)
- NiFiPropertiesLoader niFiPropertiesLoader = NiFiPropertiesLoader.withKey(KEY_HEX)
-
- final def EXPECTED_PLAIN_VALUES = [
- (NiFiProperties.SENSITIVE_PROPS_KEY): "thisIsABadSensitiveKeyPassword",
- (NiFiProperties.SECURITY_KEYSTORE_PASSWD): "thisIsABadKeystorePassword",
- (NiFiProperties.SECURITY_KEY_PASSWD): "thisIsABadKeyPassword",
- ]
-
- // This method is covered in tests above, so safe to use here to retrieve protected properties
- ProtectedNiFiProperties protectedNiFiProperties = niFiPropertiesLoader.readProtectedPropertiesFromDisk(protectedFile)
- int totalKeysCount = protectedNiFiProperties.getPropertyKeysIncludingProtectionSchemes().size()
- int protectedKeysCount = protectedNiFiProperties.getProtectedPropertyKeys().size()
- logger.info("Read ${totalKeysCount} total properties (${protectedKeysCount} protected) from ${protectedFile.canonicalPath}")
-
- // Act
- NiFiProperties niFiProperties = niFiPropertiesLoader.load(protectedFile)
-
- // Assert
- assert niFiProperties.size() == totalKeysCount - protectedKeysCount
-
- // Ensure that any key marked as protected above is different in this instance
- protectedNiFiProperties.getProtectedPropertyKeys().keySet().each { String key ->
- String plainValue = niFiProperties.getProperty(key)
- String protectedValue = protectedNiFiProperties.getProperty(key)
-
- logger.info("Checking that [${protectedValue}] -> [${plainValue}] == [${EXPECTED_PLAIN_VALUES[key]}]")
-
- assert plainValue == EXPECTED_PLAIN_VALUES[key]
- assert plainValue != protectedValue
- assert plainValue.length() <= protectedValue.length()
- }
-
- // Ensure it is not a ProtectedNiFiProperties
- assert !(niFiProperties instanceof ProtectedNiFiProperties)
- }
-
- @Test
- void testShouldExtractKeyFromBootstrapFile() throws Exception {
- // Arrange
- def defaultNiFiPropertiesFilePath = "src/test/resources/bootstrap_tests/conf/nifi.properties"
- System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, defaultNiFiPropertiesFilePath)
-
- // Act
- String key = NiFiBootstrapUtils.extractKeyFromBootstrapFile()
-
- // Assert
- assert key == KEY_HEX
- }
-
- @Test
- void testShouldNotExtractKeyFromBootstrapFileWithoutKeyLine() throws Exception {
- // Arrange
- def defaultNiFiPropertiesFilePath = "src/test/resources/bootstrap_tests/missing_key_line/nifi.properties"
- System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, defaultNiFiPropertiesFilePath)
-
- // Act
- String key = NiFiBootstrapUtils.extractKeyFromBootstrapFile()
-
- // Assert
- assert key == ""
- }
-
- @Test
- void testShouldNotExtractKeyFromBootstrapFileWithoutKey() throws Exception {
- // Arrange
- def defaultNiFiPropertiesFilePath = "src/test/resources/bootstrap_tests/missing_key_line/nifi.properties"
- System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, defaultNiFiPropertiesFilePath)
-
- // Act
- String key = NiFiBootstrapUtils.extractKeyFromBootstrapFile()
-
- // Assert
- assert key == ""
- }
-
- @Test
- void testShouldNotExtractKeyFromMissingBootstrapFile() throws Exception {
- // Arrange
- def defaultNiFiPropertiesFilePath = "src/test/resources/bootstrap_tests/missing_bootstrap/nifi.properties"
- System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, defaultNiFiPropertiesFilePath)
-
- // Act
- def msg = shouldFail(IOException) {
- String key = NiFiBootstrapUtils.extractKeyFromBootstrapFile()
- }
- logger.expected(msg)
-
- // Assert
- assert msg =~ "Cannot read from .*bootstrap.conf"
- }
-
- @Test
- void testShouldNotExtractKeyFromUnreadableBootstrapFile() throws Exception {
- // Arrange
- File unreadableFile = new File("src/test/resources/bootstrap_tests/unreadable_bootstrap/bootstrap.conf")
- Set<PosixFilePermission> originalPermissions = Files.getPosixFilePermissions(unreadableFile.toPath())
- Files.setPosixFilePermissions(unreadableFile.toPath(), [] as Set)
- assert !unreadableFile.canRead()
-
- def defaultNiFiPropertiesFilePath = "src/test/resources/bootstrap_tests/unreadable_bootstrap/nifi.properties"
- System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, defaultNiFiPropertiesFilePath)
-
- // Act
- def msg = shouldFail(IOException) {
- String key = NiFiBootstrapUtils.extractKeyFromBootstrapFile()
- }
- logger.expected(msg)
-
- // Assert
- assert msg =~ "Cannot read from .*bootstrap.conf"
-
- // Clean up to allow for indexing, etc.
- Files.setPosixFilePermissions(unreadableFile.toPath(), originalPermissions)
- }
-
- @Ignore("Unreadable conf directory breaks build")
- @Test
- void testShouldNotExtractKeyFromUnreadableConfDir() throws Exception {
- // Arrange
- File unreadableDir = new File("src/test/resources/bootstrap_tests/unreadable_conf")
- Set<PosixFilePermission> originalPermissions = Files.getPosixFilePermissions(unreadableDir.toPath())
- Files.setPosixFilePermissions(unreadableDir.toPath(), [] as Set)
- assert !unreadableDir.canRead()
-
- def defaultNiFiPropertiesFilePath = "src/test/resources/bootstrap_tests/unreadable_conf/nifi.properties"
- System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, defaultNiFiPropertiesFilePath)
-
- // Act
- def msg = shouldFail(IOException) {
- String key = NiFiBootstrapUtils.extractKeyFromBootstrapFile()
- }
- logger.expected(msg)
-
- // Assert
- assert msg == "Cannot read from bootstrap.conf"
-
- // Clean up to allow for indexing, etc.
- Files.setPosixFilePermissions(unreadableDir.toPath(), originalPermissions)
- }
-
- @Test
- void testShouldLoadUnprotectedPropertiesFromProtectedDefaultFileAndUseBootstrapKey() throws Exception {
- // Arrange
- File protectedFile = new File("src/test/resources/bootstrap_tests/conf/nifi_with_sensitive_properties_protected_aes.properties")
- System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, protectedFile.path)
- NiFiPropertiesLoader niFiPropertiesLoader = NiFiPropertiesLoader.withKey(KEY_HEX)
-
- NiFiProperties normalReadProperties = niFiPropertiesLoader.load(protectedFile)
- logger.info("Read ${normalReadProperties.size()} total properties from ${protectedFile.canonicalPath}")
-
- // Act
- NiFiProperties niFiProperties = NiFiPropertiesLoader.loadDefaultWithKeyFromBootstrap()
-
- // Assert
- assert niFiProperties.size() == normalReadProperties.size()
-
-
- def readPropertiesAndValues = niFiProperties.getPropertyKeys().collectEntries {
- [(it): niFiProperties.getProperty(it)]
- }
- def expectedPropertiesAndValues = normalReadProperties.getPropertyKeys().collectEntries {
- [(it): normalReadProperties.getProperty(it)]
- }
- assert readPropertiesAndValues == expectedPropertiesAndValues
- }
-
- @Test
- void testShouldUpdateKeyInFactory() throws Exception {
- // Arrange
- File originalKeyFile = new File("src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_128.properties")
- File passwordKeyFile = new File("src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_128_password.properties")
- System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, originalKeyFile.path)
- NiFiPropertiesLoader niFiPropertiesLoader = NiFiPropertiesLoader.withKey(KEY_HEX_128)
-
- NiFiProperties niFiProperties = niFiPropertiesLoader.load(originalKeyFile)
- logger.info("Read ${niFiProperties.size()} total properties from ${originalKeyFile.canonicalPath}")
-
- // Act
- NiFiPropertiesLoader passwordNiFiPropertiesLoader = NiFiPropertiesLoader.withKey(PASSWORD_KEY_HEX_128)
-
- NiFiProperties passwordProperties = passwordNiFiPropertiesLoader.load(passwordKeyFile)
- logger.info("Read ${passwordProperties.size()} total properties from ${passwordKeyFile.canonicalPath}")
-
- // Assert
- assert niFiProperties.size() == passwordProperties.size()
-
-
- def readPropertiesAndValues = niFiProperties.getPropertyKeys().collectEntries {
- [(it): niFiProperties.getProperty(it)]
- }
- def readPasswordPropertiesAndValues = passwordProperties.getPropertyKeys().collectEntries {
- [(it): passwordProperties.getProperty(it)]
- }
-
- assert readPropertiesAndValues == readPasswordPropertiesAndValues
- }
-}
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/java/org/apache/nifi/properties/NiFiPropertiesLoaderTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/java/org/apache/nifi/properties/NiFiPropertiesLoaderTest.java
new file mode 100644
index 0000000000..a87ef218af
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/java/org/apache/nifi/properties/NiFiPropertiesLoaderTest.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.properties;
+
+import org.apache.nifi.util.NiFiProperties;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+
+import java.net.URL;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+class NiFiPropertiesLoaderTest {
+ private static final String NULL_PATH = null;
+
+ private static final String EMPTY_PATH = "/properties/conf/empty.nifi.properties";
+
+ private static final String FLOW_PATH = "/properties/conf/flow.nifi.properties";
+
+ private static final String PROTECTED_PATH = "/properties/conf/protected.nifi.properties";
+
+ private static final String HEXADECIMAL_KEY = "12345678123456788765432187654321";
+
+ private static final String EXPECTED_PASSWORD = "propertyValue";
+
+ @AfterEach
+ void clearSystemProperty() {
+ System.clearProperty(NiFiProperties.PROPERTIES_FILE_PATH);
+ }
+
+ @Test
+ void testGetPropertiesNotFound() {
+ final NiFiPropertiesLoader loader = new NiFiPropertiesLoader();
+
+ assertThrows(IllegalArgumentException.class, loader::get);
+ }
+
+ @Test
+ void testGetProperties() {
+ final URL resource = NiFiPropertiesLoaderTest.class.getResource(FLOW_PATH);
+ assertNotNull(resource);
+
+ final String path = resource.getPath();
+
+ System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, path);
+
+ final NiFiPropertiesLoader loader = new NiFiPropertiesLoader();
+
+ final NiFiProperties properties = loader.get();
+
+ assertNotNull(properties);
+ assertNotNull(properties.getFlowConfigurationFile());
+ }
+
+ @Test
+ void testGetPropertiesWithKeyNoEncryptedProperties() {
+ final URL resource = NiFiPropertiesLoaderTest.class.getResource(FLOW_PATH);
+ assertNotNull(resource);
+
+ final String path = resource.getPath();
+
+ System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, path);
+
+ final NiFiPropertiesLoader loader = NiFiPropertiesLoader.withKey(String.class.getSimpleName());
+
+ final NiFiProperties properties = loader.get();
+
+ assertNotNull(properties);
+ assertNotNull(properties.getFlowConfigurationFile());
+ }
+
+ @Test
+ void testLoadWithKey() {
+ final NiFiPropertiesLoader loader = NiFiPropertiesLoader.withKey(HEXADECIMAL_KEY);
+
+ final URL resource = NiFiPropertiesLoaderTest.class.getResource(PROTECTED_PATH);
+ assertNotNull(resource);
+ final String path = resource.getPath();
+
+ final NiFiProperties properties = loader.load(path);
+
+ assertNotNull(properties);
+
+ assertNotNull(properties.getFlowConfigurationFile());
+ assertEquals(EXPECTED_PASSWORD, properties.getProperty(NiFiProperties.SECURITY_KEYSTORE_PASSWD));
+ }
+
+ @Test
+ void testLoadWithDefaultKeyFromBootstrap() {
+ final URL resource = NiFiPropertiesLoaderTest.class.getResource(FLOW_PATH);
+ assertNotNull(resource);
+
+ final String path = resource.getPath();
+
+ System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, path);
+
+ final NiFiProperties properties = NiFiPropertiesLoader.loadDefaultWithKeyFromBootstrap();
+
+ assertNotNull(properties);
+ }
+
+ @Test
+ void testLoadDefaultNotFound() {
+ final NiFiPropertiesLoader loader = new NiFiPropertiesLoader();
+
+ assertThrows(IllegalArgumentException.class, () -> loader.load(NULL_PATH));
+ }
+
+ @Test
+ void testLoadPath() {
+ final NiFiPropertiesLoader loader = new NiFiPropertiesLoader();
+
+ final URL resource = NiFiPropertiesLoaderTest.class.getResource(EMPTY_PATH);
+ assertNotNull(resource);
+
+ final String path = resource.getPath();
+
+ final NiFiProperties properties = loader.load(path);
+
+ assertNotNull(properties);
+ }
+}
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/conf/bootstrap.conf b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/conf/bootstrap.conf
deleted file mode 100644
index d02b43f7a3..0000000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/conf/bootstrap.conf
+++ /dev/null
@@ -1,74 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Java command to use when running NiFi
-java=java
-
-# Username to use when running NiFi. This value will be ignored on Windows.
-run.as=
-
-# Configure where NiFi's lib and conf directories live
-lib.dir=./lib
-conf.dir=./conf
-
-# How long to wait after telling NiFi to shutdown before explicitly killing the Process
-graceful.shutdown.seconds=20
-
-# Disable JSR 199 so that we can use JSP's without running a JDK
-java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true
-
-# JVM memory settings
-java.arg.2=-Xms512m
-java.arg.3=-Xmx512m
-
-# Enable Remote Debugging
-#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000
-
-java.arg.4=-Djava.net.preferIPv4Stack=true
-
-# allowRestrictedHeaders is required for Cluster/Node communications to work properly
-java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true
-java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol
-
-# The G1GC is still considered experimental but has proven to be very advantageous in providing great
-# performance without significant "stop-the-world" delays.
-java.arg.13=-XX:+UseG1GC
-
-#Set headless mode by default
-java.arg.14=-Djava.awt.headless=true
-
-# Root key in hexadecimal format for encrypted sensitive configuration values
-nifi.bootstrap.sensitive.key=0123456789ABCDEFFEDCBA98765432100123456789ABCDEFFEDCBA9876543210
-
-###
-# Notification Services for notifying interested parties when NiFi is stopped, started, dies
-###
-
-# XML File that contains the definitions of the notification services
-notification.services.file=./conf/bootstrap-notification-services.xml
-
-# In the case that we are unable to send a notification for an event, how many times should we retry?
-notification.max.attempts=5
-
-# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is started?
-#nifi.start.notification.services=email-notification
-
-# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is stopped?
-#nifi.stop.notification.services=email-notification
-
-# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi dies?
-#nifi.dead.notification.services=email-notification
\ No newline at end of file
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/conf/nifi_with_sensitive_properties_protected_aes.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/conf/nifi_with_sensitive_properties_protected_aes.properties
deleted file mode 100644
index 6e3ea7afea..0000000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/conf/nifi_with_sensitive_properties_protected_aes.properties
+++ /dev/null
@@ -1,126 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Core Properties #
-nifi.flow.configuration.file=./target/flow.xml.gz
-nifi.flow.configuration.archive.dir=./target/archive/
-nifi.flowcontroller.autoResumeState=true
-nifi.flowcontroller.graceful.shutdown.period=10 sec
-nifi.flowservice.writedelay.interval=2 sec
-nifi.administrative.yield.duration=30 sec
-
-nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
-nifi.controller.service.configuration.file=./target/controller-services.xml
-nifi.templates.directory=./target/templates
-nifi.ui.banner.text=UI Banner Text
-nifi.ui.autorefresh.interval=30 sec
-nifi.nar.library.directory=./target/resources/NiFiProperties/lib/
-nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/
-nifi.nar.working.directory=./target/work/nar/
-
-# H2 Settings
-nifi.database.directory=./target/database_repository
-nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
-
-# FlowFile Repository
-nifi.flowfile.repository.directory=./target/test-repo
-nifi.flowfile.repository.partitions=1
-nifi.flowfile.repository.checkpoint.interval=2 mins
-nifi.queue.swap.threshold=20000
-nifi.swap.storage.directory=./target/test-repo/swap
-nifi.swap.in.period=5 sec
-nifi.swap.in.threads=1
-nifi.swap.out.period=5 sec
-nifi.swap.out.threads=4
-
-# Content Repository
-nifi.content.claim.max.appendable.size=10 MB
-nifi.content.claim.max.flow.files=100
-nifi.content.repository.directory.default=./target/content_repository
-
-# Provenance Repository Properties
-nifi.provenance.repository.storage.directory=./target/provenance_repository
-nifi.provenance.repository.max.storage.time=24 hours
-nifi.provenance.repository.max.storage.size=1 GB
-nifi.provenance.repository.rollover.time=30 secs
-nifi.provenance.repository.rollover.size=100 MB
-
-# Site to Site properties
-nifi.remote.input.socket.port=9990
-nifi.remote.input.secure=true
-
-# web properties #
-nifi.web.war.directory=./target/lib
-nifi.web.http.host=
-nifi.web.http.port=
-nifi.web.https.host=nifi.nifi.apache.org
-nifi.web.https.port=8443
-nifi.web.jetty.working.directory=./target/work/jetty
-
-# security properties #
-nifi.sensitive.props.key=n2z+tTTbHuZ4V4V2||uWhdasyDXD4ZG2lMAes/vqh6u4vaz4xgL4aEbF4Y/dXevqk3ulRcOwf1vc4RDQ==
-nifi.sensitive.props.key.protected=aes/gcm/256
-nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
-nifi.sensitive.props.additional.keys=
-
-nifi.security.keystore=/path/to/keystore.jks
-nifi.security.keystoreType=JKS
-nifi.security.keystorePasswd=oBjT92hIGRElIGOh||MZ6uYuWNBrOA6usq/Jt3DaD2e4otNirZDytac/w/KFe0HOkrJR03vcbo
-nifi.security.keystorePasswd.protected=aes/gcm/256
-nifi.security.keyPasswd=ac/BaE35SL/esLiJ||+ULRvRLYdIDA2VqpE0eQXDEMjaLBMG2kbKOdOwBk/hGebDKlVg==
-nifi.security.keyPasswd.protected=aes/gcm/256
-nifi.security.truststore=
-nifi.security.truststoreType=
-nifi.security.truststorePasswd=/X/RSlNr2QCJ1Kwe||dENJevX5P61ix+97airrtoBQoyasMFS6DG6fHbX+SZtw2VAMllSSnDeT97Q=
-nifi.security.truststorePasswd.protected=aes/gcm/256
-nifi.security.user.authorizer=
-
-# cluster common properties (cluster manager and nodes must have same values) #
-nifi.cluster.protocol.heartbeat.interval=5 sec
-nifi.cluster.protocol.is.secure=false
-nifi.cluster.protocol.socket.timeout=30 sec
-nifi.cluster.protocol.connection.handshake.timeout=45 sec
-# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured #
-nifi.cluster.protocol.use.multicast=false
-nifi.cluster.protocol.multicast.address=
-nifi.cluster.protocol.multicast.port=
-nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms
-nifi.cluster.protocol.multicast.service.locator.attempts=3
-nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec
-
-# cluster node properties (only configure for cluster nodes) #
-nifi.cluster.is.node=false
-nifi.cluster.node.address=
-nifi.cluster.node.protocol.port=
-nifi.cluster.node.protocol.threads=2
-# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx #
-nifi.cluster.node.unicast.manager.address=
-nifi.cluster.node.unicast.manager.protocol.port=
-nifi.cluster.node.unicast.manager.authority.provider.port=
-
-# cluster manager properties (only configure for cluster manager) #
-nifi.cluster.is.manager=false
-nifi.cluster.manager.address=
-nifi.cluster.manager.protocol.port=
-nifi.cluster.manager.authority.provider.port=
-nifi.cluster.manager.authority.provider.threads=10
-nifi.cluster.manager.node.firewall.file=
-nifi.cluster.manager.node.event.history.size=10
-nifi.cluster.manager.node.api.connection.timeout=30 sec
-nifi.cluster.manager.node.api.read.timeout=30 sec
-nifi.cluster.manager.node.api.request.threads=10
-nifi.cluster.manager.flow.retrieval.delay=5 sec
-nifi.cluster.manager.protocol.threads=10
-nifi.cluster.manager.safemode.duration=0 sec
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key/bootstrap.conf b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key/bootstrap.conf
deleted file mode 100644
index 601e6c9f7d..0000000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key/bootstrap.conf
+++ /dev/null
@@ -1,74 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Java command to use when running NiFi
-java=java
-
-# Username to use when running NiFi. This value will be ignored on Windows.
-run.as=
-
-# Configure where NiFi's lib and conf directories live
-lib.dir=./lib
-conf.dir=./conf
-
-# How long to wait after telling NiFi to shutdown before explicitly killing the Process
-graceful.shutdown.seconds=20
-
-# Disable JSR 199 so that we can use JSP's without running a JDK
-java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true
-
-# JVM memory settings
-java.arg.2=-Xms512m
-java.arg.3=-Xmx512m
-
-# Enable Remote Debugging
-#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000
-
-java.arg.4=-Djava.net.preferIPv4Stack=true
-
-# allowRestrictedHeaders is required for Cluster/Node communications to work properly
-java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true
-java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol
-
-# The G1GC is still considered experimental but has proven to be very advantageous in providing great
-# performance without significant "stop-the-world" delays.
-java.arg.13=-XX:+UseG1GC
-
-#Set headless mode by default
-java.arg.14=-Djava.awt.headless=true
-
-# Root key in hexadecimal format for encrypted sensitive configuration values
-nifi.bootstrap.sensitive.key=
-
-###
-# Notification Services for notifying interested parties when NiFi is stopped, started, dies
-###
-
-# XML File that contains the definitions of the notification services
-notification.services.file=./conf/bootstrap-notification-services.xml
-
-# In the case that we are unable to send a notification for an event, how many times should we retry?
-notification.max.attempts=5
-
-# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is started?
-#nifi.start.notification.services=email-notification
-
-# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is stopped?
-#nifi.stop.notification.services=email-notification
-
-# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi dies?
-#nifi.dead.notification.services=email-notification
\ No newline at end of file
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key_line/bootstrap.conf b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key_line/bootstrap.conf
deleted file mode 100644
index fc917c743f..0000000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key_line/bootstrap.conf
+++ /dev/null
@@ -1,71 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Java command to use when running NiFi
-java=java
-
-# Username to use when running NiFi. This value will be ignored on Windows.
-run.as=
-
-# Configure where NiFi's lib and conf directories live
-lib.dir=./lib
-conf.dir=./conf
-
-# How long to wait after telling NiFi to shutdown before explicitly killing the Process
-graceful.shutdown.seconds=20
-
-# Disable JSR 199 so that we can use JSP's without running a JDK
-java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true
-
-# JVM memory settings
-java.arg.2=-Xms512m
-java.arg.3=-Xmx512m
-
-# Enable Remote Debugging
-#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000
-
-java.arg.4=-Djava.net.preferIPv4Stack=true
-
-# allowRestrictedHeaders is required for Cluster/Node communications to work properly
-java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true
-java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol
-
-# The G1GC is still considered experimental but has proven to be very advantageous in providing great
-# performance without significant "stop-the-world" delays.
-java.arg.13=-XX:+UseG1GC
-
-#Set headless mode by default
-java.arg.14=-Djava.awt.headless=true
-
-###
-# Notification Services for notifying interested parties when NiFi is stopped, started, dies
-###
-
-# XML File that contains the definitions of the notification services
-notification.services.file=./conf/bootstrap-notification-services.xml
-
-# In the case that we are unable to send a notification for an event, how many times should we retry?
-notification.max.attempts=5
-
-# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is started?
-#nifi.start.notification.services=email-notification
-
-# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is stopped?
-#nifi.stop.notification.services=email-notification
-
-# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi dies?
-#nifi.dead.notification.services=email-notification
\ No newline at end of file
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/unreadable_bootstrap/bootstrap.conf b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/unreadable_bootstrap/bootstrap.conf
deleted file mode 100644
index d02b43f7a3..0000000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/unreadable_bootstrap/bootstrap.conf
+++ /dev/null
@@ -1,74 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Java command to use when running NiFi
-java=java
-
-# Username to use when running NiFi. This value will be ignored on Windows.
-run.as=
-
-# Configure where NiFi's lib and conf directories live
-lib.dir=./lib
-conf.dir=./conf
-
-# How long to wait after telling NiFi to shutdown before explicitly killing the Process
-graceful.shutdown.seconds=20
-
-# Disable JSR 199 so that we can use JSP's without running a JDK
-java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true
-
-# JVM memory settings
-java.arg.2=-Xms512m
-java.arg.3=-Xmx512m
-
-# Enable Remote Debugging
-#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000
-
-java.arg.4=-Djava.net.preferIPv4Stack=true
-
-# allowRestrictedHeaders is required for Cluster/Node communications to work properly
-java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true
-java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol
-
-# The G1GC is still considered experimental but has proven to be very advantageous in providing great
-# performance without significant "stop-the-world" delays.
-java.arg.13=-XX:+UseG1GC
-
-#Set headless mode by default
-java.arg.14=-Djava.awt.headless=true
-
-# Root key in hexadecimal format for encrypted sensitive configuration values
-nifi.bootstrap.sensitive.key=0123456789ABCDEFFEDCBA98765432100123456789ABCDEFFEDCBA9876543210
-
-###
-# Notification Services for notifying interested parties when NiFi is stopped, started, dies
-###
-
-# XML File that contains the definitions of the notification services
-notification.services.file=./conf/bootstrap-notification-services.xml
-
-# In the case that we are unable to send a notification for an event, how many times should we retry?
-notification.max.attempts=5
-
-# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is started?
-#nifi.start.notification.services=email-notification
-
-# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is stopped?
-#nifi.stop.notification.services=email-notification
-
-# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi dies?
-#nifi.dead.notification.services=email-notification
\ No newline at end of file
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.blank.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.blank.properties
deleted file mode 100644
index 44fac47ffa..0000000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.blank.properties
+++ /dev/null
@@ -1,121 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Core Properties #
-nifi.flow.configuration.file=./target/flow.xml.gz
-nifi.flow.configuration.archive.dir=./target/archive/
-nifi.flowcontroller.autoResumeState=true
-nifi.flowcontroller.graceful.shutdown.period=10 sec
-nifi.flowservice.writedelay.interval=2 sec
-nifi.administrative.yield.duration=30 sec
-
-nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
-nifi.controller.service.configuration.file=./target/controller-services.xml
-nifi.templates.directory=./target/templates
-nifi.ui.banner.text=UI Banner Text
-nifi.ui.autorefresh.interval=30 sec
-nifi.nar.library.directory=
-nifi.custom.nar.library.directory.alt=
-nifi.nar.working.directory=./target/work/nar/
-
-# H2 Settings
-nifi.database.directory=./target/database_repository
-nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
-
-# FlowFile Repository
-nifi.flowfile.repository.directory=./target/test-repo
-nifi.flowfile.repository.partitions=1
-nifi.flowfile.repository.checkpoint.interval=2 mins
-nifi.queue.swap.threshold=20000
-nifi.swap.storage.directory=./target/test-repo/swap
-nifi.swap.in.period=5 sec
-nifi.swap.in.threads=1
-nifi.swap.out.period=5 sec
-nifi.swap.out.threads=4
-
-# Content Repository
-nifi.content.claim.max.appendable.size=10 MB
-nifi.content.claim.max.flow.files=100
-nifi.content.repository.directory.default=./target/content_repository
-
-# Provenance Repository Properties
-nifi.provenance.repository.storage.directory=./target/provenance_repository
-nifi.provenance.repository.max.storage.time=24 hours
-nifi.provenance.repository.max.storage.size=1 GB
-nifi.provenance.repository.rollover.time=30 secs
-nifi.provenance.repository.rollover.size=100 MB
-
-# Site to Site properties
-nifi.remote.input.socket.port=9990
-nifi.remote.input.secure=true
-
-# web properties #
-nifi.web.war.directory=./target/lib
-nifi.web.http.host=
-nifi.web.http.port=8080
-nifi.web.https.host=
-nifi.web.https.port=
-nifi.web.jetty.working.directory=./target/work/jetty
-
-# security properties #
-nifi.sensitive.props.key=key
-nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
-
-nifi.security.keystore=
-nifi.security.keystoreType=
-nifi.security.keystorePasswd=
-nifi.security.keyPasswd=
-nifi.security.truststore=
-nifi.security.truststoreType=
-nifi.security.truststorePasswd=
-nifi.security.user.authorizer=
-
-# cluster common properties (cluster manager and nodes must have same values) #
-nifi.cluster.protocol.heartbeat.interval=5 sec
-nifi.cluster.protocol.is.secure=false
-nifi.cluster.protocol.socket.timeout=30 sec
-nifi.cluster.protocol.connection.handshake.timeout=45 sec
-# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured #
-nifi.cluster.protocol.use.multicast=false
-nifi.cluster.protocol.multicast.address=
-nifi.cluster.protocol.multicast.port=
-nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms
-nifi.cluster.protocol.multicast.service.locator.attempts=3
-nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec
-
-# cluster node properties (only configure for cluster nodes) #
-nifi.cluster.is.node=false
-nifi.cluster.node.address=
-nifi.cluster.node.protocol.port=
-nifi.cluster.node.protocol.threads=2
-# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx #
-nifi.cluster.node.unicast.manager.address=
-nifi.cluster.node.unicast.manager.protocol.port=
-nifi.cluster.node.unicast.manager.authority.provider.port=
-
-# cluster manager properties (only configure for cluster manager) #
-nifi.cluster.is.manager=false
-nifi.cluster.manager.address=
-nifi.cluster.manager.protocol.port=
-nifi.cluster.manager.authority.provider.port=
-nifi.cluster.manager.authority.provider.threads=10
-nifi.cluster.manager.node.firewall.file=
-nifi.cluster.manager.node.event.history.size=10
-nifi.cluster.manager.node.api.connection.timeout=30 sec
-nifi.cluster.manager.node.api.read.timeout=30 sec
-nifi.cluster.manager.node.api.request.threads=10
-nifi.cluster.manager.flow.retrieval.delay=5 sec
-nifi.cluster.manager.protocol.threads=10
-nifi.cluster.manager.safemode.duration=0 sec
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.cluster.without.key.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.cluster.without.key.properties
deleted file mode 100644
index 681dfc7620..0000000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.cluster.without.key.properties
+++ /dev/null
@@ -1,94 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Core Properties #
-nifi.flow.configuration.file=./target/flow.xml.gz
-nifi.flow.configuration.archive.dir=./target/archive/
-nifi.flowcontroller.autoResumeState=true
-nifi.flowcontroller.graceful.shutdown.period=10 sec
-nifi.flowservice.writedelay.interval=2 sec
-nifi.administrative.yield.duration=30 sec
-
-nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
-nifi.controller.service.configuration.file=./target/controller-services.xml
-nifi.templates.directory=./target/templates
-nifi.ui.banner.text=UI Banner Text
-nifi.ui.autorefresh.interval=30 sec
-nifi.nar.library.directory=./target/resources/NiFiProperties/lib/
-nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/
-nifi.nar.working.directory=./target/work/nar/
-
-# H2 Settings
-nifi.database.directory=./target/database_repository
-nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
-
-# FlowFile Repository
-nifi.flowfile.repository.directory=./target/test-repo
-nifi.flowfile.repository.partitions=1
-nifi.flowfile.repository.checkpoint.interval=2 mins
-nifi.queue.swap.threshold=20000
-nifi.swap.storage.directory=./target/test-repo/swap
-nifi.swap.in.period=5 sec
-nifi.swap.in.threads=1
-nifi.swap.out.period=5 sec
-nifi.swap.out.threads=4
-
-# Content Repository
-nifi.content.claim.max.appendable.size=10 MB
-nifi.content.claim.max.flow.files=100
-nifi.content.repository.directory.default=./target/content_repository
-
-# Provenance Repository Properties
-nifi.provenance.repository.storage.directory=./target/provenance_repository
-nifi.provenance.repository.max.storage.time=24 hours
-nifi.provenance.repository.max.storage.size=1 GB
-nifi.provenance.repository.rollover.time=30 secs
-nifi.provenance.repository.rollover.size=100 MB
-
-# Site to Site properties
-nifi.remote.input.socket.port=9990
-nifi.remote.input.secure=true
-
-# web properties #
-nifi.web.war.directory=./target/lib
-nifi.web.http.host=
-nifi.web.http.port=8080
-nifi.web.https.host=
-nifi.web.https.port=
-nifi.web.jetty.working.directory=./target/work/jetty
-
-# security properties #
-nifi.sensitive.props.key=
-nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
-
-nifi.security.keystore=
-nifi.security.keystoreType=
-nifi.security.keystorePasswd=
-nifi.security.keyPasswd=
-nifi.security.truststore=
-nifi.security.truststoreType=
-nifi.security.truststorePasswd=
-nifi.security.user.authorizer=
-
-# cluster common properties (cluster manager and nodes must have same values) #
-nifi.cluster.protocol.heartbeat.interval=5 sec
-nifi.cluster.protocol.is.secure=false
-nifi.cluster.protocol.socket.timeout=30 sec
-
-# cluster node properties (only configure for cluster nodes) #
-nifi.cluster.is.node=true
-nifi.cluster.node.address=
-nifi.cluster.node.protocol.port=
-nifi.cluster.node.protocol.threads=2
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.missing.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.missing.properties
deleted file mode 100644
index 42a8435a7e..0000000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.missing.properties
+++ /dev/null
@@ -1,119 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Core Properties #
-nifi.flow.configuration.file=./target/flow.xml.gz
-nifi.flow.configuration.archive.dir=./target/archive/
-nifi.flowcontroller.autoResumeState=true
-nifi.flowcontroller.graceful.shutdown.period=10 sec
-nifi.flowservice.writedelay.interval=2 sec
-nifi.administrative.yield.duration=30 sec
-
-nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
-nifi.controller.service.configuration.file=./target/controller-services.xml
-nifi.templates.directory=./target/templates
-nifi.ui.banner.text=UI Banner Text
-nifi.ui.autorefresh.interval=30 sec
-nifi.nar.working.directory=./target/work/nar/
-
-# H2 Settings
-nifi.database.directory=./target/database_repository
-nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
-
-# FlowFile Repository
-nifi.flowfile.repository.directory=./target/test-repo
-nifi.flowfile.repository.partitions=1
-nifi.flowfile.repository.checkpoint.interval=2 mins
-nifi.queue.swap.threshold=20000
-nifi.swap.storage.directory=./target/test-repo/swap
-nifi.swap.in.period=5 sec
-nifi.swap.in.threads=1
-nifi.swap.out.period=5 sec
-nifi.swap.out.threads=4
-
-# Content Repository
-nifi.content.claim.max.appendable.size=10 MB
-nifi.content.claim.max.flow.files=100
-nifi.content.repository.directory.default=./target/content_repository
-
-# Provenance Repository Properties
-nifi.provenance.repository.storage.directory=./target/provenance_repository
-nifi.provenance.repository.max.storage.time=24 hours
-nifi.provenance.repository.max.storage.size=1 GB
-nifi.provenance.repository.rollover.time=30 secs
-nifi.provenance.repository.rollover.size=100 MB
-
-# Site to Site properties
-nifi.remote.input.socket.port=9990
-nifi.remote.input.secure=true
-
-# web properties #
-nifi.web.war.directory=./target/lib
-nifi.web.http.host=
-nifi.web.http.port=8080
-nifi.web.https.host=
-nifi.web.https.port=
-nifi.web.jetty.working.directory=./target/work/jetty
-
-# security properties #
-nifi.sensitive.props.key=key
-nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
-
-nifi.security.keystore=
-nifi.security.keystoreType=
-nifi.security.keystorePasswd=
-nifi.security.keyPasswd=
-nifi.security.truststore=
-nifi.security.truststoreType=
-nifi.security.truststorePasswd=
-nifi.security.user.authorizer=
-
-# cluster common properties (cluster manager and nodes must have same values) #
-nifi.cluster.protocol.heartbeat.interval=5 sec
-nifi.cluster.protocol.is.secure=false
-nifi.cluster.protocol.socket.timeout=30 sec
-nifi.cluster.protocol.connection.handshake.timeout=45 sec
-# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured #
-nifi.cluster.protocol.use.multicast=false
-nifi.cluster.protocol.multicast.address=
-nifi.cluster.protocol.multicast.port=
-nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms
-nifi.cluster.protocol.multicast.service.locator.attempts=3
-nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec
-
-# cluster node properties (only configure for cluster nodes) #
-nifi.cluster.is.node=false
-nifi.cluster.node.address=
-nifi.cluster.node.protocol.port=
-nifi.cluster.node.protocol.threads=2
-# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx #
-nifi.cluster.node.unicast.manager.address=
-nifi.cluster.node.unicast.manager.protocol.port=
-nifi.cluster.node.unicast.manager.authority.provider.port=
-
-# cluster manager properties (only configure for cluster manager) #
-nifi.cluster.is.manager=false
-nifi.cluster.manager.address=
-nifi.cluster.manager.protocol.port=
-nifi.cluster.manager.authority.provider.port=
-nifi.cluster.manager.authority.provider.threads=10
-nifi.cluster.manager.node.firewall.file=
-nifi.cluster.manager.node.event.history.size=10
-nifi.cluster.manager.node.api.connection.timeout=30 sec
-nifi.cluster.manager.node.api.read.timeout=30 sec
-nifi.cluster.manager.node.api.request.threads=10
-nifi.cluster.manager.flow.retrieval.delay=5 sec
-nifi.cluster.manager.protocol.threads=10
-nifi.cluster.manager.safemode.duration=0 sec
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.properties
deleted file mode 100644
index d0b69518eb..0000000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.properties
+++ /dev/null
@@ -1,121 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Core Properties #
-nifi.flow.configuration.file=./target/flow.xml.gz
-nifi.flow.configuration.archive.dir=./target/archive/
-nifi.flowcontroller.autoResumeState=true
-nifi.flowcontroller.graceful.shutdown.period=10 sec
-nifi.flowservice.writedelay.interval=2 sec
-nifi.administrative.yield.duration=30 sec
-
-nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
-nifi.controller.service.configuration.file=./target/controller-services.xml
-nifi.templates.directory=./target/templates
-nifi.ui.banner.text=UI Banner Text
-nifi.ui.autorefresh.interval=30 sec
-nifi.nar.library.directory=./target/resources/NiFiProperties/lib/
-nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/
-nifi.nar.working.directory=./target/work/nar/
-
-# H2 Settings
-nifi.database.directory=./target/database_repository
-nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
-
-# FlowFile Repository
-nifi.flowfile.repository.directory=./target/test-repo
-nifi.flowfile.repository.partitions=1
-nifi.flowfile.repository.checkpoint.interval=2 mins
-nifi.queue.swap.threshold=20000
-nifi.swap.storage.directory=./target/test-repo/swap
-nifi.swap.in.period=5 sec
-nifi.swap.in.threads=1
-nifi.swap.out.period=5 sec
-nifi.swap.out.threads=4
-
-# Content Repository
-nifi.content.claim.max.appendable.size=10 MB
-nifi.content.claim.max.flow.files=100
-nifi.content.repository.directory.default=./target/content_repository
-
-# Provenance Repository Properties
-nifi.provenance.repository.storage.directory=./target/provenance_repository
-nifi.provenance.repository.max.storage.time=24 hours
-nifi.provenance.repository.max.storage.size=1 GB
-nifi.provenance.repository.rollover.time=30 secs
-nifi.provenance.repository.rollover.size=100 MB
-
-# Site to Site properties
-nifi.remote.input.socket.port=9990
-nifi.remote.input.secure=true
-
-# web properties #
-nifi.web.war.directory=./target/lib
-nifi.web.http.host=
-nifi.web.http.port=8080
-nifi.web.https.host=
-nifi.web.https.port=
-nifi.web.jetty.working.directory=./target/work/jetty
-
-# security properties #
-nifi.sensitive.props.key=key
-nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
-
-nifi.security.keystore=
-nifi.security.keystoreType=
-nifi.security.keystorePasswd=
-nifi.security.keyPasswd=
-nifi.security.truststore=
-nifi.security.truststoreType=
-nifi.security.truststorePasswd=
-nifi.security.user.authorizer=
-
-# cluster common properties (cluster manager and nodes must have same values) #
-nifi.cluster.protocol.heartbeat.interval=5 sec
-nifi.cluster.protocol.is.secure=false
-nifi.cluster.protocol.socket.timeout=30 sec
-nifi.cluster.protocol.connection.handshake.timeout=45 sec
-# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured #
-nifi.cluster.protocol.use.multicast=false
-nifi.cluster.protocol.multicast.address=
-nifi.cluster.protocol.multicast.port=
-nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms
-nifi.cluster.protocol.multicast.service.locator.attempts=3
-nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec
-
-# cluster node properties (only configure for cluster nodes) #
-nifi.cluster.is.node=false
-nifi.cluster.node.address=
-nifi.cluster.node.protocol.port=
-nifi.cluster.node.protocol.threads=2
-# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx #
-nifi.cluster.node.unicast.manager.address=
-nifi.cluster.node.unicast.manager.protocol.port=
-nifi.cluster.node.unicast.manager.authority.provider.port=
-
-# cluster manager properties (only configure for cluster manager) #
-nifi.cluster.is.manager=false
-nifi.cluster.manager.address=
-nifi.cluster.manager.protocol.port=
-nifi.cluster.manager.authority.provider.port=
-nifi.cluster.manager.authority.provider.threads=10
-nifi.cluster.manager.node.firewall.file=
-nifi.cluster.manager.node.event.history.size=10
-nifi.cluster.manager.node.api.connection.timeout=30 sec
-nifi.cluster.manager.node.api.read.timeout=30 sec
-nifi.cluster.manager.node.api.request.threads=10
-nifi.cluster.manager.flow.retrieval.delay=5 sec
-nifi.cluster.manager.protocol.threads=10
-nifi.cluster.manager.safemode.duration=0 sec
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.without.key.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.without.key.properties
deleted file mode 100644
index eff05c3b70..0000000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.without.key.properties
+++ /dev/null
@@ -1,94 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Core Properties #
-nifi.flow.configuration.file=./target/flow.xml.gz
-nifi.flow.configuration.archive.dir=./target/archive/
-nifi.flowcontroller.autoResumeState=true
-nifi.flowcontroller.graceful.shutdown.period=10 sec
-nifi.flowservice.writedelay.interval=2 sec
-nifi.administrative.yield.duration=30 sec
-
-nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
-nifi.controller.service.configuration.file=./target/controller-services.xml
-nifi.templates.directory=./target/templates
-nifi.ui.banner.text=UI Banner Text
-nifi.ui.autorefresh.interval=30 sec
-nifi.nar.library.directory=./target/resources/NiFiProperties/lib/
-nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/
-nifi.nar.working.directory=./target/work/nar/
-
-# H2 Settings
-nifi.database.directory=./target/database_repository
-nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
-
-# FlowFile Repository
-nifi.flowfile.repository.directory=./target/test-repo
-nifi.flowfile.repository.partitions=1
-nifi.flowfile.repository.checkpoint.interval=2 mins
-nifi.queue.swap.threshold=20000
-nifi.swap.storage.directory=./target/test-repo/swap
-nifi.swap.in.period=5 sec
-nifi.swap.in.threads=1
-nifi.swap.out.period=5 sec
-nifi.swap.out.threads=4
-
-# Content Repository
-nifi.content.claim.max.appendable.size=10 MB
-nifi.content.claim.max.flow.files=100
-nifi.content.repository.directory.default=./target/content_repository
-
-# Provenance Repository Properties
-nifi.provenance.repository.storage.directory=./target/provenance_repository
-nifi.provenance.repository.max.storage.time=24 hours
-nifi.provenance.repository.max.storage.size=1 GB
-nifi.provenance.repository.rollover.time=30 secs
-nifi.provenance.repository.rollover.size=100 MB
-
-# Site to Site properties
-nifi.remote.input.socket.port=9990
-nifi.remote.input.secure=true
-
-# web properties #
-nifi.web.war.directory=./target/lib
-nifi.web.http.host=
-nifi.web.http.port=8080
-nifi.web.https.host=
-nifi.web.https.port=
-nifi.web.jetty.working.directory=./target/work/jetty
-
-# security properties #
-nifi.sensitive.props.key=
-nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
-
-nifi.security.keystore=
-nifi.security.keystoreType=
-nifi.security.keystorePasswd=
-nifi.security.keyPasswd=
-nifi.security.truststore=
-nifi.security.truststoreType=
-nifi.security.truststorePasswd=
-nifi.security.user.authorizer=
-
-# cluster common properties (cluster manager and nodes must have same values) #
-nifi.cluster.protocol.heartbeat.interval=5 sec
-nifi.cluster.protocol.is.secure=false
-nifi.cluster.protocol.socket.timeout=30 sec
-
-# cluster node properties (only configure for cluster nodes) #
-nifi.cluster.is.node=false
-nifi.cluster.node.address=
-nifi.cluster.node.protocol.port=
-nifi.cluster.node.protocol.threads=2
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_no_permissions.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_no_permissions.properties
deleted file mode 100644
index ae1e83eeb3..0000000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_no_permissions.properties
+++ /dev/null
@@ -1,14 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_additional_sensitive_keys.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_additional_sensitive_keys.properties
deleted file mode 100644
index 487b09fdd5..0000000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_additional_sensitive_keys.properties
+++ /dev/null
@@ -1,122 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Core Properties #
-nifi.flow.configuration.file=./target/flow.xml.gz
-nifi.flow.configuration.archive.dir=./target/archive/
-nifi.flowcontroller.autoResumeState=true
-nifi.flowcontroller.graceful.shutdown.period=10 sec
-nifi.flowservice.writedelay.interval=2 sec
-nifi.administrative.yield.duration=30 sec
-
-nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
-nifi.controller.service.configuration.file=./target/controller-services.xml
-nifi.templates.directory=./target/templates
-nifi.ui.banner.text=UI Banner Text
-nifi.ui.autorefresh.interval=30 sec
-nifi.nar.library.directory=./target/resources/NiFiProperties/lib/
-nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/
-nifi.nar.working.directory=./target/work/nar/
-
-# H2 Settings
-nifi.database.directory=./target/database_repository
-nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
-
-# FlowFile Repository
-nifi.flowfile.repository.directory=./target/test-repo
-nifi.flowfile.repository.partitions=1
-nifi.flowfile.repository.checkpoint.interval=2 mins
-nifi.queue.swap.threshold=20000
-nifi.swap.storage.directory=./target/test-repo/swap
-nifi.swap.in.period=5 sec
-nifi.swap.in.threads=1
-nifi.swap.out.period=5 sec
-nifi.swap.out.threads=4
-
-# Content Repository
-nifi.content.claim.max.appendable.size=10 MB
-nifi.content.claim.max.flow.files=100
-nifi.content.repository.directory.default=./target/content_repository
-
-# Provenance Repository Properties
-nifi.provenance.repository.storage.directory=./target/provenance_repository
-nifi.provenance.repository.max.storage.time=24 hours
-nifi.provenance.repository.max.storage.size=1 GB
-nifi.provenance.repository.rollover.time=30 secs
-nifi.provenance.repository.rollover.size=100 MB
-
-# Site to Site properties
-nifi.remote.input.socket.port=9990
-nifi.remote.input.secure=true
-
-# web properties #
-nifi.web.war.directory=./target/lib
-nifi.web.http.host=
-nifi.web.http.port=8080
-nifi.web.https.host=
-nifi.web.https.port=
-nifi.web.jetty.working.directory=./target/work/jetty
-
-# security properties #
-nifi.sensitive.props.key=key
-nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
-nifi.sensitive.props.additional.keys=nifi.ui.banner.text, nifi.version, nifi.sensitive.props.additional.keys
-
-nifi.security.keystore=
-nifi.security.keystoreType=
-nifi.security.keystorePasswd=
-nifi.security.keyPasswd=
-nifi.security.truststore=
-nifi.security.truststoreType=
-nifi.security.truststorePasswd=
-nifi.security.user.authorizer=
-
-# cluster common properties (cluster manager and nodes must have same values) #
-nifi.cluster.protocol.heartbeat.interval=5 sec
-nifi.cluster.protocol.is.secure=false
-nifi.cluster.protocol.socket.timeout=30 sec
-nifi.cluster.protocol.connection.handshake.timeout=45 sec
-# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured #
-nifi.cluster.protocol.use.multicast=false
-nifi.cluster.protocol.multicast.address=
-nifi.cluster.protocol.multicast.port=
-nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms
-nifi.cluster.protocol.multicast.service.locator.attempts=3
-nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec
-
-# cluster node properties (only configure for cluster nodes) #
-nifi.cluster.is.node=false
-nifi.cluster.node.address=
-nifi.cluster.node.protocol.port=
-nifi.cluster.node.protocol.threads=2
-# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx #
-nifi.cluster.node.unicast.manager.address=
-nifi.cluster.node.unicast.manager.protocol.port=
-nifi.cluster.node.unicast.manager.authority.provider.port=
-
-# cluster manager properties (only configure for cluster manager) #
-nifi.cluster.is.manager=false
-nifi.cluster.manager.address=
-nifi.cluster.manager.protocol.port=
-nifi.cluster.manager.authority.provider.port=
-nifi.cluster.manager.authority.provider.threads=10
-nifi.cluster.manager.node.firewall.file=
-nifi.cluster.manager.node.event.history.size=10
-nifi.cluster.manager.node.api.connection.timeout=30 sec
-nifi.cluster.manager.node.api.read.timeout=30 sec
-nifi.cluster.manager.node.api.request.threads=10
-nifi.cluster.manager.flow.retrieval.delay=5 sec
-nifi.cluster.manager.protocol.threads=10
-nifi.cluster.manager.safemode.duration=0 sec
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_all_sensitive_properties_protected_aes.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_all_sensitive_properties_protected_aes.properties
deleted file mode 100644
index 6e3ea7afea..0000000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_all_sensitive_properties_protected_aes.properties
+++ /dev/null
@@ -1,126 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Core Properties #
-nifi.flow.configuration.file=./target/flow.xml.gz
-nifi.flow.configuration.archive.dir=./target/archive/
-nifi.flowcontroller.autoResumeState=true
-nifi.flowcontroller.graceful.shutdown.period=10 sec
-nifi.flowservice.writedelay.interval=2 sec
-nifi.administrative.yield.duration=30 sec
-
-nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
-nifi.controller.service.configuration.file=./target/controller-services.xml
-nifi.templates.directory=./target/templates
-nifi.ui.banner.text=UI Banner Text
-nifi.ui.autorefresh.interval=30 sec
-nifi.nar.library.directory=./target/resources/NiFiProperties/lib/
-nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/
-nifi.nar.working.directory=./target/work/nar/
-
-# H2 Settings
-nifi.database.directory=./target/database_repository
-nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
-
-# FlowFile Repository
-nifi.flowfile.repository.directory=./target/test-repo
-nifi.flowfile.repository.partitions=1
-nifi.flowfile.repository.checkpoint.interval=2 mins
-nifi.queue.swap.threshold=20000
-nifi.swap.storage.directory=./target/test-repo/swap
-nifi.swap.in.period=5 sec
-nifi.swap.in.threads=1
-nifi.swap.out.period=5 sec
-nifi.swap.out.threads=4
-
-# Content Repository
-nifi.content.claim.max.appendable.size=10 MB
-nifi.content.claim.max.flow.files=100
-nifi.content.repository.directory.default=./target/content_repository
-
-# Provenance Repository Properties
-nifi.provenance.repository.storage.directory=./target/provenance_repository
-nifi.provenance.repository.max.storage.time=24 hours
-nifi.provenance.repository.max.storage.size=1 GB
-nifi.provenance.repository.rollover.time=30 secs
-nifi.provenance.repository.rollover.size=100 MB
-
-# Site to Site properties
-nifi.remote.input.socket.port=9990
-nifi.remote.input.secure=true
-
-# web properties #
-nifi.web.war.directory=./target/lib
-nifi.web.http.host=
-nifi.web.http.port=
-nifi.web.https.host=nifi.nifi.apache.org
-nifi.web.https.port=8443
-nifi.web.jetty.working.directory=./target/work/jetty
-
-# security properties #
-nifi.sensitive.props.key=n2z+tTTbHuZ4V4V2||uWhdasyDXD4ZG2lMAes/vqh6u4vaz4xgL4aEbF4Y/dXevqk3ulRcOwf1vc4RDQ==
-nifi.sensitive.props.key.protected=aes/gcm/256
-nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
-nifi.sensitive.props.additional.keys=
-
-nifi.security.keystore=/path/to/keystore.jks
-nifi.security.keystoreType=JKS
-nifi.security.keystorePasswd=oBjT92hIGRElIGOh||MZ6uYuWNBrOA6usq/Jt3DaD2e4otNirZDytac/w/KFe0HOkrJR03vcbo
-nifi.security.keystorePasswd.protected=aes/gcm/256
-nifi.security.keyPasswd=ac/BaE35SL/esLiJ||+ULRvRLYdIDA2VqpE0eQXDEMjaLBMG2kbKOdOwBk/hGebDKlVg==
-nifi.security.keyPasswd.protected=aes/gcm/256
-nifi.security.truststore=
-nifi.security.truststoreType=
-nifi.security.truststorePasswd=/X/RSlNr2QCJ1Kwe||dENJevX5P61ix+97airrtoBQoyasMFS6DG6fHbX+SZtw2VAMllSSnDeT97Q=
-nifi.security.truststorePasswd.protected=aes/gcm/256
-nifi.security.user.authorizer=
-
-# cluster common properties (cluster manager and nodes must have same values) #
-nifi.cluster.protocol.heartbeat.interval=5 sec
-nifi.cluster.protocol.is.secure=false
-nifi.cluster.protocol.socket.timeout=30 sec
-nifi.cluster.protocol.connection.handshake.timeout=45 sec
-# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured #
-nifi.cluster.protocol.use.multicast=false
-nifi.cluster.protocol.multicast.address=
-nifi.cluster.protocol.multicast.port=
-nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms
-nifi.cluster.protocol.multicast.service.locator.attempts=3
-nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec
-
-# cluster node properties (only configure for cluster nodes) #
-nifi.cluster.is.node=false
-nifi.cluster.node.address=
-nifi.cluster.node.protocol.port=
-nifi.cluster.node.protocol.threads=2
-# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx #
-nifi.cluster.node.unicast.manager.address=
-nifi.cluster.node.unicast.manager.protocol.port=
-nifi.cluster.node.unicast.manager.authority.provider.port=
-
-# cluster manager properties (only configure for cluster manager) #
-nifi.cluster.is.manager=false
-nifi.cluster.manager.address=
-nifi.cluster.manager.protocol.port=
-nifi.cluster.manager.authority.provider.port=
-nifi.cluster.manager.authority.provider.threads=10
-nifi.cluster.manager.node.firewall.file=
-nifi.cluster.manager.node.event.history.size=10
-nifi.cluster.manager.node.api.connection.timeout=30 sec
-nifi.cluster.manager.node.api.read.timeout=30 sec
-nifi.cluster.manager.node.api.request.threads=10
-nifi.cluster.manager.flow.retrieval.delay=5 sec
-nifi.cluster.manager.protocol.threads=10
-nifi.cluster.manager.safemode.duration=0 sec
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_recursive_additional_sensitive_keys.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_recursive_additional_sensitive_keys.properties
deleted file mode 100644
index 487b09fdd5..0000000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_recursive_additional_sensitive_keys.properties
+++ /dev/null
@@ -1,122 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Core Properties #
-nifi.flow.configuration.file=./target/flow.xml.gz
-nifi.flow.configuration.archive.dir=./target/archive/
-nifi.flowcontroller.autoResumeState=true
-nifi.flowcontroller.graceful.shutdown.period=10 sec
-nifi.flowservice.writedelay.interval=2 sec
-nifi.administrative.yield.duration=30 sec
-
-nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
-nifi.controller.service.configuration.file=./target/controller-services.xml
-nifi.templates.directory=./target/templates
-nifi.ui.banner.text=UI Banner Text
-nifi.ui.autorefresh.interval=30 sec
-nifi.nar.library.directory=./target/resources/NiFiProperties/lib/
-nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/
-nifi.nar.working.directory=./target/work/nar/
-
-# H2 Settings
-nifi.database.directory=./target/database_repository
-nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
-
-# FlowFile Repository
-nifi.flowfile.repository.directory=./target/test-repo
-nifi.flowfile.repository.partitions=1
-nifi.flowfile.repository.checkpoint.interval=2 mins
-nifi.queue.swap.threshold=20000
-nifi.swap.storage.directory=./target/test-repo/swap
-nifi.swap.in.period=5 sec
-nifi.swap.in.threads=1
-nifi.swap.out.period=5 sec
-nifi.swap.out.threads=4
-
-# Content Repository
-nifi.content.claim.max.appendable.size=10 MB
-nifi.content.claim.max.flow.files=100
-nifi.content.repository.directory.default=./target/content_repository
-
-# Provenance Repository Properties
-nifi.provenance.repository.storage.directory=./target/provenance_repository
-nifi.provenance.repository.max.storage.time=24 hours
-nifi.provenance.repository.max.storage.size=1 GB
-nifi.provenance.repository.rollover.time=30 secs
-nifi.provenance.repository.rollover.size=100 MB
-
-# Site to Site properties
-nifi.remote.input.socket.port=9990
-nifi.remote.input.secure=true
-
-# web properties #
-nifi.web.war.directory=./target/lib
-nifi.web.http.host=
-nifi.web.http.port=8080
-nifi.web.https.host=
-nifi.web.https.port=
-nifi.web.jetty.working.directory=./target/work/jetty
-
-# security properties #
-nifi.sensitive.props.key=key
-nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
-nifi.sensitive.props.additional.keys=nifi.ui.banner.text, nifi.version, nifi.sensitive.props.additional.keys
-
-nifi.security.keystore=
-nifi.security.keystoreType=
-nifi.security.keystorePasswd=
-nifi.security.keyPasswd=
-nifi.security.truststore=
-nifi.security.truststoreType=
-nifi.security.truststorePasswd=
-nifi.security.user.authorizer=
-
-# cluster common properties (cluster manager and nodes must have same values) #
-nifi.cluster.protocol.heartbeat.interval=5 sec
-nifi.cluster.protocol.is.secure=false
-nifi.cluster.protocol.socket.timeout=30 sec
-nifi.cluster.protocol.connection.handshake.timeout=45 sec
-# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured #
-nifi.cluster.protocol.use.multicast=false
-nifi.cluster.protocol.multicast.address=
-nifi.cluster.protocol.multicast.port=
-nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms
-nifi.cluster.protocol.multicast.service.locator.attempts=3
-nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec
-
-# cluster node properties (only configure for cluster nodes) #
-nifi.cluster.is.node=false
-nifi.cluster.node.address=
-nifi.cluster.node.protocol.port=
-nifi.cluster.node.protocol.threads=2
-# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx #
-nifi.cluster.node.unicast.manager.address=
-nifi.cluster.node.unicast.manager.protocol.port=
-nifi.cluster.node.unicast.manager.authority.provider.port=
-
-# cluster manager properties (only configure for cluster manager) #
-nifi.cluster.is.manager=false
-nifi.cluster.manager.address=
-nifi.cluster.manager.protocol.port=
-nifi.cluster.manager.authority.provider.port=
-nifi.cluster.manager.authority.provider.threads=10
-nifi.cluster.manager.node.firewall.file=
-nifi.cluster.manager.node.event.history.size=10
-nifi.cluster.manager.node.api.connection.timeout=30 sec
-nifi.cluster.manager.node.api.read.timeout=30 sec
-nifi.cluster.manager.node.api.request.threads=10
-nifi.cluster.manager.flow.retrieval.delay=5 sec
-nifi.cluster.manager.protocol.threads=10
-nifi.cluster.manager.safemode.duration=0 sec
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes.properties
deleted file mode 100644
index e379d43789..0000000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes.properties
+++ /dev/null
@@ -1,125 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Core Properties #
-nifi.flow.configuration.file=./target/flow.xml.gz
-nifi.flow.configuration.archive.dir=./target/archive/
-nifi.flowcontroller.autoResumeState=true
-nifi.flowcontroller.graceful.shutdown.period=10 sec
-nifi.flowservice.writedelay.interval=2 sec
-nifi.administrative.yield.duration=30 sec
-
-nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
-nifi.controller.service.configuration.file=./target/controller-services.xml
-nifi.templates.directory=./target/templates
-nifi.ui.banner.text=UI Banner Text
-nifi.ui.autorefresh.interval=30 sec
-nifi.nar.library.directory=./target/resources/NiFiProperties/lib/
-nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/
-nifi.nar.working.directory=./target/work/nar/
-
-# H2 Settings
-nifi.database.directory=./target/database_repository
-nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
-
-# FlowFile Repository
-nifi.flowfile.repository.directory=./target/test-repo
-nifi.flowfile.repository.partitions=1
-nifi.flowfile.repository.checkpoint.interval=2 mins
-nifi.queue.swap.threshold=20000
-nifi.swap.storage.directory=./target/test-repo/swap
-nifi.swap.in.period=5 sec
-nifi.swap.in.threads=1
-nifi.swap.out.period=5 sec
-nifi.swap.out.threads=4
-
-# Content Repository
-nifi.content.claim.max.appendable.size=10 MB
-nifi.content.claim.max.flow.files=100
-nifi.content.repository.directory.default=./target/content_repository
-
-# Provenance Repository Properties
-nifi.provenance.repository.storage.directory=./target/provenance_repository
-nifi.provenance.repository.max.storage.time=24 hours
-nifi.provenance.repository.max.storage.size=1 GB
-nifi.provenance.repository.rollover.time=30 secs
-nifi.provenance.repository.rollover.size=100 MB
-
-# Site to Site properties
-nifi.remote.input.socket.port=9990
-nifi.remote.input.secure=true
-
-# web properties #
-nifi.web.war.directory=./target/lib
-nifi.web.http.host=
-nifi.web.http.port=
-nifi.web.https.host=nifi.nifi.apache.org
-nifi.web.https.port=8443
-nifi.web.jetty.working.directory=./target/work/jetty
-
-# security properties #
-nifi.sensitive.props.key=n2z+tTTbHuZ4V4V2||uWhdasyDXD4ZG2lMAes/vqh6u4vaz4xgL4aEbF4Y/dXevqk3ulRcOwf1vc4RDQ==
-nifi.sensitive.props.key.protected=aes/gcm/256
-nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
-nifi.sensitive.props.additional.keys=nifi.ui.banner.text, nifi.version
-
-nifi.security.keystore=/path/to/keystore.jks
-nifi.security.keystoreType=JKS
-nifi.security.keystorePasswd=oBjT92hIGRElIGOh||MZ6uYuWNBrOA6usq/Jt3DaD2e4otNirZDytac/w/KFe0HOkrJR03vcbo
-nifi.security.keystorePasswd.protected=aes/gcm/256
-nifi.security.keyPasswd=ac/BaE35SL/esLiJ||+ULRvRLYdIDA2VqpE0eQXDEMjaLBMG2kbKOdOwBk/hGebDKlVg==
-nifi.security.keyPasswd.protected=aes/gcm/256
-nifi.security.truststore=
-nifi.security.truststoreType=
-nifi.security.truststorePasswd=
-nifi.security.user.authorizer=
-
-# cluster common properties (cluster manager and nodes must have same values) #
-nifi.cluster.protocol.heartbeat.interval=5 sec
-nifi.cluster.protocol.is.secure=false
-nifi.cluster.protocol.socket.timeout=30 sec
-nifi.cluster.protocol.connection.handshake.timeout=45 sec
-# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured #
-nifi.cluster.protocol.use.multicast=false
-nifi.cluster.protocol.multicast.address=
-nifi.cluster.protocol.multicast.port=
-nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms
-nifi.cluster.protocol.multicast.service.locator.attempts=3
-nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec
-
-# cluster node properties (only configure for cluster nodes) #
-nifi.cluster.is.node=false
-nifi.cluster.node.address=
-nifi.cluster.node.protocol.port=
-nifi.cluster.node.protocol.threads=2
-# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx #
-nifi.cluster.node.unicast.manager.address=
-nifi.cluster.node.unicast.manager.protocol.port=
-nifi.cluster.node.unicast.manager.authority.provider.port=
-
-# cluster manager properties (only configure for cluster manager) #
-nifi.cluster.is.manager=false
-nifi.cluster.manager.address=
-nifi.cluster.manager.protocol.port=
-nifi.cluster.manager.authority.provider.port=
-nifi.cluster.manager.authority.provider.threads=10
-nifi.cluster.manager.node.firewall.file=
-nifi.cluster.manager.node.event.history.size=10
-nifi.cluster.manager.node.api.connection.timeout=30 sec
-nifi.cluster.manager.node.api.read.timeout=30 sec
-nifi.cluster.manager.node.api.request.threads=10
-nifi.cluster.manager.flow.retrieval.delay=5 sec
-nifi.cluster.manager.protocol.threads=10
-nifi.cluster.manager.safemode.duration=0 sec
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_128.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_128.properties
deleted file mode 100644
index c5ccfb0d58..0000000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_128.properties
+++ /dev/null
@@ -1,125 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Core Properties #
-nifi.flow.configuration.file=./target/flow.xml.gz
-nifi.flow.configuration.archive.dir=./target/archive/
-nifi.flowcontroller.autoResumeState=true
-nifi.flowcontroller.graceful.shutdown.period=10 sec
-nifi.flowservice.writedelay.interval=2 sec
-nifi.administrative.yield.duration=30 sec
-
-nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
-nifi.controller.service.configuration.file=./target/controller-services.xml
-nifi.templates.directory=./target/templates
-nifi.ui.banner.text=UI Banner Text
-nifi.ui.autorefresh.interval=30 sec
-nifi.nar.library.directory=./target/resources/NiFiProperties/lib/
-nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/
-nifi.nar.working.directory=./target/work/nar/
-
-# H2 Settings
-nifi.database.directory=./target/database_repository
-nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
-
-# FlowFile Repository
-nifi.flowfile.repository.directory=./target/test-repo
-nifi.flowfile.repository.partitions=1
-nifi.flowfile.repository.checkpoint.interval=2 mins
-nifi.queue.swap.threshold=20000
-nifi.swap.storage.directory=./target/test-repo/swap
-nifi.swap.in.period=5 sec
-nifi.swap.in.threads=1
-nifi.swap.out.period=5 sec
-nifi.swap.out.threads=4
-
-# Content Repository
-nifi.content.claim.max.appendable.size=10 MB
-nifi.content.claim.max.flow.files=100
-nifi.content.repository.directory.default=./target/content_repository
-
-# Provenance Repository Properties
-nifi.provenance.repository.storage.directory=./target/provenance_repository
-nifi.provenance.repository.max.storage.time=24 hours
-nifi.provenance.repository.max.storage.size=1 GB
-nifi.provenance.repository.rollover.time=30 secs
-nifi.provenance.repository.rollover.size=100 MB
-
-# Site to Site properties
-nifi.remote.input.socket.port=9990
-nifi.remote.input.secure=true
-
-# web properties #
-nifi.web.war.directory=./target/lib
-nifi.web.http.host=
-nifi.web.http.port=
-nifi.web.https.host=nifi.nifi.apache.org
-nifi.web.https.port=8443
-nifi.web.jetty.working.directory=./target/work/jetty
-
-# security properties #
-nifi.sensitive.props.key=6WUpex+VZiN05LXu||joWJMuoSzYniEC7IAoingTimlG7+RGk8I2irl/WTlIuMcg
-nifi.sensitive.props.key.protected=aes/gcm/128
-nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
-nifi.sensitive.props.additional.keys=nifi.ui.banner.text, nifi.version
-
-nifi.security.keystore=/path/to/keystore.jks
-nifi.security.keystoreType=JKS
-nifi.security.keystorePasswd=6WUpex+VZiN05LXu||joWJMuoSzYniEC7IAoingTimlG7+RGk8I2irl/WTlIuMcg
-nifi.security.keystorePasswd.protected=aes/gcm/128
-nifi.security.keyPasswd=6WUpex+VZiN05LXu||joWJMuoSzYniEC7IAoingTimlG7+RGk8I2irl/WTlIuMcg
-nifi.security.keyPasswd.protected=aes/gcm/128
-nifi.security.truststore=
-nifi.security.truststoreType=
-nifi.security.truststorePasswd=
-nifi.security.user.authorizer=
-
-# cluster common properties (cluster manager and nodes must have same values) #
-nifi.cluster.protocol.heartbeat.interval=5 sec
-nifi.cluster.protocol.is.secure=false
-nifi.cluster.protocol.socket.timeout=30 sec
-nifi.cluster.protocol.connection.handshake.timeout=45 sec
-# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured #
-nifi.cluster.protocol.use.multicast=false
-nifi.cluster.protocol.multicast.address=
-nifi.cluster.protocol.multicast.port=
-nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms
-nifi.cluster.protocol.multicast.service.locator.attempts=3
-nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec
-
-# cluster node properties (only configure for cluster nodes) #
-nifi.cluster.is.node=false
-nifi.cluster.node.address=
-nifi.cluster.node.protocol.port=
-nifi.cluster.node.protocol.threads=2
-# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx #
-nifi.cluster.node.unicast.manager.address=
-nifi.cluster.node.unicast.manager.protocol.port=
-nifi.cluster.node.unicast.manager.authority.provider.port=
-
-# cluster manager properties (only configure for cluster manager) #
-nifi.cluster.is.manager=false
-nifi.cluster.manager.address=
-nifi.cluster.manager.protocol.port=
-nifi.cluster.manager.authority.provider.port=
-nifi.cluster.manager.authority.provider.threads=10
-nifi.cluster.manager.node.firewall.file=
-nifi.cluster.manager.node.event.history.size=10
-nifi.cluster.manager.node.api.connection.timeout=30 sec
-nifi.cluster.manager.node.api.read.timeout=30 sec
-nifi.cluster.manager.node.api.request.threads=10
-nifi.cluster.manager.flow.retrieval.delay=5 sec
-nifi.cluster.manager.protocol.threads=10
-nifi.cluster.manager.safemode.duration=0 sec
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_128_password.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_128_password.properties
deleted file mode 100644
index 8add3d51cd..0000000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_128_password.properties
+++ /dev/null
@@ -1,125 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Core Properties #
-nifi.flow.configuration.file=./target/flow.xml.gz
-nifi.flow.configuration.archive.dir=./target/archive/
-nifi.flowcontroller.autoResumeState=true
-nifi.flowcontroller.graceful.shutdown.period=10 sec
-nifi.flowservice.writedelay.interval=2 sec
-nifi.administrative.yield.duration=30 sec
-
-nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
-nifi.controller.service.configuration.file=./target/controller-services.xml
-nifi.templates.directory=./target/templates
-nifi.ui.banner.text=UI Banner Text
-nifi.ui.autorefresh.interval=30 sec
-nifi.nar.library.directory=./target/resources/NiFiProperties/lib/
-nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/
-nifi.nar.working.directory=./target/work/nar/
-
-# H2 Settings
-nifi.database.directory=./target/database_repository
-nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
-
-# FlowFile Repository
-nifi.flowfile.repository.directory=./target/test-repo
-nifi.flowfile.repository.partitions=1
-nifi.flowfile.repository.checkpoint.interval=2 mins
-nifi.queue.swap.threshold=20000
-nifi.swap.storage.directory=./target/test-repo/swap
-nifi.swap.in.period=5 sec
-nifi.swap.in.threads=1
-nifi.swap.out.period=5 sec
-nifi.swap.out.threads=4
-
-# Content Repository
-nifi.content.claim.max.appendable.size=10 MB
-nifi.content.claim.max.flow.files=100
-nifi.content.repository.directory.default=./target/content_repository
-
-# Provenance Repository Properties
-nifi.provenance.repository.storage.directory=./target/provenance_repository
-nifi.provenance.repository.max.storage.time=24 hours
-nifi.provenance.repository.max.storage.size=1 GB
-nifi.provenance.repository.rollover.time=30 secs
-nifi.provenance.repository.rollover.size=100 MB
-
-# Site to Site properties
-nifi.remote.input.socket.port=9990
-nifi.remote.input.secure=true
-
-# web properties #
-nifi.web.war.directory=./target/lib
-nifi.web.http.host=
-nifi.web.http.port=
-nifi.web.https.host=nifi.nifi.apache.org
-nifi.web.https.port=8443
-nifi.web.jetty.working.directory=./target/work/jetty
-
-# security properties #
-nifi.sensitive.props.key=oa6Aaz5tlFprPuKt||IlVgftF2VqvBIambkP5HVDbRoyKzZl8wwKSw4O9tjHTALA
-nifi.sensitive.props.key.protected=aes/gcm/128
-nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
-nifi.sensitive.props.additional.keys=nifi.ui.banner.text, nifi.version
-
-nifi.security.keystore=/path/to/keystore.jks
-nifi.security.keystoreType=JKS
-nifi.security.keystorePasswd=oa6Aaz5tlFprPuKt||IlVgftF2VqvBIambkP5HVDbRoyKzZl8wwKSw4O9tjHTALA
-nifi.security.keystorePasswd.protected=aes/gcm/128
-nifi.security.keyPasswd=oa6Aaz5tlFprPuKt||IlVgftF2VqvBIambkP5HVDbRoyKzZl8wwKSw4O9tjHTALA
-nifi.security.keyPasswd.protected=aes/gcm/128
-nifi.security.truststore=
-nifi.security.truststoreType=
-nifi.security.truststorePasswd=
-nifi.security.user.authorizer=
-
-# cluster common properties (cluster manager and nodes must have same values) #
-nifi.cluster.protocol.heartbeat.interval=5 sec
-nifi.cluster.protocol.is.secure=false
-nifi.cluster.protocol.socket.timeout=30 sec
-nifi.cluster.protocol.connection.handshake.timeout=45 sec
-# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured #
-nifi.cluster.protocol.use.multicast=false
-nifi.cluster.protocol.multicast.address=
-nifi.cluster.protocol.multicast.port=
-nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms
-nifi.cluster.protocol.multicast.service.locator.attempts=3
-nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec
-
-# cluster node properties (only configure for cluster nodes) #
-nifi.cluster.is.node=false
-nifi.cluster.node.address=
-nifi.cluster.node.protocol.port=
-nifi.cluster.node.protocol.threads=2
-# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx #
-nifi.cluster.node.unicast.manager.address=
-nifi.cluster.node.unicast.manager.protocol.port=
-nifi.cluster.node.unicast.manager.authority.provider.port=
-
-# cluster manager properties (only configure for cluster manager) #
-nifi.cluster.is.manager=false
-nifi.cluster.manager.address=
-nifi.cluster.manager.protocol.port=
-nifi.cluster.manager.authority.provider.port=
-nifi.cluster.manager.authority.provider.threads=10
-nifi.cluster.manager.node.firewall.file=
-nifi.cluster.manager.node.event.history.size=10
-nifi.cluster.manager.node.api.connection.timeout=30 sec
-nifi.cluster.manager.node.api.read.timeout=30 sec
-nifi.cluster.manager.node.api.request.threads=10
-nifi.cluster.manager.flow.retrieval.delay=5 sec
-nifi.cluster.manager.protocol.threads=10
-nifi.cluster.manager.safemode.duration=0 sec
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_improper_delimiter_value.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_improper_delimiter_value.properties
deleted file mode 100644
index c932aa88c5..0000000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_improper_delimiter_value.properties
+++ /dev/null
@@ -1,125 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Core Properties #
-nifi.flow.configuration.file=./target/flow.xml.gz
-nifi.flow.configuration.archive.dir=./target/archive/
-nifi.flowcontroller.autoResumeState=true
-nifi.flowcontroller.graceful.shutdown.period=10 sec
-nifi.flowservice.writedelay.interval=2 sec
-nifi.administrative.yield.duration=30 sec
-
-nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
-nifi.controller.service.configuration.file=./target/controller-services.xml
-nifi.templates.directory=./target/templates
-nifi.ui.banner.text=UI Banner Text
-nifi.ui.autorefresh.interval=30 sec
-nifi.nar.library.directory=./target/resources/NiFiProperties/lib/
-nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/
-nifi.nar.working.directory=./target/work/nar/
-
-# H2 Settings
-nifi.database.directory=./target/database_repository
-nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
-
-# FlowFile Repository
-nifi.flowfile.repository.directory=./target/test-repo
-nifi.flowfile.repository.partitions=1
-nifi.flowfile.repository.checkpoint.interval=2 mins
-nifi.queue.swap.threshold=20000
-nifi.swap.storage.directory=./target/test-repo/swap
-nifi.swap.in.period=5 sec
-nifi.swap.in.threads=1
-nifi.swap.out.period=5 sec
-nifi.swap.out.threads=4
-
-# Content Repository
-nifi.content.claim.max.appendable.size=10 MB
-nifi.content.claim.max.flow.files=100
-nifi.content.repository.directory.default=./target/content_repository
-
-# Provenance Repository Properties
-nifi.provenance.repository.storage.directory=./target/provenance_repository
-nifi.provenance.repository.max.storage.time=24 hours
-nifi.provenance.repository.max.storage.size=1 GB
-nifi.provenance.repository.rollover.time=30 secs
-nifi.provenance.repository.rollover.size=100 MB
-
-# Site to Site properties
-nifi.remote.input.socket.port=9990
-nifi.remote.input.secure=true
-
-# web properties #
-nifi.web.war.directory=./target/lib
-nifi.web.http.host=
-nifi.web.http.port=
-nifi.web.https.host=nifi.nifi.apache.org
-nifi.web.https.port=8443
-nifi.web.jetty.working.directory=./target/work/jetty
-
-# security properties #
-nifi.sensitive.props.key=n2z+tTTbHuZ4V4V2||uWhdasyDXD4ZG2lMAes/vqh6u4vaz4xgL4aEbF4Y/dXevqk3ulRcOwf1vc4RDQ==
-nifi.sensitive.props.key.protected=aes/gcm/256
-nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
-nifi.sensitive.props.additional.keys=nifi.ui.banner.text, nifi.version
-
-nifi.security.keystore=/path/to/keystore.jks
-nifi.security.keystoreType=JKS
-nifi.security.keystorePasswd=MZ6uYuWNBrOA6usq/Jt3DaD2e4otNirZDytac/w/KFe0HOkrJR03
-nifi.security.keystorePasswd.protected=aes/gcm/256
-nifi.security.keyPasswd=ac/BaE35SL/esLiJ||+ULRvRLYdIDA2VqpE0eQXDEMjaLBMG2kbKOdOwBk/hGebDKlVg==
-nifi.security.keyPasswd.protected=aes/gcm/256
-nifi.security.truststore=
-nifi.security.truststoreType=
-nifi.security.truststorePasswd=
-nifi.security.user.authorizer=
-
-# cluster common properties (cluster manager and nodes must have same values) #
-nifi.cluster.protocol.heartbeat.interval=5 sec
-nifi.cluster.protocol.is.secure=false
-nifi.cluster.protocol.socket.timeout=30 sec
-nifi.cluster.protocol.connection.handshake.timeout=45 sec
-# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured #
-nifi.cluster.protocol.use.multicast=false
-nifi.cluster.protocol.multicast.address=
-nifi.cluster.protocol.multicast.port=
-nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms
-nifi.cluster.protocol.multicast.service.locator.attempts=3
-nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec
-
-# cluster node properties (only configure for cluster nodes) #
-nifi.cluster.is.node=false
-nifi.cluster.node.address=
-nifi.cluster.node.protocol.port=
-nifi.cluster.node.protocol.threads=2
-# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx #
-nifi.cluster.node.unicast.manager.address=
-nifi.cluster.node.unicast.manager.protocol.port=
-nifi.cluster.node.unicast.manager.authority.provider.port=
-
-# cluster manager properties (only configure for cluster manager) #
-nifi.cluster.is.manager=false
-nifi.cluster.manager.address=
-nifi.cluster.manager.protocol.port=
-nifi.cluster.manager.authority.provider.port=
-nifi.cluster.manager.authority.provider.threads=10
-nifi.cluster.manager.node.firewall.file=
-nifi.cluster.manager.node.event.history.size=10
-nifi.cluster.manager.node.api.connection.timeout=30 sec
-nifi.cluster.manager.node.api.read.timeout=30 sec
-nifi.cluster.manager.node.api.request.threads=10
-nifi.cluster.manager.flow.retrieval.delay=5 sec
-nifi.cluster.manager.protocol.threads=10
-nifi.cluster.manager.safemode.duration=0 sec
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_multiple_malformed.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_multiple_malformed.properties
deleted file mode 100644
index 94223610e2..0000000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_multiple_malformed.properties
+++ /dev/null
@@ -1,125 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Core Properties #
-nifi.flow.configuration.file=./target/flow.xml.gz
-nifi.flow.configuration.archive.dir=./target/archive/
-nifi.flowcontroller.autoResumeState=true
-nifi.flowcontroller.graceful.shutdown.period=10 sec
-nifi.flowservice.writedelay.interval=2 sec
-nifi.administrative.yield.duration=30 sec
-
-nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
-nifi.controller.service.configuration.file=./target/controller-services.xml
-nifi.templates.directory=./target/templates
-nifi.ui.banner.text=UI Banner Text
-nifi.ui.autorefresh.interval=30 sec
-nifi.nar.library.directory=./target/resources/NiFiProperties/lib/
-nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/
-nifi.nar.working.directory=./target/work/nar/
-
-# H2 Settings
-nifi.database.directory=./target/database_repository
-nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
-
-# FlowFile Repository
-nifi.flowfile.repository.directory=./target/test-repo
-nifi.flowfile.repository.partitions=1
-nifi.flowfile.repository.checkpoint.interval=2 mins
-nifi.queue.swap.threshold=20000
-nifi.swap.storage.directory=./target/test-repo/swap
-nifi.swap.in.period=5 sec
-nifi.swap.in.threads=1
-nifi.swap.out.period=5 sec
-nifi.swap.out.threads=4
-
-# Content Repository
-nifi.content.claim.max.appendable.size=10 MB
-nifi.content.claim.max.flow.files=100
-nifi.content.repository.directory.default=./target/content_repository
-
-# Provenance Repository Properties
-nifi.provenance.repository.storage.directory=./target/provenance_repository
-nifi.provenance.repository.max.storage.time=24 hours
-nifi.provenance.repository.max.storage.size=1 GB
-nifi.provenance.repository.rollover.time=30 secs
-nifi.provenance.repository.rollover.size=100 MB
-
-# Site to Site properties
-nifi.remote.input.socket.port=9990
-nifi.remote.input.secure=true
-
-# web properties #
-nifi.web.war.directory=./target/lib
-nifi.web.http.host=
-nifi.web.http.port=
-nifi.web.https.host=nifi.nifi.apache.org
-nifi.web.https.port=8443
-nifi.web.jetty.working.directory=./target/work/jetty
-
-# security properties #
-nifi.sensitive.props.key=n2z+tTTbHuZ4V4V2||uWhdasyDXD4ZG2lMAes/vqh6u4vaz4xgL4aEbF4y/dXevqk3ulRcOwf1vc4RDQ==
-nifi.sensitive.props.key.protected=aes/gcm/256
-nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
-nifi.sensitive.props.additional.keys=nifi.ui.banner.text
-
-nifi.security.keystore=/path/to/keystore.jks
-nifi.security.keystoreType=JKS
-nifi.security.keystorePasswd=oBjT92hIGRElIGOh||MZ6uYuWNBrOA6usq/Jt3DaD2e4otNirZDytaC/w/KFe0HOkrJR03vcbo
-nifi.security.keystorePasswd.protected=aes/gcm/256
-nifi.security.keyPasswd=ac/BaE35SL/esLiJ||+ULRvRLYdIDA2VqpE0eQXDEMjaLBMG2kbKOdOwBK/hGebDKlVg==
-nifi.security.keyPasswd.protected=aes/gcm/256
-nifi.security.truststore=
-nifi.security.truststoreType=
-nifi.security.truststorePasswd=
-nifi.security.user.authorizer=
-
-# cluster common properties (cluster manager and nodes must have same values) #
-nifi.cluster.protocol.heartbeat.interval=5 sec
-nifi.cluster.protocol.is.secure=false
-nifi.cluster.protocol.socket.timeout=30 sec
-nifi.cluster.protocol.connection.handshake.timeout=45 sec
-# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured #
-nifi.cluster.protocol.use.multicast=false
-nifi.cluster.protocol.multicast.address=
-nifi.cluster.protocol.multicast.port=
-nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms
-nifi.cluster.protocol.multicast.service.locator.attempts=3
-nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec
-
-# cluster node properties (only configure for cluster nodes) #
-nifi.cluster.is.node=false
-nifi.cluster.node.address=
-nifi.cluster.node.protocol.port=
-nifi.cluster.node.protocol.threads=2
-# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx #
-nifi.cluster.node.unicast.manager.address=
-nifi.cluster.node.unicast.manager.protocol.port=
-nifi.cluster.node.unicast.manager.authority.provider.port=
-
-# cluster manager properties (only configure for cluster manager) #
-nifi.cluster.is.manager=false
-nifi.cluster.manager.address=
-nifi.cluster.manager.protocol.port=
-nifi.cluster.manager.authority.provider.port=
-nifi.cluster.manager.authority.provider.threads=10
-nifi.cluster.manager.node.firewall.file=
-nifi.cluster.manager.node.event.history.size=10
-nifi.cluster.manager.node.api.connection.timeout=30 sec
-nifi.cluster.manager.node.api.read.timeout=30 sec
-nifi.cluster.manager.node.api.request.threads=10
-nifi.cluster.manager.flow.retrieval.delay=5 sec
-nifi.cluster.manager.protocol.threads=10
-nifi.cluster.manager.safemode.duration=0 sec
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_single_malformed.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_single_malformed.properties
deleted file mode 100644
index 065f3598cf..0000000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_single_malformed.properties
+++ /dev/null
@@ -1,125 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Core Properties #
-nifi.flow.configuration.file=./target/flow.xml.gz
-nifi.flow.configuration.archive.dir=./target/archive/
-nifi.flowcontroller.autoResumeState=true
-nifi.flowcontroller.graceful.shutdown.period=10 sec
-nifi.flowservice.writedelay.interval=2 sec
-nifi.administrative.yield.duration=30 sec
-
-nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
-nifi.controller.service.configuration.file=./target/controller-services.xml
-nifi.templates.directory=./target/templates
-nifi.ui.banner.text=UI Banner Text
-nifi.ui.autorefresh.interval=30 sec
-nifi.nar.library.directory=./target/resources/NiFiProperties/lib/
-nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/
-nifi.nar.working.directory=./target/work/nar/
-
-# H2 Settings
-nifi.database.directory=./target/database_repository
-nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
-
-# FlowFile Repository
-nifi.flowfile.repository.directory=./target/test-repo
-nifi.flowfile.repository.partitions=1
-nifi.flowfile.repository.checkpoint.interval=2 mins
-nifi.queue.swap.threshold=20000
-nifi.swap.storage.directory=./target/test-repo/swap
-nifi.swap.in.period=5 sec
-nifi.swap.in.threads=1
-nifi.swap.out.period=5 sec
-nifi.swap.out.threads=4
-
-# Content Repository
-nifi.content.claim.max.appendable.size=10 MB
-nifi.content.claim.max.flow.files=100
-nifi.content.repository.directory.default=./target/content_repository
-
-# Provenance Repository Properties
-nifi.provenance.repository.storage.directory=./target/provenance_repository
-nifi.provenance.repository.max.storage.time=24 hours
-nifi.provenance.repository.max.storage.size=1 GB
-nifi.provenance.repository.rollover.time=30 secs
-nifi.provenance.repository.rollover.size=100 MB
-
-# Site to Site properties
-nifi.remote.input.socket.port=9990
-nifi.remote.input.secure=true
-
-# web properties #
-nifi.web.war.directory=./target/lib
-nifi.web.http.host=
-nifi.web.http.port=
-nifi.web.https.host=nifi.nifi.apache.org
-nifi.web.https.port=8443
-nifi.web.jetty.working.directory=./target/work/jetty
-
-# security properties #
-nifi.sensitive.props.key=n2z+tTTbHuZ4V4V2||uWhdasyDXD4ZG2lMAes/vqh6u4vaz4xgL4aEbF4Y/dXevqk3ulRcOwf1vc4RDQ==
-nifi.sensitive.props.key.protected=aes/gcm/256
-nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
-nifi.sensitive.props.additional.keys=nifi.ui.banner.text, nifi.version
-
-nifi.security.keystore=/path/to/keystore.jks
-nifi.security.keystoreType=JKS
-nifi.security.keystorePasswd=oBjT92hIGRElIGOh||MZ6uYuWNBrOA6usq/Jt3DaD2e4otNirZDytac/w/KFe0HOkrJR03
-nifi.security.keystorePasswd.protected=aes/gcm/256
-nifi.security.keyPasswd=ac/BaE35SL/esLiJ||+ULRvRLYdIDA2VqpE0eQXDEMjaLBMG2kbKOdOwBk/hGebDKlVg==
-nifi.security.keyPasswd.protected=aes/gcm/256
-nifi.security.truststore=
-nifi.security.truststoreType=
-nifi.security.truststorePasswd=
-nifi.security.user.authorizer=
-
-# cluster common properties (cluster manager and nodes must have same values) #
-nifi.cluster.protocol.heartbeat.interval=5 sec
-nifi.cluster.protocol.is.secure=false
-nifi.cluster.protocol.socket.timeout=30 sec
-nifi.cluster.protocol.connection.handshake.timeout=45 sec
-# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured #
-nifi.cluster.protocol.use.multicast=false
-nifi.cluster.protocol.multicast.address=
-nifi.cluster.protocol.multicast.port=
-nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms
-nifi.cluster.protocol.multicast.service.locator.attempts=3
-nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec
-
-# cluster node properties (only configure for cluster nodes) #
-nifi.cluster.is.node=false
-nifi.cluster.node.address=
-nifi.cluster.node.protocol.port=
-nifi.cluster.node.protocol.threads=2
-# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx #
-nifi.cluster.node.unicast.manager.address=
-nifi.cluster.node.unicast.manager.protocol.port=
-nifi.cluster.node.unicast.manager.authority.provider.port=
-
-# cluster manager properties (only configure for cluster manager) #
-nifi.cluster.is.manager=false
-nifi.cluster.manager.address=
-nifi.cluster.manager.protocol.port=
-nifi.cluster.manager.authority.provider.port=
-nifi.cluster.manager.authority.provider.threads=10
-nifi.cluster.manager.node.firewall.file=
-nifi.cluster.manager.node.event.history.size=10
-nifi.cluster.manager.node.api.connection.timeout=30 sec
-nifi.cluster.manager.node.api.read.timeout=30 sec
-nifi.cluster.manager.node.api.request.threads=10
-nifi.cluster.manager.flow.retrieval.delay=5 sec
-nifi.cluster.manager.protocol.threads=10
-nifi.cluster.manager.safemode.duration=0 sec
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_unknown.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_unknown.properties
deleted file mode 100644
index 8fcf22bd68..0000000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_unknown.properties
+++ /dev/null
@@ -1,125 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Core Properties #
-nifi.flow.configuration.file=./target/flow.xml.gz
-nifi.flow.configuration.archive.dir=./target/archive/
-nifi.flowcontroller.autoResumeState=true
-nifi.flowcontroller.graceful.shutdown.period=10 sec
-nifi.flowservice.writedelay.interval=2 sec
-nifi.administrative.yield.duration=30 sec
-
-nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
-nifi.controller.service.configuration.file=./target/controller-services.xml
-nifi.templates.directory=./target/templates
-nifi.ui.banner.text=UI Banner Text
-nifi.ui.autorefresh.interval=30 sec
-nifi.nar.library.directory=./target/resources/NiFiProperties/lib/
-nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/
-nifi.nar.working.directory=./target/work/nar/
-
-# H2 Settings
-nifi.database.directory=./target/database_repository
-nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
-
-# FlowFile Repository
-nifi.flowfile.repository.directory=./target/test-repo
-nifi.flowfile.repository.partitions=1
-nifi.flowfile.repository.checkpoint.interval=2 mins
-nifi.queue.swap.threshold=20000
-nifi.swap.storage.directory=./target/test-repo/swap
-nifi.swap.in.period=5 sec
-nifi.swap.in.threads=1
-nifi.swap.out.period=5 sec
-nifi.swap.out.threads=4
-
-# Content Repository
-nifi.content.claim.max.appendable.size=10 MB
-nifi.content.claim.max.flow.files=100
-nifi.content.repository.directory.default=./target/content_repository
-
-# Provenance Repository Properties
-nifi.provenance.repository.storage.directory=./target/provenance_repository
-nifi.provenance.repository.max.storage.time=24 hours
-nifi.provenance.repository.max.storage.size=1 GB
-nifi.provenance.repository.rollover.time=30 secs
-nifi.provenance.repository.rollover.size=100 MB
-
-# Site to Site properties
-nifi.remote.input.socket.port=9990
-nifi.remote.input.secure=true
-
-# web properties #
-nifi.web.war.directory=./target/lib
-nifi.web.http.host=
-nifi.web.http.port=
-nifi.web.https.host=nifi.nifi.apache.org
-nifi.web.https.port=8443
-nifi.web.jetty.working.directory=./target/work/jetty
-
-# security properties #
-nifi.sensitive.props.key=n2z+tTTbHuZ4V4V2||uWhdasyDXD4ZG2lMAes/vqh6u4vaz4xgL4aEbF4Y/dXevqk3ulRcOwf1vc4RDQ==
-nifi.sensitive.props.key.protected=unknown
-nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
-nifi.sensitive.props.additional.keys=nifi.ui.banner.text, nifi.version
-
-nifi.security.keystore=/path/to/keystore.jks
-nifi.security.keystoreType=JKS
-nifi.security.keystorePasswd=oBjT92hIGRElIGOh||MZ6uYuWNBrOA6usq/Jt3DaD2e4otNirZDytac/w/KFe0HOkrJR03vcbo
-nifi.security.keystorePasswd.protected=unknown
-nifi.security.keyPasswd=ac/BaE35SL/esLiJ||+ULRvRLYdIDA2VqpE0eQXDEMjaLBMG2kbKOdOwBk/hGebDKlVg==
-nifi.security.keyPasswd.protected=unknown
-nifi.security.truststore=
-nifi.security.truststoreType=
-nifi.security.truststorePasswd=
-nifi.security.user.authorizer=
-
-# cluster common properties (cluster manager and nodes must have same values) #
-nifi.cluster.protocol.heartbeat.interval=5 sec
-nifi.cluster.protocol.is.secure=false
-nifi.cluster.protocol.socket.timeout=30 sec
-nifi.cluster.protocol.connection.handshake.timeout=45 sec
-# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured #
-nifi.cluster.protocol.use.multicast=false
-nifi.cluster.protocol.multicast.address=
-nifi.cluster.protocol.multicast.port=
-nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms
-nifi.cluster.protocol.multicast.service.locator.attempts=3
-nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec
-
-# cluster node properties (only configure for cluster nodes) #
-nifi.cluster.is.node=false
-nifi.cluster.node.address=
-nifi.cluster.node.protocol.port=
-nifi.cluster.node.protocol.threads=2
-# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx #
-nifi.cluster.node.unicast.manager.address=
-nifi.cluster.node.unicast.manager.protocol.port=
-nifi.cluster.node.unicast.manager.authority.provider.port=
-
-# cluster manager properties (only configure for cluster manager) #
-nifi.cluster.is.manager=false
-nifi.cluster.manager.address=
-nifi.cluster.manager.protocol.port=
-nifi.cluster.manager.authority.provider.port=
-nifi.cluster.manager.authority.provider.threads=10
-nifi.cluster.manager.node.firewall.file=
-nifi.cluster.manager.node.event.history.size=10
-nifi.cluster.manager.node.api.connection.timeout=30 sec
-nifi.cluster.manager.node.api.read.timeout=30 sec
-nifi.cluster.manager.node.api.request.threads=10
-nifi.cluster.manager.flow.retrieval.delay=5 sec
-nifi.cluster.manager.protocol.threads=10
-nifi.cluster.manager.safemode.duration=0 sec
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_unprotected.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_unprotected.properties
deleted file mode 100644
index 7e8f2eb0a1..0000000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_unprotected.properties
+++ /dev/null
@@ -1,122 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Core Properties #
-nifi.flow.configuration.file=./target/flow.xml.gz
-nifi.flow.configuration.archive.dir=./target/archive/
-nifi.flowcontroller.autoResumeState=true
-nifi.flowcontroller.graceful.shutdown.period=10 sec
-nifi.flowservice.writedelay.interval=2 sec
-nifi.administrative.yield.duration=30 sec
-
-nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
-nifi.controller.service.configuration.file=./target/controller-services.xml
-nifi.templates.directory=./target/templates
-nifi.ui.banner.text=UI Banner Text
-nifi.ui.autorefresh.interval=30 sec
-nifi.nar.library.directory=./target/resources/NiFiProperties/lib/
-nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/
-nifi.nar.working.directory=./target/work/nar/
-
-# H2 Settings
-nifi.database.directory=./target/database_repository
-nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
-
-# FlowFile Repository
-nifi.flowfile.repository.directory=./target/test-repo
-nifi.flowfile.repository.partitions=1
-nifi.flowfile.repository.checkpoint.interval=2 mins
-nifi.queue.swap.threshold=20000
-nifi.swap.storage.directory=./target/test-repo/swap
-nifi.swap.in.period=5 sec
-nifi.swap.in.threads=1
-nifi.swap.out.period=5 sec
-nifi.swap.out.threads=4
-
-# Content Repository
-nifi.content.claim.max.appendable.size=10 MB
-nifi.content.claim.max.flow.files=100
-nifi.content.repository.directory.default=./target/content_repository
-
-# Provenance Repository Properties
-nifi.provenance.repository.storage.directory=./target/provenance_repository
-nifi.provenance.repository.max.storage.time=24 hours
-nifi.provenance.repository.max.storage.size=1 GB
-nifi.provenance.repository.rollover.time=30 secs
-nifi.provenance.repository.rollover.size=100 MB
-
-# Site to Site properties
-nifi.remote.input.socket.port=9990
-nifi.remote.input.secure=true
-
-# web properties #
-nifi.web.war.directory=./target/lib
-nifi.web.http.host=
-nifi.web.http.port=
-nifi.web.https.host=nifi.nifi.apache.org
-nifi.web.https.port=8443
-nifi.web.jetty.working.directory=./target/work/jetty
-
-# security properties #
-nifi.sensitive.props.key=thisIsABadSensitiveKeyPassword
-nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
-nifi.sensitive.props.additional.keys=nifi.ui.banner.text, nifi.version
-
-nifi.security.keystore=/path/to/keystore.jks
-nifi.security.keystoreType=JKS
-nifi.security.keystorePasswd=thisIsABadKeystorePassword
-nifi.security.keyPasswd=thisIsABadKeyPassword
-nifi.security.truststore=
-nifi.security.truststoreType=
-nifi.security.truststorePasswd=
-nifi.security.user.authorizer=
-
-# cluster common properties (cluster manager and nodes must have same values) #
-nifi.cluster.protocol.heartbeat.interval=5 sec
-nifi.cluster.protocol.is.secure=false
-nifi.cluster.protocol.socket.timeout=30 sec
-nifi.cluster.protocol.connection.handshake.timeout=45 sec
-# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured #
-nifi.cluster.protocol.use.multicast=false
-nifi.cluster.protocol.multicast.address=
-nifi.cluster.protocol.multicast.port=
-nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms
-nifi.cluster.protocol.multicast.service.locator.attempts=3
-nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec
-
-# cluster node properties (only configure for cluster nodes) #
-nifi.cluster.is.node=false
-nifi.cluster.node.address=
-nifi.cluster.node.protocol.port=
-nifi.cluster.node.protocol.threads=2
-# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx #
-nifi.cluster.node.unicast.manager.address=
-nifi.cluster.node.unicast.manager.protocol.port=
-nifi.cluster.node.unicast.manager.authority.provider.port=
-
-# cluster manager properties (only configure for cluster manager) #
-nifi.cluster.is.manager=false
-nifi.cluster.manager.address=
-nifi.cluster.manager.protocol.port=
-nifi.cluster.manager.authority.provider.port=
-nifi.cluster.manager.authority.provider.threads=10
-nifi.cluster.manager.node.firewall.file=
-nifi.cluster.manager.node.event.history.size=10
-nifi.cluster.manager.node.api.connection.timeout=30 sec
-nifi.cluster.manager.node.api.read.timeout=30 sec
-nifi.cluster.manager.node.api.request.threads=10
-nifi.cluster.manager.flow.retrieval.delay=5 sec
-nifi.cluster.manager.protocol.threads=10
-nifi.cluster.manager.safemode.duration=0 sec
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_unprotected_extra_line.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_unprotected_extra_line.properties
deleted file mode 100644
index 2263b0e90f..0000000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_unprotected_extra_line.properties
+++ /dev/null
@@ -1,123 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Core Properties #
-nifi.flow.configuration.file=./target/flow.xml.gz
-nifi.flow.configuration.archive.dir=./target/archive/
-nifi.flowcontroller.autoResumeState=true
-nifi.flowcontroller.graceful.shutdown.period=10 sec
-nifi.flowservice.writedelay.interval=2 sec
-nifi.administrative.yield.duration=30 sec
-
-nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
-nifi.controller.service.configuration.file=./target/controller-services.xml
-nifi.templates.directory=./target/templates
-nifi.ui.banner.text=UI Banner Text
-nifi.ui.autorefresh.interval=30 sec
-nifi.nar.library.directory=./target/resources/NiFiProperties/lib/
-nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/
-nifi.nar.working.directory=./target/work/nar/
-
-# H2 Settings
-nifi.database.directory=./target/database_repository
-nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
-
-# FlowFile Repository
-nifi.flowfile.repository.directory=./target/test-repo
-nifi.flowfile.repository.partitions=1
-nifi.flowfile.repository.checkpoint.interval=2 mins
-nifi.queue.swap.threshold=20000
-nifi.swap.storage.directory=./target/test-repo/swap
-nifi.swap.in.period=5 sec
-nifi.swap.in.threads=1
-nifi.swap.out.period=5 sec
-nifi.swap.out.threads=4
-
-# Content Repository
-nifi.content.claim.max.appendable.size=10 MB
-nifi.content.claim.max.flow.files=100
-nifi.content.repository.directory.default=./target/content_repository
-
-# Provenance Repository Properties
-nifi.provenance.repository.storage.directory=./target/provenance_repository
-nifi.provenance.repository.max.storage.time=24 hours
-nifi.provenance.repository.max.storage.size=1 GB
-nifi.provenance.repository.rollover.time=30 secs
-nifi.provenance.repository.rollover.size=100 MB
-
-# Site to Site properties
-nifi.remote.input.socket.port=9990
-nifi.remote.input.secure=true
-
-# web properties #
-nifi.web.war.directory=./target/lib
-nifi.web.http.host=
-nifi.web.http.port=
-nifi.web.https.host=nifi.nifi.apache.org
-nifi.web.https.port=8443
-nifi.web.jetty.working.directory=./target/work/jetty
-
-# security properties #
-nifi.sensitive.props.key=thisIsABadSensitiveKeyPassword
-nifi.sensitive.props.key.protected=
-nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
-nifi.sensitive.props.additional.keys=nifi.ui.banner.text, nifi.version
-
-nifi.security.keystore=/path/to/keystore.jks
-nifi.security.keystoreType=JKS
-nifi.security.keystorePasswd=thisIsABadKeystorePassword
-nifi.security.keyPasswd=thisIsABadKeyPassword
-nifi.security.truststore=
-nifi.security.truststoreType=
-nifi.security.truststorePasswd=
-nifi.security.user.authorizer=
-
-# cluster common properties (cluster manager and nodes must have same values) #
-nifi.cluster.protocol.heartbeat.interval=5 sec
-nifi.cluster.protocol.is.secure=false
-nifi.cluster.protocol.socket.timeout=30 sec
-nifi.cluster.protocol.connection.handshake.timeout=45 sec
-# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured #
-nifi.cluster.protocol.use.multicast=false
-nifi.cluster.protocol.multicast.address=
-nifi.cluster.protocol.multicast.port=
-nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms
-nifi.cluster.protocol.multicast.service.locator.attempts=3
-nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec
-
-# cluster node properties (only configure for cluster nodes) #
-nifi.cluster.is.node=false
-nifi.cluster.node.address=
-nifi.cluster.node.protocol.port=
-nifi.cluster.node.protocol.threads=2
-# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx #
-nifi.cluster.node.unicast.manager.address=
-nifi.cluster.node.unicast.manager.protocol.port=
-nifi.cluster.node.unicast.manager.authority.provider.port=
-
-# cluster manager properties (only configure for cluster manager) #
-nifi.cluster.is.manager=false
-nifi.cluster.manager.address=
-nifi.cluster.manager.protocol.port=
-nifi.cluster.manager.authority.provider.port=
-nifi.cluster.manager.authority.provider.threads=10
-nifi.cluster.manager.node.firewall.file=
-nifi.cluster.manager.node.event.history.size=10
-nifi.cluster.manager.node.api.connection.timeout=30 sec
-nifi.cluster.manager.node.api.read.timeout=30 sec
-nifi.cluster.manager.node.api.request.threads=10
-nifi.cluster.manager.flow.retrieval.delay=5 sec
-nifi.cluster.manager.protocol.threads=10
-nifi.cluster.manager.safemode.duration=0 sec
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_whitespace.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_whitespace.properties
deleted file mode 100644
index 420752eb0b..0000000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_whitespace.properties
+++ /dev/null
@@ -1,24 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# properties with whitespace
-nifi.whitespace.propWithNoSpace=foo
-nifi.whitespace.propWithLeadingSpace= foo
-nifi.whitespace.propWithTrailingSpace=foo
-nifi.whitespace.propWithLeadingAndTrailingSpace= foo
-nifi.whitespace.propWithTrailingTab=foo\t
-nifi.whitespace.propWithMultipleLines=foo\
- bar\
- baz
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_bootstrap/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/properties/conf/empty.nifi.properties
similarity index 100%
copy from nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_bootstrap/nifi.properties
copy to nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/properties/conf/empty.nifi.properties
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_bootstrap/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/properties/conf/flow.nifi.properties
similarity index 88%
copy from nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_bootstrap/nifi.properties
copy to nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/properties/conf/flow.nifi.properties
index ae1e83eeb3..77241d5bee 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_bootstrap/nifi.properties
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/properties/conf/flow.nifi.properties
@@ -12,3 +12,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
+nifi.flow.configuration.file=./conf/flow.xml.gz
+nifi.flow.configuration.json.file=./conf/flow.json.gz
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_bootstrap/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/properties/conf/protected.nifi.properties
similarity index 76%
rename from nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_bootstrap/nifi.properties
rename to nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/properties/conf/protected.nifi.properties
index ae1e83eeb3..44fd450c3d 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_bootstrap/nifi.properties
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/properties/conf/protected.nifi.properties
@@ -12,3 +12,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
+nifi.flow.configuration.file=./conf/flow.xml.gz
+nifi.flow.configuration.json.file=./conf/flow.json.gz
+
+nifi.security.keystorePasswd=Ycac+pAe3AdHbCAC||ImnArC6KZJ+WXDMwJw2cjpOpNJFk1s5XyfbkrB8=
+nifi.security.keystorePasswd.protected=aes/gcm/128
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/pom.xml
index b58019648f..aeed555ed6 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/pom.xml
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/pom.xml
@@ -57,6 +57,21 @@
<artifactId>logback-classic</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-property-protection-loader</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-property-protection-factory</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/unreadable_bootstrap/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/bootstrap.conf
similarity index 89%
rename from nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/unreadable_bootstrap/nifi.properties
rename to nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/bootstrap.conf
index ae1e83eeb3..e743fadac0 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/unreadable_bootstrap/nifi.properties
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/bootstrap.conf
@@ -12,3 +12,5 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
+nifi.bootstrap.sensitive.key=0000000000000000000000000000000000000000000000000000000000000000
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/pom.xml
index 6cf7b7e347..30a5dfe759 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/pom.xml
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/pom.xml
@@ -161,6 +161,14 @@
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-framework-authorization</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-property-protection-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-property-protection-loader</artifactId>
+ </dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk15on</artifactId>
@@ -239,6 +247,11 @@
<artifactId>jackson-datatype-jsr310</artifactId>
<version>${jackson.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-property-protection-factory</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.codehaus.jettison</groupId>
<artifactId>jettison</artifactId>
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/spring/LoginIdentityProviderFactoryBean.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/spring/LoginIdentityProviderFactoryBean.java
index f8c9967d9e..6861603e8d 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/spring/LoginIdentityProviderFactoryBean.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/spring/LoginIdentityProviderFactoryBean.java
@@ -32,7 +32,14 @@ import org.apache.nifi.authentication.generated.Provider;
import org.apache.nifi.bundle.Bundle;
import org.apache.nifi.nar.ExtensionManager;
import org.apache.nifi.nar.NarCloseable;
-import org.apache.nifi.properties.SensitivePropertyProviderFactoryAware;
+import org.apache.nifi.properties.ProtectedPropertyContext;
+import org.apache.nifi.properties.SensitivePropertyProvider;
+import org.apache.nifi.properties.SensitivePropertyProviderFactory;
+import org.apache.nifi.properties.scheme.ProtectionScheme;
+import org.apache.nifi.properties.scheme.ProtectionSchemeResolver;
+import org.apache.nifi.property.protection.loader.PropertyProtectionURLClassLoader;
+import org.apache.nifi.property.protection.loader.PropertyProviderFactoryLoader;
+import org.apache.nifi.property.protection.loader.ProtectionSchemeResolverLoader;
import org.apache.nifi.util.NiFiProperties;
import org.apache.nifi.xml.processing.stream.StandardXMLStreamReaderProvider;
import org.apache.nifi.xml.processing.stream.XMLStreamReaderProvider;
@@ -59,10 +66,9 @@ import java.util.List;
import java.util.Map;
/**
- *
+ * Spring Factory Bean implementation requires a generic Object return type to handle a null Provider configuration
*/
-public class LoginIdentityProviderFactoryBean extends SensitivePropertyProviderFactoryAware
- implements FactoryBean, DisposableBean, LoginIdentityProviderLookup {
+public class LoginIdentityProviderFactoryBean implements FactoryBean<Object>, DisposableBean, LoginIdentityProviderLookup {
private static final String LOGIN_IDENTITY_PROVIDERS_XSD = "/login-identity-providers.xsd";
private static final String JAXB_GENERATED_PATH = "org.apache.nifi.authentication.generated";
@@ -94,6 +100,12 @@ public class LoginIdentityProviderFactoryBean extends SensitivePropertyProviderF
return loginIdentityProviders.get(identifier);
}
+ /**
+ * Get Login Identity Provider Object or null when not configured
+ *
+ * @return Login Identity Provider instance or null when not configured
+ * @throws Exception Thrown on configuration failures
+ */
@Override
public Object getObject() throws Exception {
if (loginIdentityProvider == null) {
@@ -109,11 +121,7 @@ public class LoginIdentityProviderFactoryBean extends SensitivePropertyProviderF
loginIdentityProviders.put(provider.getIdentifier(), createLoginIdentityProvider(provider.getIdentifier(), provider.getClazz()));
}
- // configure each login identity provider
- for (final Provider provider : loginIdentityProviderConfiguration.getProvider()) {
- final LoginIdentityProvider instance = loginIdentityProviders.get(provider.getIdentifier());
- instance.onConfigured(loadLoginIdentityProviderConfiguration(provider));
- }
+ loadProviderProperties(loginIdentityProviderConfiguration);
// get the login identity provider instance
loginIdentityProvider = getLoginIdentityProvider(loginIdentityProviderIdentifier);
@@ -181,8 +189,8 @@ public class LoginIdentityProviderFactoryBean extends SensitivePropertyProviderF
Class<? extends LoginIdentityProvider> loginIdentityProviderClass = rawLoginIdentityProviderClass.asSubclass(LoginIdentityProvider.class);
// otherwise create a new instance
- Constructor constructor = loginIdentityProviderClass.getConstructor();
- instance = (LoginIdentityProvider) constructor.newInstance();
+ Constructor<? extends LoginIdentityProvider> constructor = loginIdentityProviderClass.getConstructor();
+ instance = constructor.newInstance();
// method injection
performMethodInjection(instance, loginIdentityProviderClass);
@@ -201,23 +209,66 @@ public class LoginIdentityProviderFactoryBean extends SensitivePropertyProviderF
return withNarLoader(instance);
}
- private LoginIdentityProviderConfigurationContext loadLoginIdentityProviderConfiguration(final Provider provider) {
+ private void loadProviderProperties(final LoginIdentityProviders loginIdentityProviderConfiguration) {
+ final ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
+
+ try {
+ final PropertyProtectionURLClassLoader protectionClassLoader = new PropertyProtectionURLClassLoader(contextClassLoader);
+ Thread.currentThread().setContextClassLoader(protectionClassLoader);
+
+ final ProtectionSchemeResolverLoader resolverLoader = new ProtectionSchemeResolverLoader();
+ final ProtectionSchemeResolver protectionSchemeResolver = resolverLoader.getProtectionSchemeResolver();
+
+ final PropertyProviderFactoryLoader factoryLoader = new PropertyProviderFactoryLoader();
+ final SensitivePropertyProviderFactory sensitivePropertyProviderFactory = factoryLoader.getPropertyProviderFactory();
+
+ for (final Provider provider : loginIdentityProviderConfiguration.getProvider()) {
+ final LoginIdentityProvider instance = loginIdentityProviders.get(provider.getIdentifier());
+ final LoginIdentityProviderConfigurationContext configurationContext = getConfigurationContext(provider, sensitivePropertyProviderFactory, protectionSchemeResolver);
+ instance.onConfigured(configurationContext);
+ }
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ private LoginIdentityProviderConfigurationContext getConfigurationContext(
+ final Provider provider,
+ final SensitivePropertyProviderFactory sensitivePropertyProviderFactory,
+ final ProtectionSchemeResolver protectionSchemeResolver
+ ) {
+ final String providerIdentifier = provider.getIdentifier();
final Map<String, String> providerProperties = new HashMap<>();
for (final Property property : provider.getProperty()) {
- if (!StringUtils.isBlank(property.getEncryption())) {
- String decryptedValue = decryptValue(property.getValue(), property.getEncryption(), property.getName(), provider
- .getIdentifier());
- providerProperties.put(property.getName(), decryptedValue);
- } else {
+ final String encryption = property.getEncryption();
+
+ if (StringUtils.isBlank(encryption)) {
providerProperties.put(property.getName(), property.getValue());
+ } else {
+ final String propertyDecrypted = getPropertyDecrypted(providerIdentifier, property, sensitivePropertyProviderFactory, protectionSchemeResolver);
+ providerProperties.put(property.getName(), propertyDecrypted);
}
}
- return new StandardLoginIdentityProviderConfigurationContext(provider.getIdentifier(), providerProperties);
+ return new StandardLoginIdentityProviderConfigurationContext(providerIdentifier, providerProperties);
+ }
+
+ private String getPropertyDecrypted(
+ final String providerIdentifier,
+ final Property property,
+ final SensitivePropertyProviderFactory sensitivePropertyProviderFactory,
+ final ProtectionSchemeResolver protectionSchemeResolver
+ ) {
+ final String scheme = property.getEncryption();
+ final ProtectionScheme protectionScheme = protectionSchemeResolver.getProtectionScheme(scheme);
+ final SensitivePropertyProvider propertyProvider = sensitivePropertyProviderFactory.getProvider(protectionScheme);
+ final ProtectedPropertyContext protectedPropertyContext = sensitivePropertyProviderFactory.getPropertyContext(providerIdentifier, property.getName());
+ final String protectedProperty = property.getValue();
+ return propertyProvider.unprotect(protectedProperty, protectedPropertyContext);
}
- private void performMethodInjection(final LoginIdentityProvider instance, final Class loginIdentityProviderClass)
+ private void performMethodInjection(final LoginIdentityProvider instance, final Class<?> loginIdentityProviderClass)
throws IllegalAccessException, IllegalArgumentException, InvocationTargetException {
for (final Method method : loginIdentityProviderClass.getMethods()) {
@@ -245,13 +296,13 @@ public class LoginIdentityProviderFactoryBean extends SensitivePropertyProviderF
}
}
- final Class parentClass = loginIdentityProviderClass.getSuperclass();
+ final Class<?> parentClass = loginIdentityProviderClass.getSuperclass();
if (parentClass != null && LoginIdentityProvider.class.isAssignableFrom(parentClass)) {
performMethodInjection(instance, parentClass);
}
}
- private void performFieldInjection(final LoginIdentityProvider instance, final Class loginIdentityProviderClass) throws IllegalArgumentException, IllegalAccessException {
+ private void performFieldInjection(final LoginIdentityProvider instance, final Class<?> loginIdentityProviderClass) throws IllegalArgumentException, IllegalAccessException {
for (final Field field : loginIdentityProviderClass.getDeclaredFields()) {
if (field.isAnnotationPresent(LoginIdentityProviderContext.class)) {
// make the method accessible
@@ -277,7 +328,7 @@ public class LoginIdentityProviderFactoryBean extends SensitivePropertyProviderF
}
}
- final Class parentClass = loginIdentityProviderClass.getSuperclass();
+ final Class<?> parentClass = loginIdentityProviderClass.getSuperclass();
if (parentClass != null && LoginIdentityProvider.class.isAssignableFrom(parentClass)) {
performFieldInjection(instance, parentClass);
}
@@ -317,7 +368,7 @@ public class LoginIdentityProviderFactoryBean extends SensitivePropertyProviderF
}
@Override
- public Class getObjectType() {
+ public Class<?> getObjectType() {
return LoginIdentityProvider.class;
}
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/test/groovy/org/apache/nifi/web/security/spring/LoginIdentityProviderFactoryBeanTest.groovy b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/test/groovy/org/apache/nifi/web/security/spring/LoginIdentityProviderFactoryBeanTest.groovy
deleted file mode 100644
index 79a55f1614..0000000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/test/groovy/org/apache/nifi/web/security/spring/LoginIdentityProviderFactoryBeanTest.groovy
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.web.security.spring
-
-import org.apache.nifi.authentication.generated.Property
-import org.apache.nifi.authentication.generated.Provider
-import org.bouncycastle.jce.provider.BouncyCastleProvider
-import org.junit.Before
-import org.junit.BeforeClass
-import org.junit.Test
-import org.junit.runner.RunWith
-import org.junit.runners.JUnit4
-import org.slf4j.Logger
-import org.slf4j.LoggerFactory
-
-import javax.crypto.Cipher
-import java.security.Security
-
-@RunWith(JUnit4.class)
-class LoginIdentityProviderFactoryBeanTest extends GroovyTestCase {
- private static final Logger logger = LoggerFactory.getLogger(LoginIdentityProviderFactoryBeanTest.class)
-
- // These blocks configure the constant values depending on JCE policies of the machine running the tests
- private static final String KEY_HEX_128 = "0123456789ABCDEFFEDCBA9876543210"
- private static final String KEY_HEX_256 = KEY_HEX_128 * 2
- public static final String KEY_HEX = isUnlimitedStrengthCryptoAvailable() ? KEY_HEX_256 : KEY_HEX_128
-
- private static final String CIPHER_TEXT_128 = "6pqdM1urBEPHtj+L||ds0Z7RpqOA2321c/+7iPMfxDrqmH5Qx6UwQG0eIYB//3Ng"
- private static final String CIPHER_TEXT_256 = "TepMCD7v3LAMF0KX||ydSRWPRl1/JXgTsZtfzCnDXu7a0lTLysjPL2I06EPUCHzw"
- public static final String CIPHER_TEXT = isUnlimitedStrengthCryptoAvailable() ? CIPHER_TEXT_256 : CIPHER_TEXT_128
-
- private static final String ENCRYPTION_SCHEME_128 = "aes/gcm/128"
- private static final String ENCRYPTION_SCHEME_256 = "aes/gcm/256"
- public static
- final String ENCRYPTION_SCHEME = isUnlimitedStrengthCryptoAvailable() ? ENCRYPTION_SCHEME_256 : ENCRYPTION_SCHEME_128
-
- private static final String PASSWORD = "thisIsABadPassword"
-
- private LoginIdentityProviderFactoryBean bean
-
- @BeforeClass
- static void setUpOnce() throws Exception {
- Security.addProvider(new BouncyCastleProvider())
-
- logger.metaClass.methodMissing = { String name, args ->
- logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}")
- }
- }
-
- @Before
- void setUp() {
- bean = new LoginIdentityProviderFactoryBean()
- bean.configureSensitivePropertyProviderFactory(KEY_HEX, null)
- }
-
- private static boolean isUnlimitedStrengthCryptoAvailable() {
- Cipher.getMaxAllowedKeyLength("AES") > 128
- }
-
- @Test
- void testShouldDecryptValue() {
- // Arrange
- logger.info("Encryption scheme: ${ENCRYPTION_SCHEME}")
- logger.info("Cipher text: ${CIPHER_TEXT}")
-
- // Act
- String decrypted = bean.decryptValue(CIPHER_TEXT, ENCRYPTION_SCHEME, "propertyName", "ldap-provider")
- logger.info("Decrypted ${CIPHER_TEXT} -> ${decrypted}")
-
- // Assert
- assert decrypted == PASSWORD
- }
-
- @Test
- void testShouldLoadEncryptedLoginIdentityProviderConfiguration() {
- // Arrange
- Provider encryptedProvider = new Provider()
- encryptedProvider.identifier ="ldap-provider"
- encryptedProvider.clazz = "org.apache.nifi.ldap.LdapProvider"
- def managerPasswordName = "Manager Password"
- Property managerPasswordProperty = new Property(name: managerPasswordName, value: CIPHER_TEXT, encryption: ENCRYPTION_SCHEME)
- encryptedProvider.property = [managerPasswordProperty]
-
- logger.info("Manager Password property: ${managerPasswordProperty.dump()}")
-
- // Act
- def context = bean.loadLoginIdentityProviderConfiguration(encryptedProvider)
- logger.info("Loaded context: ${context.dump()}")
-
- // Assert
- assert context.getProperty(managerPasswordName) == PASSWORD
- }
-}
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/test/java/org/apache/nifi/web/security/spring/LoginIdentityProviderFactoryBeanTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/test/java/org/apache/nifi/web/security/spring/LoginIdentityProviderFactoryBeanTest.java
new file mode 100644
index 0000000000..c72deeb41b
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/test/java/org/apache/nifi/web/security/spring/LoginIdentityProviderFactoryBeanTest.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.security.spring;
+
+import org.apache.nifi.authentication.LoginIdentityProvider;
+import org.apache.nifi.bundle.Bundle;
+import org.apache.nifi.nar.ExtensionManager;
+import org.apache.nifi.util.NiFiProperties;
+import org.apache.nifi.web.security.spring.mock.MockLoginIdentityProvider;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import java.io.File;
+import java.net.URL;
+import java.util.Collections;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.when;
+
+@ExtendWith(MockitoExtension.class)
+class LoginIdentityProviderFactoryBeanTest {
+ private static final String PROVIDERS_PATH = "/login-identity-providers.xml";
+
+ private static final String PROVIDER_ID = "login-identity-provider";
+
+ @Mock
+ NiFiProperties properties;
+
+ @Mock
+ ExtensionManager extensionManager;
+
+ @Mock
+ Bundle bundle;
+
+ @Test
+ void testGetObjectNotConfigured() throws Exception {
+ final LoginIdentityProviderFactoryBean bean = new LoginIdentityProviderFactoryBean();
+
+ bean.setProperties(properties);
+ bean.setExtensionManager(extensionManager);
+
+ final Object object = bean.getObject();
+
+ assertNull(object);
+ }
+
+ @Test
+ void testGetObject() throws Exception {
+ when(properties.getProperty(eq(NiFiProperties.SECURITY_USER_LOGIN_IDENTITY_PROVIDER))).thenReturn(PROVIDER_ID);
+ when(properties.getLoginIdentityProviderConfigurationFile()).thenReturn(getLoginIdentityProvidersFile());
+
+ when(bundle.getClassLoader()).thenReturn(getClass().getClassLoader());
+ final List<Bundle> bundles = Collections.singletonList(bundle);
+
+ when(extensionManager.getBundles(eq(MockLoginIdentityProvider.class.getName()))).thenReturn(bundles);
+
+ final LoginIdentityProviderFactoryBean bean = new LoginIdentityProviderFactoryBean();
+
+ bean.setProperties(properties);
+ bean.setExtensionManager(extensionManager);
+
+ final Object object = bean.getObject();
+
+ assertInstanceOf(LoginIdentityProvider.class, object);
+ }
+
+ private File getLoginIdentityProvidersFile() {
+ final URL url = LoginIdentityProviderFactoryBeanTest.class.getResource(PROVIDERS_PATH);
+ if (url == null) {
+ throw new IllegalStateException(String.format("Providers [%s] not found", PROVIDERS_PATH));
+ }
+ return new File(url.getPath());
+ }
+}
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/test/java/org/apache/nifi/web/security/spring/mock/MockLoginIdentityProvider.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/test/java/org/apache/nifi/web/security/spring/mock/MockLoginIdentityProvider.java
new file mode 100644
index 0000000000..ef3e438ed2
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/test/java/org/apache/nifi/web/security/spring/mock/MockLoginIdentityProvider.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.security.spring.mock;
+
+import org.apache.nifi.authentication.AuthenticationResponse;
+import org.apache.nifi.authentication.LoginCredentials;
+import org.apache.nifi.authentication.LoginIdentityProvider;
+import org.apache.nifi.authentication.LoginIdentityProviderConfigurationContext;
+import org.apache.nifi.authentication.LoginIdentityProviderInitializationContext;
+import org.apache.nifi.authentication.exception.IdentityAccessException;
+import org.apache.nifi.authentication.exception.InvalidLoginCredentialsException;
+import org.apache.nifi.authentication.exception.ProviderCreationException;
+import org.apache.nifi.authentication.exception.ProviderDestructionException;
+
+public class MockLoginIdentityProvider implements LoginIdentityProvider {
+ @Override
+ public AuthenticationResponse authenticate(final LoginCredentials credentials) throws InvalidLoginCredentialsException, IdentityAccessException {
+ return null;
+ }
+
+ @Override
+ public void initialize(LoginIdentityProviderInitializationContext initializationContext) throws ProviderCreationException {
+
+ }
+
+ @Override
+ public void onConfigured(LoginIdentityProviderConfigurationContext configurationContext) throws ProviderCreationException {
+
+ }
+
+ @Override
+ public void preDestruction() throws ProviderDestructionException {
+
+ }
+}
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/test/resources/login-identity-providers.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/test/resources/login-identity-providers.xml
new file mode 100644
index 0000000000..6739b02212
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/test/resources/login-identity-providers.xml
@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<loginIdentityProviders>
+ <provider>
+ <identifier>login-identity-provider</identifier>
+ <class>org.apache.nifi.web.security.spring.mock.MockLoginIdentityProvider</class>
+ <property name="strategy">MOCK</property>
+ </provider>
+</loginIdentityProviders>
diff --git a/nifi-nar-bundles/nifi-framework-bundle/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/pom.xml
index d6870af16c..eeeb1de35b 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/pom.xml
+++ b/nifi-nar-bundles/nifi-framework-bundle/pom.xml
@@ -155,6 +155,11 @@
<artifactId>nifi-property-protection-api</artifactId>
<version>1.16.1-SNAPSHOT</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-property-protection-loader</artifactId>
+ <version>1.17.0-SNAPSHOT</version>
+ </dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-property-protection-factory</artifactId>
diff --git a/nifi-registry/nifi-registry-core/nifi-registry-properties-loader/pom.xml b/nifi-registry/nifi-registry-core/nifi-registry-properties-loader/pom.xml
index 21127df36f..411279a72a 100644
--- a/nifi-registry/nifi-registry-core/nifi-registry-properties-loader/pom.xml
+++ b/nifi-registry/nifi-registry-core/nifi-registry-properties-loader/pom.xml
@@ -50,6 +50,10 @@
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-property-protection-factory</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-property-protection-loader</artifactId>
+ </dependency>
</dependencies>
diff --git a/nifi-registry/nifi-registry-core/nifi-registry-web-api/src/test/java/org/apache/nifi/registry/web/api/SecureLdapIT.java b/nifi-registry/nifi-registry-core/nifi-registry-web-api/src/test/java/org/apache/nifi/registry/web/api/SecureLdapIT.java
deleted file mode 100644
index 92a1a28e58..0000000000
--- a/nifi-registry/nifi-registry-core/nifi-registry-web-api/src/test/java/org/apache/nifi/registry/web/api/SecureLdapIT.java
+++ /dev/null
@@ -1,813 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.registry.web.api;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.nifi.properties.SensitivePropertyProviderFactory;
-import org.apache.nifi.properties.StandardSensitivePropertyProviderFactory;
-import org.apache.nifi.registry.SecureLdapTestApiApplication;
-import org.apache.nifi.registry.authorization.AccessPolicy;
-import org.apache.nifi.registry.authorization.AccessPolicySummary;
-import org.apache.nifi.registry.authorization.CurrentUser;
-import org.apache.nifi.registry.authorization.Permissions;
-import org.apache.nifi.registry.authorization.Tenant;
-import org.apache.nifi.registry.bucket.Bucket;
-import org.apache.nifi.registry.client.AccessClient;
-import org.apache.nifi.registry.client.NiFiRegistryClient;
-import org.apache.nifi.registry.client.NiFiRegistryClientConfig;
-import org.apache.nifi.registry.client.NiFiRegistryException;
-import org.apache.nifi.registry.client.RequestConfig;
-import org.apache.nifi.registry.client.UserClient;
-import org.apache.nifi.registry.client.impl.JerseyNiFiRegistryClient;
-import org.apache.nifi.registry.client.impl.request.BearerTokenRequestConfig;
-import org.apache.nifi.registry.extension.ExtensionManager;
-import org.apache.nifi.registry.properties.NiFiRegistryProperties;
-import org.apache.nifi.registry.revision.entity.RevisionInfo;
-import org.apache.nifi.registry.security.authorization.Authorizer;
-import org.apache.nifi.registry.security.authorization.AuthorizerFactory;
-import org.apache.nifi.registry.security.crypto.BootstrapFileCryptoKeyProvider;
-import org.apache.nifi.registry.security.crypto.CryptoKeyProvider;
-import org.apache.nifi.registry.security.identity.IdentityMapper;
-import org.apache.nifi.registry.service.RegistryService;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.skyscreamer.jsonassert.JSONAssert;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.boot.test.context.TestConfiguration;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.DependsOn;
-import org.springframework.context.annotation.Import;
-import org.springframework.context.annotation.Primary;
-import org.springframework.context.annotation.Profile;
-import org.springframework.test.context.jdbc.Sql;
-import org.springframework.test.context.junit4.SpringRunner;
-
-import javax.sql.DataSource;
-import javax.ws.rs.client.Entity;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Base64;
-import java.util.List;
-import java.util.Set;
-import java.util.UUID;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Deploy the Web API Application using an embedded Jetty Server for local integration testing, with the follow characteristics:
- *
- * - A NiFiRegistryProperties has to be explicitly provided to the ApplicationContext using a profile unique to this test suite.
- * - A NiFiRegistryClientConfig has been configured to create a client capable of completing one-way TLS
- * - The database is embed H2 using volatile (in-memory) persistence
- * - Custom SQL is clearing the DB before each test method by default, unless method overrides this behavior
- */
-@RunWith(SpringRunner.class)
-@SpringBootTest(
- classes = SecureLdapTestApiApplication.class,
- webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT,
- properties = "spring.profiles.include=ITSecureLdap")
-@Import(SecureITClientConfiguration.class)
-@Sql(executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD, scripts = "classpath:db/clearDB.sql")
-public class SecureLdapIT extends IntegrationTestBase {
-
- private static Logger LOGGER = LoggerFactory.getLogger(SecureLdapIT.class);
-
- private static final String tokenLoginPath = "access/token/login";
- private static final String tokenIdentityProviderPath = "access/token/identity-provider";
- // A JWT signed by a key of 'secret'
- private static final String SIGNED_BY_WRONG_KEY = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9" +
- ".eyJzdWIiOiJuaWZpYWRtaW4iLCJpc3MiOiJMZGFwSWRlbnRpdHlQcm92aWRlciIsImF1ZCI6IkxkYXB" +
- "JZGVudGl0eVByb3ZpZGVyIiwicHJlZmVycmVkX3VzZXJuYW1lIjoibmlmaWFkbWluIiwia2lkIjoiNDd" +
- "lMjA1NzctY2I3Yi00M2MzLWFhOGYtZjI0ZDcyODQ3MDEwIiwiaWF0IjoxNTgxNTI5NTA1LCJleHAiOjE" +
- "1ODE1NzI3MDV9.vvMpwLJt1w_6Id_tlS1knxTkJ2gv7_j5ySG6PmNjF0s";
-
- @TestConfiguration
- @Profile("ITSecureLdap")
- public static class LdapTestConfiguration {
-
- static AuthorizerFactory authorizerFactory;
-
- @Primary
- @Bean
- @DependsOn({"directoryServer"}) // Can't load LdapUserGroupProvider until the embedded LDAP server, which creates the "directoryServer" bean, is running
- public static Authorizer getAuthorizer(
- @Autowired NiFiRegistryProperties properties,
- ExtensionManager extensionManager,
- RegistryService registryService,
- DataSource dataSource,
- IdentityMapper identityMapper) throws Exception {
-
- if (authorizerFactory == null) {
- authorizerFactory = new AuthorizerFactory(
- properties,
- extensionManager,
- sensitivePropertyProviderFactory(),
- registryService,
- dataSource,
- identityMapper);
- }
- return authorizerFactory.getAuthorizer();
- }
-
- @Primary
- @Bean
- public static SensitivePropertyProviderFactory sensitivePropertyProviderFactory() throws Exception {
- return StandardSensitivePropertyProviderFactory.withKey(getNiFiRegistryMasterKeyProvider().getKey());
- }
-
- private static CryptoKeyProvider getNiFiRegistryMasterKeyProvider() {
- return new BootstrapFileCryptoKeyProvider("src/test/resources/conf/secure-ldap/bootstrap.conf");
- }
-
- }
-
- private String adminAuthToken;
- private List<AccessPolicy> beforeTestAccessPoliciesSnapshot;
-
- @Before
- public void setup() {
- final String basicAuthCredentials = encodeCredentialsForBasicAuth("nifiadmin", "password");
- final String token = client
- .target(createURL(tokenIdentityProviderPath))
- .request()
- .header("Authorization", "Basic " + basicAuthCredentials)
- .post(null, String.class);
- adminAuthToken = token;
-
- beforeTestAccessPoliciesSnapshot = createAccessPoliciesSnapshot();
- }
-
- @After
- public void cleanup() {
- restoreAccessPoliciesSnapshot(beforeTestAccessPoliciesSnapshot);
- }
-
- @Test
- public void testTokenGenerationAndAccessStatus() throws Exception {
-
- // Note: this test intentionally does not use the token generated
- // for nifiadmin by the @Before method
-
- // Given: the client and server have been configured correctly for LDAP authentication
- String expectedJwtPayloadJson = "{" +
- "\"sub\":\"nobel\"," +
- "\"preferred_username\":\"nobel\"," +
- "\"iss\":\"LdapIdentityProvider\"" +
- "}";
- String expectedAccessStatusJson = "{" +
- "\"identity\":\"nobel\"," +
- "\"anonymous\":false" +
- "}";
-
- // When: the /access/token/login endpoint is queried
- final String basicAuthCredentials = encodeCredentialsForBasicAuth("nobel", "password");
- final Response tokenResponse = client
- .target(createURL(tokenIdentityProviderPath))
- .request()
- .header("Authorization", "Basic " + basicAuthCredentials)
- .post(null, Response.class);
-
- // Then: the server returns 200 OK with an access token
- assertEquals(201, tokenResponse.getStatus());
- String token = tokenResponse.readEntity(String.class);
- assertTrue(StringUtils.isNotEmpty(token));
- String[] jwtParts = token.split("\\.");
- assertEquals(3, jwtParts.length);
- String jwtPayload = new String(Base64.getDecoder().decode(jwtParts[1]), "UTF-8");
- JSONAssert.assertEquals(expectedJwtPayloadJson, jwtPayload, false);
-
- // When: the token is returned in the Authorization header
- final Response accessResponse = client
- .target(createURL("access"))
- .request()
- .header("Authorization", "Bearer " + token)
- .get(Response.class);
-
- // Then: the server acknowledges the client has access
- assertEquals(200, accessResponse.getStatus());
- String accessStatus = accessResponse.readEntity(String.class);
- JSONAssert.assertEquals(expectedAccessStatusJson, accessStatus, false);
-
- }
-
- @Test
- public void testTokenGenerationWithIdentityProvider() throws Exception {
-
- // Given: the client and server have been configured correctly for LDAP authentication
- String expectedJwtPayloadJson = "{" +
- "\"sub\":\"nobel\"," +
- "\"preferred_username\":\"nobel\"," +
- "\"iss\":\"LdapIdentityProvider\"," +
- "\"aud\":\"LdapIdentityProvider\"" +
- "}";
- String expectedAccessStatusJson = "{" +
- "\"identity\":\"nobel\"," +
- "\"anonymous\":false" +
- "}";
-
- // When: the /access/token/identity-provider endpoint is queried
- final String basicAuthCredentials = encodeCredentialsForBasicAuth("nobel", "password");
- final Response tokenResponse = client
- .target(createURL(tokenIdentityProviderPath))
- .request()
- .header("Authorization", "Basic " + basicAuthCredentials)
- .post(null, Response.class);
-
- // Then: the server returns 200 OK with an access token
- assertEquals(201, tokenResponse.getStatus());
- String token = tokenResponse.readEntity(String.class);
- assertTrue(StringUtils.isNotEmpty(token));
- String[] jwtParts = token.split("\\.");
- assertEquals(3, jwtParts.length);
- String jwtPayload = new String(Base64.getDecoder().decode(jwtParts[1]), "UTF-8");
- JSONAssert.assertEquals(expectedJwtPayloadJson, jwtPayload, false);
-
- // When: the token is returned in the Authorization header
- final Response accessResponse = client
- .target(createURL("access"))
- .request()
- .header("Authorization", "Bearer " + token)
- .get(Response.class);
-
- // Then: the server acknowledges the client has access
- assertEquals(200, accessResponse.getStatus());
- String accessStatus = accessResponse.readEntity(String.class);
- JSONAssert.assertEquals(expectedAccessStatusJson, accessStatus, false);
-
- }
-
- @Test
- public void testGetCurrentUserFailsForAnonymous() throws Exception {
-
- // Given: the client is connected to an secured NiFi Registry
- final String expectedJson = "{" +
- "\"anonymous\":true," +
- "\"identity\":\"anonymous\"," +
- "\"loginSupported\":true," +
- "\"resourcePermissions\":{" +
- "\"anyTopLevelResource\":{\"canDelete\":false,\"canRead\":false,\"canWrite\":false}," +
- "\"buckets\":{\"canDelete\":false,\"canRead\":false,\"canWrite\":false}," +
- "\"policies\":{\"canDelete\":false,\"canRead\":false,\"canWrite\":false}," +
- "\"proxy\":{\"canDelete\":false,\"canRead\":false,\"canWrite\":false}," +
- "\"tenants\":{\"canDelete\":false,\"canRead\":false,\"canWrite\":false}}" +
- "}";
-
- // When: the /access endpoint is queried with no credentials
- final Response response = client
- .target(createURL("/access"))
- .request()
- .get(Response.class);
-
- // Then: the server returns a 200 OK with the expected current user
- assertEquals(200, response.getStatus());
-
- final String actualJson = response.readEntity(String.class);
- JSONAssert.assertEquals(expectedJson, actualJson, false);
- }
-
- @Test
- public void testGetCurrentUser() throws Exception {
-
- // Given: the client is connected to an unsecured NiFi Registry
- String expectedJson = "{" +
- "\"identity\":\"nifiadmin\"," +
- "\"anonymous\":false," +
- "\"resourcePermissions\":{" +
- "\"anyTopLevelResource\":{\"canRead\":true,\"canWrite\":true,\"canDelete\":true}," +
- "\"buckets\":{\"canRead\":true,\"canWrite\":true,\"canDelete\":true}," +
- "\"tenants\":{\"canRead\":true,\"canWrite\":true,\"canDelete\":true}," +
- "\"policies\":{\"canRead\":true,\"canWrite\":true,\"canDelete\":true}," +
- "\"proxy\":{\"canRead\":true,\"canWrite\":true,\"canDelete\":true}}" +
- "}";
-
- // When: the /access endpoint is queried using a JWT for the nifiadmin LDAP user
- final Response response = client
- .target(createURL("/access"))
- .request()
- .header("Authorization", "Bearer " + adminAuthToken)
- .get(Response.class);
-
- // Then: the server returns a 200 OK with the expected current user
- assertEquals(200, response.getStatus());
- String actualJson = response.readEntity(String.class);
- JSONAssert.assertEquals(expectedJson, actualJson, false);
-
- }
-
- @Test
- public void testLogout() {
-
- // Given: the client is connected to an unsecured NiFi Registry
- // and the /access endpoint is queried using a JWT for the nifiadmin LDAP user
- final Response response = client
- .target(createURL("/access"))
- .request()
- .header("Authorization", "Bearer " + adminAuthToken)
- .get(Response.class);
-
- // and the server returns a 200 OK with the expected current user
- assertEquals(200, response.getStatus());
-
- // When: the /access/logout endpoint with the JWT for the nifiadmin logs out the user
- final Response logout_response = client
- .target(createURL("/access/logout"))
- .request()
- .header("Authorization", "Bearer " + adminAuthToken)
- .delete(Response.class);
-
- assertEquals(200, logout_response.getStatus());
-
- // Then: the /access endpoint is queried using the logged out JWT
- LOGGER.info("*** THE FOLLOWING JwtException IS EXPECTED ***");
- LOGGER.info("*** We are validating the access token no longer works following logout ***");
- final Response retryResponse = client
- .target(createURL("/access"))
- .request()
- .header("Authorization", "Bearer " + adminAuthToken)
- .get(Response.class);
-
- // and the server returns a 401 Unauthorized as the user is now logged out
- assertEquals(401, retryResponse.getStatus());
- String retryJson = retryResponse.readEntity(String.class);
- assertEquals("Unable to validate the access token. Contact the system administrator.\n", retryJson);
-
- // Reset: We successfully logged out our user. Run setup to fix up the user, so the @After code can run to re-establish authorizations.
- setup();
- }
-
- @Test
- public void testLogoutWithJWTSignedByWrongKey() throws Exception {
-
- // Given: use the /access/logout endpoint with the JWT for the nifiadmin LDAP user to log out
- LOGGER.info("*** THE FOLLOWING JwtException IS EXPECTED ***");
- final Response logoutResponse = client
- .target(createURL("/access"))
- .request()
- .header("Authorization", "Bearer " + SIGNED_BY_WRONG_KEY)
- .delete(Response.class);
-
- assertEquals(401, logoutResponse.getStatus());
- String responseMessage = logoutResponse.readEntity(String.class);
- assertEquals("Unable to validate the access token. Contact the system administrator.\n", responseMessage);
- }
-
- @Test
- public void testUsers() throws Exception {
-
- // Given: the client and server have been configured correctly for LDAP authentication
- String expectedJson = "[" +
- "{\"identity\":\"nifiadmin\",\"userGroups\":[],\"configurable\":false," +
- "\"resourcePermissions\":{" +
- "\"anyTopLevelResource\":{\"canRead\":true,\"canWrite\":true,\"canDelete\":true}," +
- "\"buckets\":{\"canRead\":true,\"canWrite\":true,\"canDelete\":true}," +
- "\"tenants\":{\"canRead\":true,\"canWrite\":true,\"canDelete\":true}," +
- "\"policies\":{\"canRead\":true,\"canWrite\":true,\"canDelete\":true}," +
- "\"proxy\":{\"canRead\":true,\"canWrite\":true,\"canDelete\":true}}}," +
- "{\"identity\":\"euler\",\"userGroups\":[{\"identity\":\"mathematicians\"}],\"accessPolicies\":[],\"configurable\":false}," +
- "{\"identity\":\"euclid\",\"userGroups\":[{\"identity\":\"mathematicians\"}],\"accessPolicies\":[],\"configurable\":false}," +
- "{\"identity\":\"boyle\",\"userGroups\":[{\"identity\":\"chemists\"}],\"accessPolicies\":[],\"configurable\":false}," +
- "{\"identity\":\"newton\",\"userGroups\":[{\"identity\":\"scientists\"}],\"accessPolicies\":[],\"configurable\":false}," +
- "{\"identity\":\"riemann\",\"userGroups\":[{\"identity\":\"mathematicians\"}],\"accessPolicies\":[],\"configurable\":false}," +
- "{\"identity\":\"gauss\",\"userGroups\":[{\"identity\":\"mathematicians\"}],\"accessPolicies\":[],\"configurable\":false}," +
- "{\"identity\":\"galileo\",\"userGroups\":[{\"identity\":\"scientists\"},{\"identity\":\"italians\"}],\"accessPolicies\":[],\"configurable\":false}," +
- "{\"identity\":\"nobel\",\"userGroups\":[{\"identity\":\"chemists\"}],\"accessPolicies\":[],\"configurable\":false}," +
- "{\"identity\":\"pasteur\",\"userGroups\":[{\"identity\":\"chemists\"}],\"accessPolicies\":[],\"configurable\":false}," +
- "{\"identity\":\"tesla\",\"userGroups\":[{\"identity\":\"scientists\"}],\"accessPolicies\":[],\"configurable\":false}," +
- "{\"identity\":\"nogroup\",\"userGroups\":[],\"accessPolicies\":[],\"configurable\":false}," +
- "{\"identity\":\"einstein\",\"userGroups\":[{\"identity\":\"scientists\"}],\"accessPolicies\":[],\"configurable\":false}," +
- "{\"identity\":\"curie\",\"userGroups\":[{\"identity\":\"chemists\"}],\"accessPolicies\":[],\"configurable\":false}]";
-
- // When: the /tenants/users endpoint is queried
- final String usersJson = client
- .target(createURL("tenants/users"))
- .request()
- .header("Authorization", "Bearer " + adminAuthToken)
- .get(String.class);
-
- // Then: the server returns a list of all users (see test-ldap-data.ldif)
- JSONAssert.assertEquals(expectedJson, usersJson, false);
- }
-
- @Test
- public void testUserGroups() throws Exception {
-
- // Given: the client and server have been configured correctly for LDAP authentication
- String expectedJson = "[" +
- "{" +
- "\"identity\":\"chemists\"," +
- "\"users\":[{\"identity\":\"pasteur\"},{\"identity\":\"boyle\"},{\"identity\":\"curie\"},{\"identity\":\"nobel\"}]," +
- "\"accessPolicies\":[]," +
- "\"configurable\":false" +
- "}," +
- "{" +
- "\"identity\":\"mathematicians\"," +
- "\"users\":[{\"identity\":\"gauss\"},{\"identity\":\"euclid\"},{\"identity\":\"riemann\"},{\"identity\":\"euler\"}]," +
- "\"accessPolicies\":[]," +
- "\"configurable\":false" +
- "}," +
- "{" +
- "\"identity\":\"scientists\"," +
- "\"users\":[{\"identity\":\"einstein\"},{\"identity\":\"tesla\"},{\"identity\":\"newton\"},{\"identity\":\"galileo\"}]," +
- "\"accessPolicies\":[]," +
- "\"configurable\":false" +
- "}," +
- "{" +
- "\"identity\":\"italians\"," +
- "\"users\":[{\"identity\":\"galileo\"}]," +
- "\"accessPolicies\":[]," +
- "\"configurable\":false" +
- "}]";
-
- // When: the /tenants/users endpoint is queried
- final String groupsJson = client
- .target(createURL("tenants/user-groups"))
- .request()
- .header("Authorization", "Bearer " + adminAuthToken)
- .get(String.class);
-
- // Then: the server returns a list of all users (see test-ldap-data.ldif)
- JSONAssert.assertEquals(expectedJson, groupsJson, false);
- }
-
- @Test
- public void testCreateTenantFails() throws Exception {
- Long initialVersion = new Long(0);
- String clientId = UUID.randomUUID().toString();
- RevisionInfo initialRevisionInfo = new RevisionInfo(clientId, initialVersion);
-
- // Given: the server has been configured with the LdapUserGroupProvider, which is non-configurable,
- // and: the client wants to create a tenant
- Tenant tenant = new Tenant();
- tenant.setIdentity("new_tenant");
- tenant.setRevision(initialRevisionInfo);
-
- // When: the POST /tenants/users endpoint is accessed
- final Response createUserResponse = client
- .target(createURL("tenants/users"))
- .request()
- .header("Authorization", "Bearer " + adminAuthToken)
- .post(Entity.entity(tenant, MediaType.APPLICATION_JSON_TYPE), Response.class);
-
- // Then: an error is returned
- assertEquals(409, createUserResponse.getStatus());
-
- // When: the POST /tenants/users endpoint is accessed
- final Response createUserGroupResponse = client
- .target(createURL("tenants/user-groups"))
- .request()
- .header("Authorization", "Bearer " + adminAuthToken)
- .post(Entity.entity(tenant, MediaType.APPLICATION_JSON_TYPE), Response.class);
-
- // Then: an error is returned because the UserGroupProvider is non-configurable
- assertEquals(409, createUserGroupResponse.getStatus());
- }
-
- @Test
- public void testAccessPolicyCreation() throws Exception {
- Long initialVersion = new Long(0);
- String clientId = UUID.randomUUID().toString();
- RevisionInfo initialRevisionInfo = new RevisionInfo(clientId, initialVersion);
-
- // Given: the server has been configured with an initial admin "nifiadmin" and a user with no accessPolicies "nobel"
- String nobelId = getTenantIdentifierByIdentity("nobel");
- String chemistsId = getTenantIdentifierByIdentity("chemists"); // a group containing user "nobel"
-
- final String basicAuthCredentials = encodeCredentialsForBasicAuth("nobel", "password");
- final String nobelAuthToken = client
- .target(createURL(tokenIdentityProviderPath))
- .request()
- .header("Authorization", "Basic " + basicAuthCredentials)
- .post(null, String.class);
-
- // When: user nobel re-checks top-level permissions
- final CurrentUser currentUser = client
- .target(createURL("/access"))
- .request()
- .header("Authorization", "Bearer " + nobelAuthToken)
- .get(CurrentUser.class);
-
- // Then: 200 OK is returned indicating user has access to no top-level resources
- assertEquals(new Permissions(), currentUser.getResourcePermissions().getBuckets());
- assertEquals(new Permissions(), currentUser.getResourcePermissions().getTenants());
- assertEquals(new Permissions(), currentUser.getResourcePermissions().getPolicies());
- assertEquals(new Permissions(), currentUser.getResourcePermissions().getProxy());
-
- // When: nifiadmin creates a bucket
- final Bucket bucket = new Bucket();
- bucket.setName("Integration Test Bucket");
- bucket.setDescription("A bucket created by an integration test.");
- bucket.setRevision(new RevisionInfo(null, 0L));
-
- Response adminCreatesBucketResponse = client
- .target(createURL("buckets"))
- .request()
- .header("Authorization", "Bearer " + adminAuthToken)
- .post(Entity.entity(bucket, MediaType.APPLICATION_JSON), Response.class);
-
- // Then: the server returns a 200 OK
- assertEquals(200, adminCreatesBucketResponse.getStatus());
- Bucket createdBucket = adminCreatesBucketResponse.readEntity(Bucket.class);
-
-
- // When: user nobel initial queries /buckets
- final Bucket[] buckets1 = client
- .target(createURL("buckets"))
- .request()
- .header("Authorization", "Bearer " + nobelAuthToken)
- .get(Bucket[].class);
-
- // Then: an empty list is returned (nobel has no read access yet)
- assertNotNull(buckets1);
- assertEquals(0, buckets1.length);
-
-
- // When: nifiadmin grants read access on createdBucket to 'chemists' a group containing nobel
- AccessPolicy readPolicy = new AccessPolicy();
- readPolicy.setResource("/buckets/" + createdBucket.getIdentifier());
- readPolicy.setAction("read");
- readPolicy.addUserGroups(Arrays.asList(new Tenant(chemistsId, "chemists")));
- readPolicy.setRevision(initialRevisionInfo);
-
- Response adminGrantsReadAccessResponse = client
- .target(createURL("policies"))
- .request()
- .header("Authorization", "Bearer " + adminAuthToken)
- .post(Entity.entity(readPolicy, MediaType.APPLICATION_JSON), Response.class);
-
- // Then: the server returns a 201 Created
- assertEquals(201, adminGrantsReadAccessResponse.getStatus());
-
-
- // When: nifiadmin tries to list all buckets
- final Bucket[] adminBuckets = client
- .target(createURL("buckets"))
- .request()
- .header("Authorization", "Bearer " + adminAuthToken)
- .get(Bucket[].class);
-
- // Then: the full list is returned (verifies that per-bucket access policies are additive to base /buckets policy)
- assertNotNull(adminBuckets);
- assertEquals(1, adminBuckets.length);
- assertEquals(createdBucket.getIdentifier(), adminBuckets[0].getIdentifier());
- assertEquals(new Permissions().withCanRead(true).withCanWrite(true).withCanDelete(true), adminBuckets[0].getPermissions());
-
-
- // When: user nobel re-queries /buckets
- final Bucket[] buckets2 = client
- .target(createURL("buckets"))
- .request()
- .header("Authorization", "Bearer " + nobelAuthToken)
- .get(Bucket[].class);
-
- // Then: the created bucket is now present
- assertNotNull(buckets2);
- assertEquals(1, buckets2.length);
- assertEquals(createdBucket.getIdentifier(), buckets2[0].getIdentifier());
- assertEquals(new Permissions().withCanRead(true), buckets2[0].getPermissions());
-
-
- // When: nifiadmin grants write access on createdBucket to user 'nobel'
- AccessPolicy writePolicy = new AccessPolicy();
- writePolicy.setResource("/buckets/" + createdBucket.getIdentifier());
- writePolicy.setAction("write");
- writePolicy.addUsers(Arrays.asList(new Tenant(nobelId, "nobel")));
- writePolicy.setRevision(initialRevisionInfo);
-
- Response adminGrantsWriteAccessResponse = client
- .target(createURL("policies"))
- .request()
- .header("Authorization", "Bearer " + adminAuthToken)
- .post(Entity.entity(writePolicy, MediaType.APPLICATION_JSON), Response.class);
-
- // Then: the server returns a 201 Created
- assertEquals(201, adminGrantsWriteAccessResponse.getStatus());
-
-
- // When: user nobel re-queries /buckets
- final Bucket[] buckets3 = client
- .target(createURL("buckets"))
- .request()
- .header("Authorization", "Bearer " + nobelAuthToken)
- .get(Bucket[].class);
-
- // Then: the authorizedActions are updated
- assertNotNull(buckets3);
- assertEquals(1, buckets3.length);
- assertEquals(createdBucket.getIdentifier(), buckets3[0].getIdentifier());
- assertEquals(new Permissions().withCanRead(true).withCanWrite(true), buckets3[0].getPermissions());
-
- }
-
- @Test
- public void testAccessClient() throws IOException, NiFiRegistryException {
- final String baseUrl = createBaseURL();
- LOGGER.info("Using base url = " + baseUrl);
-
- final NiFiRegistryClientConfig clientConfig = createClientConfig(baseUrl);
- Assert.assertNotNull(clientConfig);
-
- final NiFiRegistryClient client = new JerseyNiFiRegistryClient.Builder()
- .config(clientConfig)
- .build();
-
- final String username = "pasteur";
- final String password = "password";
-
- // authenticate with the username and password to obtain a token
- final AccessClient accessClient = client.getAccessClient();
- final String token = accessClient.getToken(username, password);
- assertNotNull(token);
-
- // use the token to check the status of the current user
- final RequestConfig requestConfig = new BearerTokenRequestConfig(token);
- final UserClient userClient = client.getUserClient(requestConfig);
- assertEquals(username, userClient.getAccessStatus().getIdentity());
-
- // use the token to logout
- accessClient.logout(token);
-
- // check the status of the current user again and should be unauthorized
- LOGGER.info("*** THE FOLLOWING JwtException IS EXPECTED ***");
- LOGGER.info("*** We are validating the access token no longer works following logout ***");
- try {
- userClient.getAccessStatus();
- Assert.fail("Should have failed with an unauthorized exception");
- } catch (Exception e) {
- //LOGGER.error(e.getMessage(), e);
- }
-
- // try to get a token with an invalid username and password
- try {
- accessClient.getToken("user-does-not-exist", "bad-password");
- Assert.fail("Should have failed with an unauthorized exception");
- } catch (Exception e) {
-
- }
- }
-
- /** A helper method to lookup identifiers for tenant identities using the REST API
- *
- * @param tenantIdentity - the identity to lookup
- * @return A string containing the identifier of the tenant, or null if the tenant identity is not found.
- */
- private String getTenantIdentifierByIdentity(String tenantIdentity) {
-
- final Tenant[] users = client
- .target(createURL("tenants/users"))
- .request()
- .header("Authorization", "Bearer " + adminAuthToken)
- .get(Tenant[].class);
-
- final Tenant[] groups = client
- .target(createURL("tenants/user-groups"))
- .request()
- .header("Authorization", "Bearer " + adminAuthToken)
- .get(Tenant[].class);
-
- final Tenant matchedTenant = Stream.concat(Arrays.stream(users), Arrays.stream(groups))
- .filter(tenant -> tenant.getIdentity().equalsIgnoreCase(tenantIdentity))
- .findFirst()
- .orElse(null);
-
- return matchedTenant != null ? matchedTenant.getIdentifier() : null;
- }
-
- /** A helper method to lookup access policies
- *
- * @return A string containing the identifier of the policy, or null if the policy identity is not found.
- */
- private AccessPolicy getPolicyByResourceAction(String action, String resource) {
-
- final AccessPolicySummary[] policies = client
- .target(createURL("policies"))
- .request()
- .header("Authorization", "Bearer " + adminAuthToken)
- .get(AccessPolicySummary[].class);
-
- final AccessPolicySummary matchedPolicy = Arrays.stream(policies)
- .filter(p -> p.getAction().equalsIgnoreCase(action) && p.getResource().equalsIgnoreCase(resource))
- .findFirst()
- .orElse(null);
-
- if (matchedPolicy == null) {
- return null;
- }
-
- String policyId = matchedPolicy.getIdentifier();
-
- final AccessPolicy policy = client
- .target(createURL("policies/" + policyId))
- .request()
- .header("Authorization", "Bearer " + adminAuthToken)
- .get(AccessPolicy.class);
-
- return policy;
- }
-
- private List<AccessPolicy> createAccessPoliciesSnapshot() {
-
- final AccessPolicySummary[] policySummaries = client
- .target(createURL("policies"))
- .request()
- .header("Authorization", "Bearer " + adminAuthToken)
- .get(AccessPolicySummary[].class);
-
- final List<AccessPolicy> policies = new ArrayList<>(policySummaries.length);
- for (AccessPolicySummary s : policySummaries) {
- AccessPolicy policy = client
- .target(createURL("policies/" + s.getIdentifier()))
- .request()
- .header("Authorization", "Bearer " + adminAuthToken)
- .get(AccessPolicy.class);
- policies.add(policy);
- }
-
- return policies;
- }
-
- private void restoreAccessPoliciesSnapshot(List<AccessPolicy> accessPoliciesSnapshot) {
-
- List<AccessPolicy> currentAccessPolicies = createAccessPoliciesSnapshot();
-
- Set<String> policiesToRestore = accessPoliciesSnapshot.stream()
- .map(AccessPolicy::getIdentifier)
- .collect(Collectors.toSet());
-
- Set<AccessPolicy> policiesToDelete = currentAccessPolicies.stream()
- .filter(p -> !policiesToRestore.contains(p.getIdentifier()))
- .collect(Collectors.toSet());
-
- for (AccessPolicy originalPolicy : accessPoliciesSnapshot) {
-
- Response getCurrentPolicy = client
- .target(createURL("policies/" + originalPolicy.getIdentifier()))
- .request()
- .header("Authorization", "Bearer " + adminAuthToken)
- .get(Response.class);
-
- if (getCurrentPolicy.getStatus() == 200) {
- // update policy to match original
- client.target(createURL("policies/" + originalPolicy.getIdentifier()))
- .request()
- .header("Authorization", "Bearer " + adminAuthToken)
- .put(Entity.entity(originalPolicy, MediaType.APPLICATION_JSON));
- } else {
- // post the original policy
- client.target(createURL("policies"))
- .request()
- .header("Authorization", "Bearer " + adminAuthToken)
- .post(Entity.entity(originalPolicy, MediaType.APPLICATION_JSON));
- }
-
- }
-
- for (AccessPolicy policyToDelete : policiesToDelete) {
- try {
- final RevisionInfo revisionInfo = policyToDelete.getRevision();
- final Long version = revisionInfo == null ? 0 : revisionInfo.getVersion();
- client.target(createURL("policies/" + policyToDelete.getIdentifier()))
- .queryParam("version", version.longValue())
- .request()
- .header("Authorization", "Bearer " + adminAuthToken)
- .delete();
- } catch (Exception e) {
- LOGGER.error("Error cleaning up policies after test due to: " + e.getMessage(), e);
- }
- }
-
- }
-
- private static String encodeCredentialsForBasicAuth(String username, String password) {
- final String credentials = username + ":" + password;
- final String base64credentials = new String(Base64.getEncoder().encode(credentials.getBytes(Charset.forName("UTF-8"))));
- return base64credentials;
- }
-}
diff --git a/nifi-registry/pom.xml b/nifi-registry/pom.xml
index 06f8c70e32..4e8174b0f7 100644
--- a/nifi-registry/pom.xml
+++ b/nifi-registry/pom.xml
@@ -127,6 +127,11 @@
<artifactId>nifi-property-protection-factory</artifactId>
<version>1.16.1-SNAPSHOT</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-property-protection-loader</artifactId>
+ <version>1.17.0-SNAPSHOT</version>
+ </dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-property-utils</artifactId>
diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml b/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml
index 95496b4e6b..326d600d90 100644
--- a/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml
+++ b/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml
@@ -44,6 +44,11 @@
<artifactId>nifi-property-protection-factory</artifactId>
<version>1.16.1-SNAPSHOT</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-property-protection-loader</artifactId>
+ <version>1.17.0-SNAPSHOT</version>
+ </dependency>
<dependency>
<groupId>org.apache.nifi.registry</groupId>
<artifactId>nifi-registry-properties</artifactId>
diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy
index 578db83973..ec826493fa 100644
--- a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy
+++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy
@@ -880,7 +880,7 @@ class ConfigEncryptionTool {
passwords.each { password ->
if (isVerbose) {
- logger.info("Attempting to encrypt ${password.name()} using protection scheme ${protectionScheme}")
+ logger.info("Attempting to encrypt ${password.name()} using protection scheme ${protectionScheme.path}")
}
final ProtectedPropertyContext context = getContext(providerFactory, (String) password.@name, groupIdentifier)
String encryptedValue = sensitivePropertyProvider.protect((String) password.text().trim(), context)
@@ -928,7 +928,7 @@ class ConfigEncryptionTool {
passwords.each { password ->
if (isVerbose) {
- logger.info("Attempting to encrypt ${password.name()} using protection scheme ${protectionScheme}")
+ logger.info("Attempting to encrypt ${password.name()} using protection scheme ${protectionScheme.path}")
}
final ProtectedPropertyContext context = getContext(providerFactory, (String) password.@name, groupIdentifier)
String encryptedValue = sensitivePropertyProvider.protect((String) password.text().trim(), context)
@@ -982,18 +982,18 @@ class ConfigEncryptionTool {
// Iterate over each -- encrypt and add .protected if populated
sensitivePropertyKeys.each { String key ->
if (!plainProperties.getProperty(key)) {
- logger.debug("Skipping encryption of ${key} because it is empty")
+ logger.debug("Skipping encryption of [${key}] because it is empty")
} else {
String protectedValue = spp.protect(plainProperties.getProperty(key), ProtectedPropertyContext.defaultContext(key))
// Add the encrypted value
encryptedProperties.setProperty(key, protectedValue)
- logger.info("Protected ${key} with ${protectionScheme} -> \t${protectedValue}")
+ logger.info("Protected [${key}] using [${protectionScheme.path}] -> \t${protectedValue}")
// Add the protection key ("x.y.z.protected" -> "aes/gcm/{128,256}")
String protectionKey = ApplicationPropertiesProtector.getProtectionKey(key)
encryptedProperties.setProperty(protectionKey, spp.getIdentifierKey())
- logger.info("Updated protection key ${protectionKey}")
+ logger.info("Updated protection key [${protectionKey}]")
keysToSkip << key << protectionKey
}
@@ -1330,7 +1330,7 @@ class ConfigEncryptionTool {
boolean niFiPropertiesAreEncrypted() {
if (niFiPropertiesPath) {
try {
- def nfp = getNiFiPropertiesLoader(keyHex).readProtectedPropertiesFromDisk(new File(niFiPropertiesPath))
+ def nfp = getNiFiPropertiesLoader(keyHex).loadProtectedProperties(new File(niFiPropertiesPath))
return nfp.hasProtectedKeys()
} catch (SensitivePropertyProtectionException | IOException e) {
logger.debug("Read Protected Properties failed {}", e.getMessage())
diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/properties/ConfigEncryptionToolTest.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/properties/ConfigEncryptionToolTest.groovy
index 45c3ee0483..987d1686e9 100644
--- a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/properties/ConfigEncryptionToolTest.groovy
+++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/properties/ConfigEncryptionToolTest.groovy
@@ -17,8 +17,6 @@
package org.apache.nifi.properties
import groovy.test.GroovyLogTestCase
-import groovy.test.GroovyShellTestCase
-import groovy.test.GroovyTestCase
import org.apache.commons.cli.CommandLine
import org.apache.commons.cli.CommandLineParser
import org.apache.commons.cli.DefaultParser
@@ -974,7 +972,7 @@ class ConfigEncryptionToolTest extends GroovyLogTestCase {
File originalFile = new File(originalNiFiPropertiesPath)
List<String> originalLines = originalFile.readLines()
- ProtectedNiFiProperties protectedProperties = NiFiPropertiesLoader.withKey(KEY_HEX).readProtectedPropertiesFromDisk(new File(originalNiFiPropertiesPath))
+ ProtectedNiFiProperties protectedProperties = NiFiPropertiesLoader.withKey(KEY_HEX).loadProtectedProperties(new File(originalNiFiPropertiesPath))
int originalProtectedPropertyCount = protectedProperties.getProtectedPropertyKeys().size()
protectedProperties.addSensitivePropertyProvider(DEFAULT_PROVIDER_FACTORY.getProvider(ConfigEncryptionTool.DEFAULT_PROTECTION_SCHEME))
@@ -1181,7 +1179,7 @@ class ConfigEncryptionToolTest extends GroovyLogTestCase {
logger.info("\n" * 2 + updatedPropertiesLines.join("\n"))
// Check that the output values for sensitive properties are not the same as the original (i.e. it was encrypted)
- NiFiProperties updatedProperties = new NiFiPropertiesLoader().readProtectedPropertiesFromDisk(outputPropertiesFile)
+ NiFiProperties updatedProperties = new NiFiPropertiesLoader().loadProtectedProperties(outputPropertiesFile)
assert updatedProperties.size() >= inputProperties.size()
originalSensitiveValues.every { String key, String originalValue ->
assert updatedProperties.getProperty(key) != originalValue
@@ -1257,7 +1255,7 @@ class ConfigEncryptionToolTest extends GroovyLogTestCase {
logger.info("\n" * 2 + updatedPropertiesLines.join("\n"))
// Check that the output values for sensitive properties are not the same as the original (i.e. it was encrypted)
- NiFiProperties updatedProperties = new NiFiPropertiesLoader().readProtectedPropertiesFromDisk(outputPropertiesFile)
+ NiFiProperties updatedProperties = new NiFiPropertiesLoader().loadProtectedProperties(outputPropertiesFile)
assert updatedProperties.size() >= inputProperties.size()
originalSensitiveValues.every { String key, String originalValue ->
assert updatedProperties.getProperty(key) != originalValue
@@ -1345,7 +1343,7 @@ class ConfigEncryptionToolTest extends GroovyLogTestCase {
logger.info("\n" * 2 + updatedPropertiesLines.join("\n"))
// Check that the output values for sensitive properties are not the same as the original (i.e. it was encrypted)
- NiFiProperties updatedProperties = new NiFiPropertiesLoader().readProtectedPropertiesFromDisk(outputPropertiesFile)
+ NiFiProperties updatedProperties = new NiFiPropertiesLoader().loadProtectedProperties(outputPropertiesFile)
assert updatedProperties.size() >= inputProperties.size()
originalSensitiveValues.every { String key, String originalValue ->
assert updatedProperties.getProperty(key) != originalValue
@@ -1460,7 +1458,7 @@ class ConfigEncryptionToolTest extends GroovyLogTestCase {
logger.info("\n" * 2 + updatedPropertiesLines.join("\n"))
// Check that the output values for sensitive properties are not the same as the original (i.e. it was re-encrypted)
- NiFiProperties updatedProperties = new NiFiPropertiesLoader().readProtectedPropertiesFromDisk(outputPropertiesFile)
+ NiFiProperties updatedProperties = new NiFiPropertiesLoader().loadProtectedProperties(outputPropertiesFile)
assert updatedProperties.size() >= inputProperties.size()
originalSensitiveValues.every { String key, String originalValue ->
assert updatedProperties.getProperty(key) != originalValue
@@ -3276,7 +3274,7 @@ class ConfigEncryptionToolTest extends GroovyLogTestCase {
logger.info("\n" * 2 + updatedPropertiesLines.join("\n"))
// Check that the output values for sensitive properties are not the same as the original (i.e. it was encrypted)
- NiFiProperties updatedProperties = new NiFiPropertiesLoader().readProtectedPropertiesFromDisk(outputPropertiesFile)
+ NiFiProperties updatedProperties = new NiFiPropertiesLoader().loadProtectedProperties(outputPropertiesFile)
assert updatedProperties.size() >= inputProperties.size()
originalSensitiveValues.every { String key, String originalValue ->
assert updatedProperties.getProperty(key) != originalValue
@@ -3472,7 +3470,7 @@ class ConfigEncryptionToolTest extends GroovyLogTestCase {
logger.info("\n" * 2 + updatedPropertiesLines.join("\n"))
// Check that the output values for everything is the same except the sensitive props key
- NiFiProperties updatedProperties = new NiFiPropertiesLoader().readProtectedPropertiesFromDisk(workingNiFiPropertiesFile)
+ NiFiProperties updatedProperties = new NiFiPropertiesLoader().loadProtectedProperties(workingNiFiPropertiesFile)
assert updatedProperties.size() == inputProperties.size()
assert updatedProperties.getProperty(NiFiProperties.SENSITIVE_PROPS_KEY) == newFlowPassword
originalSensitiveValues.every { String key, String originalValue ->
@@ -3569,7 +3567,7 @@ class ConfigEncryptionToolTest extends GroovyLogTestCase {
final List<String> updatedPropertiesLines = workingNiFiPropertiesFile.readLines()
// Check that the output values for everything is the same including the sensitive props key
- NiFiProperties updatedProperties = new NiFiPropertiesLoader().readProtectedPropertiesFromDisk(workingNiFiPropertiesFile)
+ NiFiProperties updatedProperties = new NiFiPropertiesLoader().loadProtectedProperties(workingNiFiPropertiesFile)
assert updatedProperties.size() == inputProperties.size()
originalSensitiveValues.every { String key, String originalValue ->
assert updatedProperties.getProperty(key) == originalValue
@@ -3654,7 +3652,7 @@ class ConfigEncryptionToolTest extends GroovyLogTestCase {
final String SENSITIVE_PROTECTION_KEY = ApplicationPropertiesProtector.getProtectionKey(NiFiProperties.SENSITIVE_PROPS_KEY)
- ProtectedNiFiProperties encryptedProperties = niFiPropertiesLoader.readProtectedPropertiesFromDisk(workingNiFiPropertiesFile)
+ ProtectedNiFiProperties encryptedProperties = niFiPropertiesLoader.loadProtectedProperties(workingNiFiPropertiesFile)
def originalEncryptedValues = encryptedProperties.getSensitivePropertyKeys().collectEntries { String key -> [(key): encryptedProperties.getProperty(key)] }
logger.info("Original encrypted values: ${originalEncryptedValues}")
String originalSensitiveKeyProtectionScheme = encryptedProperties.getProperty(SENSITIVE_PROTECTION_KEY)
@@ -3675,7 +3673,7 @@ class ConfigEncryptionToolTest extends GroovyLogTestCase {
.getProvider(ConfigEncryptionTool.DEFAULT_PROTECTION_SCHEME)
// Check that the output values for everything is the same except the sensitive props key
- NiFiProperties updatedProperties = new NiFiPropertiesLoader().readProtectedPropertiesFromDisk(workingNiFiPropertiesFile)
+ NiFiProperties updatedProperties = new NiFiPropertiesLoader().loadProtectedProperties(workingNiFiPropertiesFile)
assert updatedProperties.size() == inputProperties.size()
String newSensitivePropertyKey = updatedProperties.getProperty(NiFiProperties.SENSITIVE_PROPS_KEY)
@@ -3787,7 +3785,7 @@ class ConfigEncryptionToolTest extends GroovyLogTestCase {
final String SENSITIVE_PROTECTION_KEY = ApplicationPropertiesProtector.getProtectionKey(NiFiProperties.SENSITIVE_PROPS_KEY)
- ProtectedNiFiProperties encryptedProperties = niFiPropertiesLoader.readProtectedPropertiesFromDisk(workingNiFiPropertiesFile)
+ ProtectedNiFiProperties encryptedProperties = niFiPropertiesLoader.loadProtectedProperties(workingNiFiPropertiesFile)
def originalEncryptedValues = encryptedProperties.getSensitivePropertyKeys().collectEntries { String key -> [(key): encryptedProperties.getProperty(key)] }
logger.info("Original encrypted values: ${originalEncryptedValues}")
String originalSensitiveKeyProtectionScheme = encryptedProperties.getProperty(SENSITIVE_PROTECTION_KEY)
@@ -3808,7 +3806,7 @@ class ConfigEncryptionToolTest extends GroovyLogTestCase {
.getProvider(ConfigEncryptionTool.DEFAULT_PROTECTION_SCHEME)
// Check that the output values for everything is the same except the sensitive props key
- NiFiProperties updatedProperties = new NiFiPropertiesLoader().readProtectedPropertiesFromDisk(workingNiFiPropertiesFile)
+ NiFiProperties updatedProperties = new NiFiPropertiesLoader().loadProtectedProperties(workingNiFiPropertiesFile)
assert updatedProperties.size() == inputProperties.size()
String newSensitivePropertyKey = updatedProperties.getProperty(NiFiProperties.SENSITIVE_PROPS_KEY)
@@ -3922,7 +3920,7 @@ class ConfigEncryptionToolTest extends GroovyLogTestCase {
logger.info("Original sensitive values: ${originalSensitiveValues}")
final String SENSITIVE_PROTECTION_KEY = ApplicationPropertiesProtector.getProtectionKey(NiFiProperties.SENSITIVE_PROPS_KEY)
- ProtectedNiFiProperties encryptedProperties = niFiPropertiesLoader.readProtectedPropertiesFromDisk(workingNiFiPropertiesFile)
+ ProtectedNiFiProperties encryptedProperties = niFiPropertiesLoader.loadProtectedProperties(workingNiFiPropertiesFile)
def originalEncryptedValues = encryptedProperties.getSensitivePropertyKeys().collectEntries { String key -> [(key): encryptedProperties.getProperty(key)] }
logger.info("Original encrypted values: ${originalEncryptedValues}")
String originalSensitiveKeyProtectionScheme = encryptedProperties.getProperty(SENSITIVE_PROTECTION_KEY)
@@ -3960,7 +3958,7 @@ class ConfigEncryptionToolTest extends GroovyLogTestCase {
logger.info("Updated key line: ${updatedSensitiveKeyLine}")
// Check that the output values for everything are the same except the sensitive props key
- NiFiProperties updatedProperties = new NiFiPropertiesLoader().readProtectedPropertiesFromDisk(workingNiFiPropertiesFile)
+ NiFiProperties updatedProperties = new NiFiPropertiesLoader().loadProtectedProperties(workingNiFiPropertiesFile)
assert updatedProperties.size() == inputProperties.size()
String newSensitivePropertyKey = updatedProperties.getProperty(NiFiProperties.SENSITIVE_PROPS_KEY)
diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/toolkit/encryptconfig/EncryptConfigMainTest.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/toolkit/encryptconfig/EncryptConfigMainTest.groovy
index 6756c7ec8b..efaab486c0 100644
--- a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/toolkit/encryptconfig/EncryptConfigMainTest.groovy
+++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/toolkit/encryptconfig/EncryptConfigMainTest.groovy
@@ -193,11 +193,9 @@ class EncryptConfigMainTest extends GroovyTestCase {
/*** NiFi Properties Assertions ***/
final List<String> updatedPropertiesLines = outputPropertiesFile.readLines()
- logger.info("Updated nifi.properties:")
- logger.info("\n" * 2 + updatedPropertiesLines.join("\n"))
// Check that the output values for sensitive properties are not the same as the original (i.e. it was encrypted)
- NiFiProperties updatedProperties = new NiFiPropertiesLoader().readProtectedPropertiesFromDisk(outputPropertiesFile)
+ NiFiProperties updatedProperties = NiFiPropertiesLoader.withKey(TestUtil.KEY_HEX).load(outputPropertiesFile)
assert updatedProperties.size() >= inputProperties.size()
// Check that the new NiFiProperties instance matches the output file (values still encrypted)
[nifi] 04/09: NIFI-9928 Removed nifi-security-utils from nifi-prometheus-reporting-task
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a commit to branch support/nifi-1.16
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit 5588f32bb705ffa07c755d85428c74584053553d
Author: exceptionfactory <ex...@apache.org>
AuthorDate: Thu Apr 14 17:10:03 2022 -0500
NIFI-9928 Removed nifi-security-utils from nifi-prometheus-reporting-task
- Adjusted PrometheusServer configuration to use SSLContextService.createContext() instead of individual properties
This closes #5970
Signed-off-by: Paul Grey <gr...@apache.org>
---
.../nifi-prometheus-reporting-task/pom.xml | 11 +++--------
.../nifi/reporting/prometheus/PrometheusServer.java | 17 ++++-------------
2 files changed, 7 insertions(+), 21 deletions(-)
diff --git a/nifi-nar-bundles/nifi-prometheus-bundle/nifi-prometheus-reporting-task/pom.xml b/nifi-nar-bundles/nifi-prometheus-bundle/nifi-prometheus-reporting-task/pom.xml
index eabec46a8e..0169af6636 100644
--- a/nifi-nar-bundles/nifi-prometheus-bundle/nifi-prometheus-reporting-task/pom.xml
+++ b/nifi-nar-bundles/nifi-prometheus-bundle/nifi-prometheus-reporting-task/pom.xml
@@ -48,14 +48,9 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.apache.nifi</groupId>
- <artifactId>nifi-security-utils</artifactId>
- <version>1.16.1-SNAPSHOT</version>
- </dependency>
- <dependency>
- <groupId>org.apache.nifi</groupId>
- <artifactId>nifi-ssl-context-service-api</artifactId>
- <version>1.16.1-SNAPSHOT</version>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-ssl-context-service-api</artifactId>
+ <version>1.17.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
diff --git a/nifi-nar-bundles/nifi-prometheus-bundle/nifi-prometheus-reporting-task/src/main/java/org/apache/nifi/reporting/prometheus/PrometheusServer.java b/nifi-nar-bundles/nifi-prometheus-bundle/nifi-prometheus-reporting-task/src/main/java/org/apache/nifi/reporting/prometheus/PrometheusServer.java
index cd815f59f5..6af7bec28a 100644
--- a/nifi-nar-bundles/nifi-prometheus-bundle/nifi-prometheus-reporting-task/src/main/java/org/apache/nifi/reporting/prometheus/PrometheusServer.java
+++ b/nifi-nar-bundles/nifi-prometheus-bundle/nifi-prometheus-reporting-task/src/main/java/org/apache/nifi/reporting/prometheus/PrometheusServer.java
@@ -33,6 +33,7 @@ import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import org.eclipse.jetty.util.ssl.SslContextFactory;
+import javax.net.ssl.SSLContext;
import javax.servlet.ServletException;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServlet;
@@ -123,23 +124,13 @@ public class PrometheusServer {
}
private SslContextFactory createSslFactory(final SSLContextService sslService, boolean needClientAuth, boolean wantClientAuth) {
- SslContextFactory sslFactory = new SslContextFactory.Server();
+ final SslContextFactory.Server sslFactory = new SslContextFactory.Server();
sslFactory.setNeedClientAuth(needClientAuth);
sslFactory.setWantClientAuth(wantClientAuth);
- sslFactory.setProtocol(sslService.getSslAlgorithm());
- if (sslService.isKeyStoreConfigured()) {
- sslFactory.setKeyStorePath(sslService.getKeyStoreFile());
- sslFactory.setKeyStorePassword(sslService.getKeyStorePassword());
- sslFactory.setKeyStoreType(sslService.getKeyStoreType());
- }
-
- if (sslService.isTrustStoreConfigured()) {
- sslFactory.setTrustStorePath(sslService.getTrustStoreFile());
- sslFactory.setTrustStorePassword(sslService.getTrustStorePassword());
- sslFactory.setTrustStoreType(sslService.getTrustStoreType());
- }
+ final SSLContext sslContext = sslService.createContext();
+ sslFactory.setSslContext(sslContext);
return sslFactory;
}
[nifi] 07/09: NIFI-9933 Upgraded Apache Ant to 1.10.12
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a commit to branch support/nifi-1.16
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit 906dc6ca9b7e0e36b87c9022fbad84d05053b327
Author: exceptionfactory <ex...@apache.org>
AuthorDate: Mon Apr 18 14:11:40 2022 -0500
NIFI-9933 Upgraded Apache Ant to 1.10.12
- Upgraded nifi-groovyx-nar
- Upgraded nifi-scripting-nar
- Upgraded nifi-toolkit-admin
- Upgraded nifi-toolkit-encrypt-config
- Upgraded multiple nifi-hive modules
Signed-off-by: Pierre Villard <pi...@gmail.com>
This closes #5973.
---
.../nifi-groovyx-bundle/nifi-groovyx-nar/pom.xml | 17 +++++++++++++++++
nifi-nar-bundles/nifi-hive-bundle/pom.xml | 6 ++++++
.../nifi-scripting-bundle/nifi-scripting-nar/pom.xml | 17 +++++++++++++++++
nifi-toolkit/nifi-toolkit-admin/pom.xml | 4 ++++
nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml | 4 ++++
5 files changed, 48 insertions(+)
diff --git a/nifi-nar-bundles/nifi-groovyx-bundle/nifi-groovyx-nar/pom.xml b/nifi-nar-bundles/nifi-groovyx-bundle/nifi-groovyx-nar/pom.xml
index 133dad03e7..c90b9dcf94 100644
--- a/nifi-nar-bundles/nifi-groovyx-bundle/nifi-groovyx-nar/pom.xml
+++ b/nifi-nar-bundles/nifi-groovyx-bundle/nifi-groovyx-nar/pom.xml
@@ -27,6 +27,7 @@
<properties>
<maven.javadoc.skip>true</maven.javadoc.skip>
<source.skip>true</source.skip>
+ <ant.version>1.10.12</ant.version>
</properties>
<dependencies>
@@ -60,8 +61,24 @@
<groupId>org.codehaus.groovy</groupId>
<artifactId>groovy-testng</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.ant</groupId>
+ <artifactId>ant-junit</artifactId>
+ </exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <!-- Override ant from groovy-ant -->
+ <groupId>org.apache.ant</groupId>
+ <artifactId>ant</artifactId>
+ <version>${ant.version}</version>
+ </dependency>
+ <dependency>
+ <!-- Override ant from groovy-ant -->
+ <groupId>org.apache.ant</groupId>
+ <artifactId>ant-antlr</artifactId>
+ <version>${ant.version}</version>
+ </dependency>
</dependencies>
<scm>
diff --git a/nifi-nar-bundles/nifi-hive-bundle/pom.xml b/nifi-nar-bundles/nifi-hive-bundle/pom.xml
index 0beb4dbff5..1dd951e582 100644
--- a/nifi-nar-bundles/nifi-hive-bundle/pom.xml
+++ b/nifi-nar-bundles/nifi-hive-bundle/pom.xml
@@ -102,6 +102,12 @@
<artifactId>zookeeper</artifactId>
<version>3.4.14</version>
</dependency>
+ <!-- Override ant -->
+ <dependency>
+ <groupId>org.apache.ant</groupId>
+ <artifactId>ant</artifactId>
+ <version>1.10.12</version>
+ </dependency>
</dependencies>
</dependencyManagement>
diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-nar/pom.xml b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-nar/pom.xml
index 071050b249..e8c49c0028 100644
--- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-nar/pom.xml
+++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-nar/pom.xml
@@ -27,6 +27,7 @@
<properties>
<maven.javadoc.skip>true</maven.javadoc.skip>
<source.skip>true</source.skip>
+ <ant.version>1.10.12</ant.version>
</properties>
<dependencies>
@@ -68,8 +69,24 @@
<groupId>org.codehaus.groovy</groupId>
<artifactId>groovy-testng</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.ant</groupId>
+ <artifactId>ant-junit</artifactId>
+ </exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <!-- Override ant from groovy-ant -->
+ <groupId>org.apache.ant</groupId>
+ <artifactId>ant</artifactId>
+ <version>${ant.version}</version>
+ </dependency>
+ <dependency>
+ <!-- Override ant from groovy-ant -->
+ <groupId>org.apache.ant</groupId>
+ <artifactId>ant-antlr</artifactId>
+ <version>${ant.version}</version>
+ </dependency>
</dependencies>
<scm>
diff --git a/nifi-toolkit/nifi-toolkit-admin/pom.xml b/nifi-toolkit/nifi-toolkit-admin/pom.xml
index 42db8aa6fe..73eb7c5429 100644
--- a/nifi-toolkit/nifi-toolkit-admin/pom.xml
+++ b/nifi-toolkit/nifi-toolkit-admin/pom.xml
@@ -158,6 +158,10 @@ language governing permissions and limitations under the License. -->
<groupId>org.codehaus.groovy</groupId>
<artifactId>groovy-groovysh</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-ant</artifactId>
+ </exclusion>
</exclusions>
</dependency>
</dependencies>
diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml b/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml
index ad99b3c295..95496b4e6b 100644
--- a/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml
+++ b/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml
@@ -148,6 +148,10 @@
<groupId>org.codehaus.groovy</groupId>
<artifactId>groovy-groovysh</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-ant</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
[nifi] 01/09: NIFI-9924 Corrected text encoding in PutEmail filenames and TestFTP
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a commit to branch support/nifi-1.16
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit f50b615a1243f8ed93446bec70cf8645abf8da8a
Author: Paul Grey <gr...@yahoo.com>
AuthorDate: Thu Apr 14 11:53:50 2022 -0400
NIFI-9924 Corrected text encoding in PutEmail filenames and TestFTP
This closes #5967
Signed-off-by: David Handermann <ex...@apache.org>
---
.../apache/nifi/processors/standard/PutEmail.java | 2 +-
.../apache/nifi/processors/standard/TestFTP.java | 23 ++++++++++++----------
2 files changed, 14 insertions(+), 11 deletions(-)
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutEmail.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutEmail.java
index f03da5b737..c083f26abe 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutEmail.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutEmail.java
@@ -382,7 +382,7 @@ public class PutEmail extends AbstractProcessor {
}
});
- mimeFile.setFileName(MimeUtility.encodeText(flowFile.getAttribute(CoreAttributes.FILENAME.key())));
+ mimeFile.setFileName(MimeUtility.encodeText(flowFile.getAttribute(CoreAttributes.FILENAME.key()), CONTENT_CHARSET.name(), null));
final MimeMultipart multipart = new MimeMultipart();
multipart.addBodyPart(mimeText);
multipart.addBodyPart(mimeFile);
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestFTP.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestFTP.java
index 5809ea4359..1993aad6ea 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestFTP.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestFTP.java
@@ -35,10 +35,11 @@ import org.apache.nifi.util.MockFlowFile;
import org.apache.nifi.util.MockProcessContext;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
import org.mockftpserver.fake.FakeFtpServer;
import org.mockftpserver.fake.UserAccount;
import org.mockftpserver.fake.filesystem.DirectoryEntry;
@@ -53,7 +54,7 @@ public class TestFTP {
final String password = "Test test test chocolate";
int ftpPort;
- @Before
+ @BeforeEach
public void setUp() throws Exception {
fakeFtpServer.setServerControlPort(0);
fakeFtpServer.addUserAccount(new UserAccount(username, password, "c:\\data"));
@@ -67,7 +68,7 @@ public class TestFTP {
ftpPort = fakeFtpServer.getServerControlPort();
}
- @After
+ @AfterEach
public void tearDown() throws Exception {
fakeFtpServer.stop();
}
@@ -142,7 +143,7 @@ public class TestFTP {
FileSystem results = fakeFtpServer.getFileSystem();
// Check file was uploaded
- Assert.assertTrue(results.exists("c:\\data\\randombytes-1"));
+ Assertions.assertTrue(results.exists("c:\\data\\randombytes-1"));
}
@Test
@@ -205,7 +206,7 @@ public class TestFTP {
results.add(sampleFile);
// Check file exists
- Assert.assertTrue(results.exists("c:\\data\\randombytes-2"));
+ Assertions.assertTrue(results.exists("c:\\data\\randombytes-2"));
TestRunner runner = TestRunners.newTestRunner(GetFTP.class);
runner.setProperty(FTPTransfer.HOSTNAME, "localhost");
@@ -229,7 +230,7 @@ public class TestFTP {
results.add(sampleFile);
// Check file exists
- Assert.assertTrue(results.exists("c:\\data\\randombytes-2"));
+ Assertions.assertTrue(results.exists("c:\\data\\randombytes-2"));
TestRunner runner = TestRunners.newTestRunner(FetchFTP.class);
runner.setProperty(FetchFTP.HOSTNAME, "${host}");
@@ -254,6 +255,8 @@ public class TestFTP {
}
@Test
+ @EnabledIfSystemProperty(named = "file.encoding", matches = "UTF-8",
+ disabledReason = "org.mockftpserver does not support specification of charset")
public void basicFileFetchWithUTF8FileName() throws IOException {
FileSystem fs = fakeFtpServer.getFileSystem();
@@ -289,7 +292,7 @@ public class TestFTP {
results.add(sampleFile);
// Check file exists
- Assert.assertTrue(results.exists("c:\\data\\randombytes-2"));
+ Assertions.assertTrue(results.exists("c:\\data\\randombytes-2"));
TestRunner runner = TestRunners.newTestRunner(ListFTP.class);
runner.setProperty(ListFTP.HOSTNAME, "localhost");
[nifi] 03/09: NIFI-9899 Corrected MongoDBLookupService attribute handling
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a commit to branch support/nifi-1.16
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit d6176fea41c3b703443c5d77ca19b20aac7ef04a
Author: ravinarayansingh <ra...@gmail.com>
AuthorDate: Thu Apr 14 10:05:48 2022 -0500
NIFI-9899 Corrected MongoDBLookupService attribute handling
- Corrected collection and database name properties to handle FlowFile attributes
This closes #5966
Signed-off-by: David Handermann <ex...@apache.org>
---
.../java/org/apache/nifi/mongodb/MongoDBLookupService.java | 13 ++++---------
1 file changed, 4 insertions(+), 9 deletions(-)
diff --git a/nifi-nar-bundles/nifi-mongodb-bundle/nifi-mongodb-services/src/main/java/org/apache/nifi/mongodb/MongoDBLookupService.java b/nifi-nar-bundles/nifi-mongodb-bundle/nifi-mongodb-services/src/main/java/org/apache/nifi/mongodb/MongoDBLookupService.java
index d97c5decf8..ecf1db1659 100644
--- a/nifi-nar-bundles/nifi-mongodb-bundle/nifi-mongodb-services/src/main/java/org/apache/nifi/mongodb/MongoDBLookupService.java
+++ b/nifi-nar-bundles/nifi-mongodb-bundle/nifi-mongodb-services/src/main/java/org/apache/nifi/mongodb/MongoDBLookupService.java
@@ -70,8 +70,6 @@ public class MongoDBLookupService extends JsonInferenceSchemaRegistryService imp
.fromPropertyDescriptor(SCHEMA_NAME)
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
.build();
- private volatile String databaseName;
- private volatile String collection;
public static final PropertyDescriptor CONTROLLER_SERVICE = new PropertyDescriptor.Builder()
.name("mongo-lookup-client-service")
@@ -138,7 +136,7 @@ public class MongoDBLookupService extends JsonInferenceSchemaRegistryService imp
}
try {
- Document result = findOne(query, projection);
+ Document result = findOne(query, projection,context);
if(result == null) {
return Optional.empty();
@@ -173,17 +171,12 @@ public class MongoDBLookupService extends JsonInferenceSchemaRegistryService imp
this.controllerService = context.getProperty(CONTROLLER_SERVICE).asControllerService(MongoDBClientService.class);
this.schemaNameProperty = context.getProperty(LOCAL_SCHEMA_NAME).evaluateAttributeExpressions().getValue();
-
- this.databaseName = context.getProperty(DATABASE_NAME).evaluateAttributeExpressions().getValue();
- this.collection = context.getProperty(COLLECTION_NAME).evaluateAttributeExpressions().getValue();
-
String configuredProjection = context.getProperty(PROJECTION).isSet()
? context.getProperty(PROJECTION).getValue()
: null;
if (!StringUtils.isBlank(configuredProjection)) {
projection = Document.parse(configuredProjection);
}
-
super.onEnabled(context);
}
@@ -223,7 +216,9 @@ public class MongoDBLookupService extends JsonInferenceSchemaRegistryService imp
return Collections.unmodifiableList(_temp);
}
- private Document findOne(Document query, Document projection) {
+ private Document findOne(Document query, Document projection,Map<String, String> context) {
+ final String databaseName = getProperty(DATABASE_NAME).evaluateAttributeExpressions(context).getValue();
+ final String collection = getProperty(COLLECTION_NAME).evaluateAttributeExpressions(context).getValue();
MongoCollection col = controllerService.getDatabase(databaseName).getCollection(collection);
MongoCursor<Document> it = (projection != null ? col.find(query).projection(projection) : col.find(query)).iterator();
Document retVal = it.hasNext() ? it.next() : null;
[nifi] 05/09: NIFI-9935 Upgraded Zip4j from 2.9.1 to 2.10.0
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a commit to branch support/nifi-1.16
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit 14c1470755d3171b4d0499443419c7d5b57dbde1
Author: exceptionfactory <ex...@apache.org>
AuthorDate: Mon Apr 18 19:03:17 2022 -0500
NIFI-9935 Upgraded Zip4j from 2.9.1 to 2.10.0
Signed-off-by: Pierre Villard <pi...@gmail.com>
This closes #5975.
---
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
index 0bd7208d4d..dd1a7f2f1e 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
@@ -336,7 +336,7 @@
<dependency>
<groupId>net.lingala.zip4j</groupId>
<artifactId>zip4j</artifactId>
- <version>2.9.1</version>
+ <version>2.10.0</version>
</dependency>
<dependency>
<groupId>com.h2database</groupId>
[nifi] 09/09: NIFI-9868 fix version references to 1.16.1-SNAP after all cherry-picks
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a commit to branch support/nifi-1.16
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit 927f31129a3c5fd8ed6cbc8726f439b79003151f
Author: Joe Witt <jo...@apache.org>
AuthorDate: Tue Apr 19 09:24:54 2022 -0700
NIFI-9868 fix version references to 1.16.1-SNAP after all cherry-picks
---
nifi-assembly/pom.xml | 6 +++---
nifi-commons/nifi-property-protection-loader/pom.xml | 4 ++--
nifi-nar-bundles/nifi-framework-bundle/pom.xml | 2 +-
.../nifi-prometheus-bundle/nifi-prometheus-reporting-task/pom.xml | 2 +-
nifi-registry/pom.xml | 2 +-
nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml | 2 +-
6 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/nifi-assembly/pom.xml b/nifi-assembly/pom.xml
index c3f4d87362..ab6b868650 100644
--- a/nifi-assembly/pom.xml
+++ b/nifi-assembly/pom.xml
@@ -170,17 +170,17 @@ language governing permissions and limitations under the License. -->
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-property-utils</artifactId>
- <version>1.17.0-SNAPSHOT</version>
+ <version>1.16.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-property-protection-api</artifactId>
- <version>1.17.0-SNAPSHOT</version>
+ <version>1.16.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-property-protection-factory</artifactId>
- <version>1.17.0-SNAPSHOT</version>
+ <version>1.16.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
diff --git a/nifi-commons/nifi-property-protection-loader/pom.xml b/nifi-commons/nifi-property-protection-loader/pom.xml
index f98dbc5d71..339f4b16ed 100644
--- a/nifi-commons/nifi-property-protection-loader/pom.xml
+++ b/nifi-commons/nifi-property-protection-loader/pom.xml
@@ -18,14 +18,14 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-commons</artifactId>
- <version>1.17.0-SNAPSHOT</version>
+ <version>1.16.1-SNAPSHOT</version>
</parent>
<artifactId>nifi-property-protection-loader</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-property-protection-api</artifactId>
- <version>1.17.0-SNAPSHOT</version>
+ <version>1.16.1-SNAPSHOT</version>
</dependency>
</dependencies>
</project>
diff --git a/nifi-nar-bundles/nifi-framework-bundle/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/pom.xml
index eeeb1de35b..2ec78d32b0 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/pom.xml
+++ b/nifi-nar-bundles/nifi-framework-bundle/pom.xml
@@ -158,7 +158,7 @@
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-property-protection-loader</artifactId>
- <version>1.17.0-SNAPSHOT</version>
+ <version>1.16.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
diff --git a/nifi-nar-bundles/nifi-prometheus-bundle/nifi-prometheus-reporting-task/pom.xml b/nifi-nar-bundles/nifi-prometheus-bundle/nifi-prometheus-reporting-task/pom.xml
index 0169af6636..c77032b566 100644
--- a/nifi-nar-bundles/nifi-prometheus-bundle/nifi-prometheus-reporting-task/pom.xml
+++ b/nifi-nar-bundles/nifi-prometheus-bundle/nifi-prometheus-reporting-task/pom.xml
@@ -50,7 +50,7 @@
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-ssl-context-service-api</artifactId>
- <version>1.17.0-SNAPSHOT</version>
+ <version>1.16.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
diff --git a/nifi-registry/pom.xml b/nifi-registry/pom.xml
index 4e8174b0f7..d304004698 100644
--- a/nifi-registry/pom.xml
+++ b/nifi-registry/pom.xml
@@ -130,7 +130,7 @@
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-property-protection-loader</artifactId>
- <version>1.17.0-SNAPSHOT</version>
+ <version>1.16.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml b/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml
index 326d600d90..6f50bb6cf5 100644
--- a/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml
+++ b/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml
@@ -47,7 +47,7 @@
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-property-protection-loader</artifactId>
- <version>1.17.0-SNAPSHOT</version>
+ <version>1.16.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi.registry</groupId>
[nifi] 02/09: NIFI-9862 Updated JsonTreeReader to read Records from Nested Field
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a commit to branch support/nifi-1.16
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit fc2149f610634aac3672c1a3f3c721e917bd0949
Author: Lehel <le...@hotmail.com>
AuthorDate: Wed Apr 6 21:50:22 2022 +0200
NIFI-9862 Updated JsonTreeReader to read Records from Nested Field
This closes #5937
Signed-off-by: David Handermann <ex...@apache.org>
---
.../nifi-record-serialization-services/pom.xml | 3 +
.../nifi/json/AbstractJsonRowRecordReader.java | 98 ++++-
.../org/apache/nifi/json/JsonRecordSource.java | 26 +-
.../java/org/apache/nifi/json/JsonTreeReader.java | 59 ++-
.../apache/nifi/json/JsonTreeRowRecordReader.java | 12 +-
.../apache/nifi/json/StartingFieldStrategy.java | 38 ++
.../additionalDetails.html | 398 ++++++++++-------
.../json/TestInferJsonSchemaAccessStrategy.java | 107 +++--
.../nifi/json/TestJsonPathRowRecordReader.java | 74 ++--
.../apache/nifi/json/TestJsonSchemaInference.java | 4 +-
.../nifi/json/TestJsonTreeRowRecordReader.java | 490 +++++++++++++--------
.../org/apache/nifi/json/TestWriteJsonResult.java | 38 +-
.../test/resources/json/multiple-nested-field.json | 22 +
.../json/nested-array-then-start-object.json | 20 +
.../json/single-element-nested-array-middle.json | 16 +
15 files changed, 932 insertions(+), 473 deletions(-)
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/pom.xml b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/pom.xml
index 48c8aaa048..cca137d0e1 100755
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/pom.xml
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/pom.xml
@@ -206,11 +206,14 @@
<exclude>src/test/resources/json/data-types.json</exclude>
<exclude>src/test/resources/json/timestamp.json</exclude>
<exclude>src/test/resources/json/json-with-unicode.json</exclude>
+ <exclude>src/test/resources/json/multiple-nested-field.json</exclude>
<exclude>src/test/resources/json/primitive-type-array.json</exclude>
<exclude>src/test/resources/json/single-bank-account.json</exclude>
<exclude>src/test/resources/json/single-bank-account-wrong-field-type.json</exclude>
<exclude>src/test/resources/json/single-element-nested-array.json</exclude>
<exclude>src/test/resources/json/single-element-nested.json</exclude>
+ <exclude>src/test/resources/json/single-element-nested-array-middle.json</exclude>
+ <exclude>src/test/resources/json/nested-array-then-start-object.json</exclude>
<exclude>src/test/resources/json/output/dataTypes.json</exclude>
<exclude>src/test/resources/json/elements-for-record-choice.json</exclude>
<exclude>src/test/resources/json/record-choice.avsc</exclude>
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/AbstractJsonRowRecordReader.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/AbstractJsonRowRecordReader.java
index 51d9533992..e248bfd518 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/AbstractJsonRowRecordReader.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/AbstractJsonRowRecordReader.java
@@ -21,6 +21,7 @@ import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.core.io.SerializedString;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
@@ -51,8 +52,6 @@ import java.util.function.Supplier;
public abstract class AbstractJsonRowRecordReader implements RecordReader {
private final ComponentLog logger;
- private final JsonParser jsonParser;
- private final JsonNode firstJsonNode;
private final Supplier<DateFormat> LAZY_DATE_FORMAT;
private final Supplier<DateFormat> LAZY_TIME_FORMAT;
private final Supplier<DateFormat> LAZY_TIMESTAMP_FORMAT;
@@ -61,11 +60,12 @@ public abstract class AbstractJsonRowRecordReader implements RecordReader {
private static final JsonFactory jsonFactory = new JsonFactory();
private static final ObjectMapper codec = new ObjectMapper();
+ private JsonParser jsonParser;
+ private JsonNode firstJsonNode;
+ private StartingFieldStrategy strategy;
- public AbstractJsonRowRecordReader(final InputStream in, final ComponentLog logger, final String dateFormat, final String timeFormat, final String timestampFormat)
- throws IOException, MalformedRecordException {
-
+ private AbstractJsonRowRecordReader(final ComponentLog logger, final String dateFormat, final String timeFormat, final String timestampFormat) {
this.logger = logger;
final DateFormat df = dateFormat == null ? null : DataTypeUtils.getDateFormat(dateFormat);
@@ -75,9 +75,15 @@ public abstract class AbstractJsonRowRecordReader implements RecordReader {
LAZY_DATE_FORMAT = () -> df;
LAZY_TIME_FORMAT = () -> tf;
LAZY_TIMESTAMP_FORMAT = () -> tsf;
+ }
+
+ protected AbstractJsonRowRecordReader(final InputStream in, final ComponentLog logger, final String dateFormat, final String timeFormat, final String timestampFormat)
+ throws IOException, MalformedRecordException {
+
+ this(logger, dateFormat, timeFormat, timestampFormat);
try {
- jsonParser = jsonFactory.createJsonParser(in);
+ jsonParser = jsonFactory.createParser(in);
jsonParser.setCodec(codec);
JsonToken token = jsonParser.nextToken();
@@ -95,6 +101,37 @@ public abstract class AbstractJsonRowRecordReader implements RecordReader {
}
}
+ protected AbstractJsonRowRecordReader(final InputStream in, final ComponentLog logger, final String dateFormat, final String timeFormat, final String timestampFormat,
+ final StartingFieldStrategy strategy, final String nestedFieldName) throws IOException, MalformedRecordException {
+
+ this(logger, dateFormat, timeFormat, timestampFormat);
+
+ this.strategy = strategy;
+
+ try {
+ jsonParser = jsonFactory.createParser(in);
+ jsonParser.setCodec(codec);
+
+ if (strategy == StartingFieldStrategy.NESTED_FIELD) {
+ final SerializedString serializedStartingFieldName = new SerializedString(nestedFieldName);
+ while (!jsonParser.nextFieldName(serializedStartingFieldName) && jsonParser.hasCurrentToken());
+ logger.debug("Parsing starting at nested field [{}]", nestedFieldName);
+ }
+
+ JsonToken token = jsonParser.nextToken();
+ if (token == JsonToken.START_ARRAY) {
+ token = jsonParser.nextToken(); // advance to START_OBJECT token
+ }
+ if (token == JsonToken.START_OBJECT) { // could be END_ARRAY also
+ firstJsonNode = jsonParser.readValueAsTree();
+ } else {
+ firstJsonNode = null;
+ }
+ } catch (final JsonParseException e) {
+ throw new MalformedRecordException("Could not parse data as JSON", e);
+ }
+ }
+
protected Supplier<DateFormat> getLazyDateFormat() {
return LAZY_DATE_FORMAT;
}
@@ -121,7 +158,7 @@ public abstract class AbstractJsonRowRecordReader implements RecordReader {
} catch (final MalformedRecordException mre) {
throw mre;
} catch (final Exception e) {
- logger.debug("Failed to convert JSON Element {} into a Record object using schema {} due to {}", new Object[] {nextNode, schema, e.toString(), e});
+ logger.debug("Failed to convert JSON Element {} into a Record object using schema {} due to {}", nextNode, schema, e.toString(), e);
throw new MalformedRecordException("Successfully parsed a JSON object from input but failed to convert into a Record object with the given schema", e);
}
}
@@ -178,11 +215,11 @@ public abstract class AbstractJsonRowRecordReader implements RecordReader {
final ArrayDataType arrayDataType = (ArrayDataType) dataType;
elementDataType = arrayDataType.getElementType();
} else if (dataType != null && dataType.getFieldType() == RecordFieldType.CHOICE) {
- List<DataType> possibleSubTypes = ((ChoiceDataType)dataType).getPossibleSubTypes();
+ List<DataType> possibleSubTypes = ((ChoiceDataType) dataType).getPossibleSubTypes();
for (DataType possibleSubType : possibleSubTypes) {
if (possibleSubType.getFieldType() == RecordFieldType.ARRAY) {
- ArrayDataType possibleArrayDataType = (ArrayDataType)possibleSubType;
+ ArrayDataType possibleArrayDataType = (ArrayDataType) possibleSubType;
DataType possibleElementType = possibleArrayDataType.getElementType();
final Object[] possibleArrayElements = new Object[numElements];
@@ -214,7 +251,6 @@ public abstract class AbstractJsonRowRecordReader implements RecordReader {
}
if (fieldNode.isObject()) {
- RecordSchema childSchema = null;
if (dataType != null && RecordFieldType.MAP == dataType.getFieldType()) {
return getMapFromRawValue(fieldNode, dataType, fieldName);
}
@@ -291,8 +327,7 @@ public abstract class AbstractJsonRowRecordReader implements RecordReader {
} else if (dataType.getFieldType() == RecordFieldType.ARRAY) {
final ArrayDataType arrayDataType = (ArrayDataType) dataType;
final DataType elementType = arrayDataType.getElementType();
- final Record record = createOptionalRecord(fieldNode, elementType, strict);
- return record;
+ return createOptionalRecord(fieldNode, elementType, strict);
}
return null;
@@ -309,8 +344,7 @@ public abstract class AbstractJsonRowRecordReader implements RecordReader {
childValues.put(childFieldName, childValue);
}
- final MapRecord record = new MapRecord(childSchema, childValues);
- return record;
+ return new MapRecord(childSchema, childValues);
}
protected JsonNode getNextJsonNode() throws IOException, MalformedRecordException {
@@ -318,7 +352,15 @@ public abstract class AbstractJsonRowRecordReader implements RecordReader {
firstObjectConsumed = true;
return firstJsonNode;
}
+ if (strategy == StartingFieldStrategy.NESTED_FIELD) {
+ return getJsonNodeWithNestedNodeStrategy();
+ } else {
+ return getJsonNode();
+ }
+ }
+
+ private JsonNode getJsonNodeWithNestedNodeStrategy() throws IOException, MalformedRecordException {
while (true) {
final JsonToken token = jsonParser.nextToken();
if (token == null) {
@@ -326,14 +368,34 @@ public abstract class AbstractJsonRowRecordReader implements RecordReader {
}
switch (token) {
+ case START_ARRAY:
+ break;
+ case END_ARRAY:
case END_OBJECT:
- continue;
+ case FIELD_NAME:
+ return null;
case START_OBJECT:
return jsonParser.readValueAsTree();
- case END_ARRAY:
- case START_ARRAY:
- continue;
+ default:
+ throw new MalformedRecordException("Expected to get a JSON Object but got a token of type " + token.name());
+ }
+ }
+ }
+
+ private JsonNode getJsonNode() throws IOException, MalformedRecordException {
+ while (true) {
+ final JsonToken token = jsonParser.nextToken();
+ if (token == null) {
+ return null;
+ }
+ switch (token) {
+ case START_ARRAY:
+ case END_ARRAY:
+ case END_OBJECT:
+ break;
+ case START_OBJECT:
+ return jsonParser.readValueAsTree();
default:
throw new MalformedRecordException("Expected to get a JSON Object but got a token of type " + token.name());
}
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/JsonRecordSource.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/JsonRecordSource.java
index 1c6afce9d4..348c2ef02f 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/JsonRecordSource.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/JsonRecordSource.java
@@ -19,16 +19,22 @@ package org.apache.nifi.json;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.core.io.SerializedString;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.nifi.schema.inference.RecordSource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InputStream;
public class JsonRecordSource implements RecordSource<JsonNode> {
+ private static final Logger logger = LoggerFactory.getLogger(JsonRecordSource.class);
private static final JsonFactory jsonFactory;
private final JsonParser jsonParser;
+ private final StartingFieldStrategy strategy;
+ private final String startingFieldName;
static {
jsonFactory = new JsonFactory();
@@ -36,7 +42,21 @@ public class JsonRecordSource implements RecordSource<JsonNode> {
}
public JsonRecordSource(final InputStream in) throws IOException {
- jsonParser = jsonFactory.createJsonParser(in);
+ jsonParser = jsonFactory.createParser(in);
+ strategy = null;
+ startingFieldName = null;
+ }
+
+ public JsonRecordSource(final InputStream in, final StartingFieldStrategy strategy, final String startingFieldName) throws IOException {
+ jsonParser = jsonFactory.createParser(in);
+ this.strategy = strategy;
+ this.startingFieldName = startingFieldName;
+
+ if (strategy == StartingFieldStrategy.NESTED_FIELD) {
+ final SerializedString serializedNestedField = new SerializedString(this.startingFieldName);
+ while (!jsonParser.nextFieldName(serializedNestedField) && jsonParser.hasCurrentToken());
+ logger.debug("Parsing starting at nested field [{}]", startingFieldName);
+ }
}
@Override
@@ -50,6 +70,10 @@ public class JsonRecordSource implements RecordSource<JsonNode> {
if (token == JsonToken.START_OBJECT) {
return jsonParser.readValueAsTree();
}
+
+ if (strategy == StartingFieldStrategy.NESTED_FIELD && (token == JsonToken.END_ARRAY || token == JsonToken.END_OBJECT)) {
+ return null;
+ }
}
}
}
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/JsonTreeReader.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/JsonTreeReader.java
index b0ecad1c80..e5a2ed20fd 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/JsonTreeReader.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/JsonTreeReader.java
@@ -27,10 +27,11 @@ import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.context.PropertyContext;
import org.apache.nifi.controller.ConfigurationContext;
import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.schema.access.SchemaAccessStrategy;
import org.apache.nifi.schema.access.SchemaNotFoundException;
-import org.apache.nifi.schema.inference.SchemaInferenceEngine;
import org.apache.nifi.schema.inference.RecordSourceFactory;
+import org.apache.nifi.schema.inference.SchemaInferenceEngine;
import org.apache.nifi.schema.inference.SchemaInferenceUtil;
import org.apache.nifi.schema.inference.TimeValueInference;
import org.apache.nifi.schemaregistry.services.SchemaRegistry;
@@ -44,6 +45,7 @@ import org.apache.nifi.serialization.record.RecordSchema;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;
@@ -59,21 +61,49 @@ import static org.apache.nifi.schema.inference.SchemaInferenceUtil.SCHEMA_CACHE;
+ "If an array is encountered, each element in that array will be treated as a separate record. "
+ "If the schema that is configured contains a field that is not present in the JSON, a null value will be used. If the JSON contains "
+ "a field that is not present in the schema, that field will be skipped. "
- + "See the Usage of the Controller Service for more information and examples.")
+ + "See the Usage of the Controller Service for more information and examples.")
@SeeAlso(JsonPathReader.class)
public class JsonTreeReader extends SchemaRegistryService implements RecordReaderFactory {
private volatile String dateFormat;
private volatile String timeFormat;
private volatile String timestampFormat;
+ private volatile String startingFieldName;
+ private volatile StartingFieldStrategy startingFieldStrategy;
+
+ public static final PropertyDescriptor STARTING_FIELD_STRATEGY = new PropertyDescriptor.Builder()
+ .name("starting-field-strategy")
+ .displayName("Starting Field Strategy")
+ .description("Start processing from the root node or from a specified nested node.")
+ .required(true)
+ .addValidator(StandardValidators.NON_BLANK_VALIDATOR)
+ .defaultValue(StartingFieldStrategy.ROOT_NODE.name())
+ .allowableValues(
+ Arrays.stream(StartingFieldStrategy.values()).map(startingStrategy ->
+ new AllowableValue(startingStrategy.name(), startingStrategy.getDisplayName(), startingStrategy.getDescription())
+ ).toArray(AllowableValue[]::new))
+ .build();
+
+
+ public static final PropertyDescriptor STARTING_FIELD_NAME = new PropertyDescriptor.Builder()
+ .name("starting-field-name")
+ .displayName("Starting Field Name")
+ .description("Skips forward to the given nested JSON field (array or object) to begin processing.")
+ .required(false)
+ .addValidator(StandardValidators.NON_BLANK_VALIDATOR)
+ .defaultValue(null)
+ .dependsOn(STARTING_FIELD_STRATEGY, StartingFieldStrategy.NESTED_FIELD.name())
+ .build();
@Override
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
final List<PropertyDescriptor> properties = new ArrayList<>(super.getSupportedPropertyDescriptors());
properties.add(new PropertyDescriptor.Builder()
- .fromPropertyDescriptor(SCHEMA_CACHE)
- .dependsOn(SCHEMA_ACCESS_STRATEGY, INFER_SCHEMA)
- .build());
+ .fromPropertyDescriptor(SCHEMA_CACHE)
+ .dependsOn(SCHEMA_ACCESS_STRATEGY, INFER_SCHEMA)
+ .build());
+ properties.add(STARTING_FIELD_STRATEGY);
+ properties.add(STARTING_FIELD_NAME);
properties.add(DateTimeUtils.DATE_FORMAT);
properties.add(DateTimeUtils.TIME_FORMAT);
properties.add(DateTimeUtils.TIMESTAMP_FORMAT);
@@ -81,10 +111,12 @@ public class JsonTreeReader extends SchemaRegistryService implements RecordReade
}
@OnEnabled
- public void storeFormats(final ConfigurationContext context) {
+ public void storePropertyValues(final ConfigurationContext context) {
this.dateFormat = context.getProperty(DateTimeUtils.DATE_FORMAT).getValue();
this.timeFormat = context.getProperty(DateTimeUtils.TIME_FORMAT).getValue();
this.timestampFormat = context.getProperty(DateTimeUtils.TIMESTAMP_FORMAT).getValue();
+ this.startingFieldStrategy = StartingFieldStrategy.valueOf(context.getProperty(STARTING_FIELD_STRATEGY).getValue());
+ this.startingFieldName = context.getProperty(STARTING_FIELD_NAME).getValue();
}
@Override
@@ -96,12 +128,15 @@ public class JsonTreeReader extends SchemaRegistryService implements RecordReade
}
@Override
- protected SchemaAccessStrategy getSchemaAccessStrategy(final String strategy, final SchemaRegistry schemaRegistry, final PropertyContext context) {
- final RecordSourceFactory<JsonNode> jsonSourceFactory = (var, in) -> new JsonRecordSource(in);
- final Supplier<SchemaInferenceEngine<JsonNode>> inferenceSupplier = () -> new JsonSchemaInference(new TimeValueInference(dateFormat, timeFormat, timestampFormat));
+ protected SchemaAccessStrategy getSchemaAccessStrategy(final String schemaAccessStrategy, final SchemaRegistry schemaRegistry, final PropertyContext context) {
+ final RecordSourceFactory<JsonNode> jsonSourceFactory =
+ (var, in) -> new JsonRecordSource(in, startingFieldStrategy, startingFieldName);
+
+ final Supplier<SchemaInferenceEngine<JsonNode>> inferenceSupplier =
+ () -> new JsonSchemaInference(new TimeValueInference(dateFormat, timeFormat, timestampFormat));
- return SchemaInferenceUtil.getSchemaAccessStrategy(strategy, context, getLogger(), jsonSourceFactory, inferenceSupplier,
- () -> super.getSchemaAccessStrategy(strategy, schemaRegistry, context));
+ return SchemaInferenceUtil.getSchemaAccessStrategy(schemaAccessStrategy, context, getLogger(), jsonSourceFactory, inferenceSupplier,
+ () -> super.getSchemaAccessStrategy(schemaAccessStrategy, schemaRegistry, context));
}
@Override
@@ -113,6 +148,6 @@ public class JsonTreeReader extends SchemaRegistryService implements RecordReade
public RecordReader createRecordReader(final Map<String, String> variables, final InputStream in, final long inputLength, final ComponentLog logger)
throws IOException, MalformedRecordException, SchemaNotFoundException {
final RecordSchema schema = getSchema(variables, in, null);
- return new JsonTreeRowRecordReader(in, logger, schema, dateFormat, timeFormat, timestampFormat);
+ return new JsonTreeRowRecordReader(in, logger, schema, dateFormat, timeFormat, timestampFormat, startingFieldStrategy, startingFieldName);
}
}
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/JsonTreeRowRecordReader.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/JsonTreeRowRecordReader.java
index 1bb34549c8..eee487c026 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/JsonTreeRowRecordReader.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/JsonTreeRowRecordReader.java
@@ -53,7 +53,12 @@ public class JsonTreeRowRecordReader extends AbstractJsonRowRecordReader {
this.schema = schema;
}
-
+ public JsonTreeRowRecordReader(final InputStream in, final ComponentLog logger, final RecordSchema schema,
+ final String dateFormat, final String timeFormat, final String timestampFormat,
+ final StartingFieldStrategy strategy, final String startingFieldName) throws IOException, MalformedRecordException {
+ super(in, logger, dateFormat, timeFormat, timestampFormat, strategy, startingFieldName);
+ this.schema = schema;
+ }
@Override
protected Record convertJsonNodeToRecord(final JsonNode jsonNode, final RecordSchema schema, final boolean coerceTypes, final boolean dropUnknownFields)
@@ -104,7 +109,7 @@ public class JsonTreeRowRecordReader extends AbstractJsonRowRecordReader {
final String fullFieldName = fieldNamePrefix == null ? fieldName : fieldNamePrefix + fieldName;
value = convertField(childNode, fullFieldName, desiredType, dropUnknown);
} else {
- value = getRawNodeValue(childNode, recordField == null ? null : recordField.getDataType(), fieldName);
+ value = getRawNodeValue(childNode, recordField.getDataType(), fieldName);
}
values.put(fieldName, value);
@@ -157,8 +162,7 @@ public class JsonTreeRowRecordReader extends AbstractJsonRowRecordReader {
case TIME:
case TIMESTAMP: {
final Object rawValue = getRawNodeValue(fieldNode, fieldName);
- final Object converted = DataTypeUtils.convertType(rawValue, desiredType, getLazyDateFormat(), getLazyTimeFormat(), getLazyTimestampFormat(), fieldName);
- return converted;
+ return DataTypeUtils.convertType(rawValue, desiredType, getLazyDateFormat(), getLazyTimeFormat(), getLazyTimestampFormat(), fieldName);
}
case MAP: {
final DataType valueType = ((MapDataType) desiredType).getValueType();
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/StartingFieldStrategy.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/StartingFieldStrategy.java
new file mode 100644
index 0000000000..44727e72fd
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/StartingFieldStrategy.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.json;
+
+public enum StartingFieldStrategy {
+ ROOT_NODE("Root Node", "Begins processing from the root node."),
+ NESTED_FIELD("Nested Field", "Skips forward to the given nested JSON field (array or object) to begin processing.");
+
+ private final String displayName;
+ private final String description;
+
+ StartingFieldStrategy(final String displayName, final String description) {
+ this.displayName = displayName;
+ this.description = description;
+ }
+
+ public String getDisplayName() {
+ return displayName;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+}
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/resources/docs/org.apache.nifi.json.JsonTreeReader/additionalDetails.html b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/resources/docs/org.apache.nifi.json.JsonTreeReader/additionalDetails.html
index b5415c3b74..9a563d4f65 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/resources/docs/org.apache.nifi.json.JsonTreeReader/additionalDetails.html
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/resources/docs/org.apache.nifi.json.JsonTreeReader/additionalDetails.html
@@ -22,59 +22,60 @@
<body>
<p>
- The JsonTreeReader Controller Service reads a JSON Object and creates a Record object for the entire
- JSON Object tree. The Controller Service must be configured with a Schema that describes the structure
- of the JSON data. If any field exists in the JSON that is not in the schema, that field will be skipped.
- If the schema contains a field for which no JSON field exists, a null value will be used in the Record
- (or the default value defined in the schema, if applicable).
+ The JsonTreeReader Controller Service reads a JSON Object and creates a Record object either for the
+ entire JSON Object tree or a subpart (see "Starting Field Strategies" section). The Controller Service
+ must be configured with a Schema that describes the structure of the JSON data. If any field exists in
+ the JSON that is not in the schema, that field will be skipped. If the schema contains a field for which
+ no JSON field exists, a null value will be used in the Record (or the default value defined in the schema,
+ if applicable).
</p>
<p>
- If the root element of the JSON is a JSON Array, each JSON Object within that array will be treated as
- its own separate Record. If the root element is a JSON Object, the JSON will all be treated as a single
- Record.
+ If the root element of the JSON is a JSON Array, each JSON Object within that array will be treated as
+ its own separate Record. If the root element is a JSON Object, the JSON will all be treated as a single
+ Record.
</p>
- <h2>Schemas and Type Coercion</h2>
-
- <p>
- When a record is parsed from incoming data, it is separated into fields. Each of these fields is then looked up against the
- configured schema (by field name) in order to determine what the type of the data should be. If the field is not present in
- the schema, that field is omitted from the Record. If the field is found in the schema, the data type of the received data
- is compared against the data type specified in the schema. If the types match, the value of that field is used as-is. If the
- schema indicates that the field should be of a different type, then the Controller Service will attempt to coerce the data
- into the type specified by the schema. If the field cannot be coerced into the specified type, an Exception will be thrown.
- </p>
-
- <p>
- The following rules apply when attempting to coerce a field value from one data type to another:
- </p>
-
- <ul>
- <li>Any data type can be coerced into a String type.</li>
- <li>Any numeric data type (Byte, Short, Int, Long, Float, Double) can be coerced into any other numeric data type.</li>
- <li>Any numeric value can be coerced into a Date, Time, or Timestamp type, by assuming that the Long value is the number of
- milliseconds since epoch (Midnight GMT, January 1, 1970).</li>
- <li>A String value can be coerced into a Date, Time, or Timestamp type, if its format matches the configured "Date Format," "Time Format,"
- or "Timestamp Format."</li>
- <li>A String value can be coerced into a numeric value if the value is of the appropriate type. For example, the String value
- <code>8</code> can be coerced into any numeric type. However, the String value <code>8.2</code> can be coerced into a Double or Float
- type but not an Integer.</li>
- <li>A String value of "true" or "false" (regardless of case) can be coerced into a Boolean value.</li>
- <li>A String value that is not empty can be coerced into a Char type. If the String contains more than 1 character, the first character is used
- and the rest of the characters are ignored.</li>
- <li>Any "date/time" type (Date, Time, Timestamp) can be coerced into any other "date/time" type.</li>
- <li>Any "date/time" type can be coerced into a Long type, representing the number of milliseconds since epoch (Midnight GMT, January 1, 1970).</li>
- <li>Any "date/time" type can be coerced into a String. The format of the String is whatever DateFormat is configured for the corresponding
- property (Date Format, Time Format, Timestamp Format property). If no value is specified, then the value will be converted into a String
- representation of the number of milliseconds since epoch (Midnight GMT, January 1, 1970).</li>
- </ul>
-
- <p>
- If none of the above rules apply when attempting to coerce a value from one data type to another, the coercion will fail and an Exception
- will be thrown.
- </p>
+ <h2>Schemas and Type Coercion</h2>
+
+ <p>
+ When a record is parsed from incoming data, it is separated into fields. Each of these fields is then looked up against the
+ configured schema (by field name) in order to determine what the type of the data should be. If the field is not present in
+ the schema, that field is omitted from the Record. If the field is found in the schema, the data type of the received data
+ is compared against the data type specified in the schema. If the types match, the value of that field is used as-is. If the
+ schema indicates that the field should be of a different type, then the Controller Service will attempt to coerce the data
+ into the type specified by the schema. If the field cannot be coerced into the specified type, an Exception will be thrown.
+ </p>
+
+ <p>
+ The following rules apply when attempting to coerce a field value from one data type to another:
+ </p>
+
+ <ul>
+ <li>Any data type can be coerced into a String type.</li>
+ <li>Any numeric data type (Byte, Short, Int, Long, Float, Double) can be coerced into any other numeric data type.</li>
+ <li>Any numeric value can be coerced into a Date, Time, or Timestamp type, by assuming that the Long value is the number of
+ milliseconds since epoch (Midnight GMT, January 1, 1970).</li>
+ <li>A String value can be coerced into a Date, Time, or Timestamp type, if its format matches the configured "Date Format," "Time Format,"
+ or "Timestamp Format."</li>
+ <li>A String value can be coerced into a numeric value if the value is of the appropriate type. For example, the String value
+ <code>8</code> can be coerced into any numeric type. However, the String value <code>8.2</code> can be coerced into a Double or Float
+ type but not an Integer.</li>
+ <li>A String value of "true" or "false" (regardless of case) can be coerced into a Boolean value.</li>
+ <li>A String value that is not empty can be coerced into a Char type. If the String contains more than 1 character, the first character is used
+ and the rest of the characters are ignored.</li>
+ <li>Any "date/time" type (Date, Time, Timestamp) can be coerced into any other "date/time" type.</li>
+ <li>Any "date/time" type can be coerced into a Long type, representing the number of milliseconds since epoch (Midnight GMT, January 1, 1970).</li>
+ <li>Any "date/time" type can be coerced into a String. The format of the String is whatever DateFormat is configured for the corresponding
+ property (Date Format, Time Format, Timestamp Format property). If no value is specified, then the value will be converted into a String
+ representation of the number of milliseconds since epoch (Midnight GMT, January 1, 1970).</li>
+ </ul>
+
+ <p>
+ If none of the above rules apply when attempting to coerce a value from one data type to another, the coercion will fail and an Exception
+ will be thrown.
+ </p>
@@ -168,165 +169,242 @@
</p>
+ <h2>Starting Field Strategies</h2>
- <h2>Examples</h2>
+ <p>
+ When using JsonTreeReader, two different starting field strategies can be selected. With the default Root Node strategy, the JsonTreeReader begins processing from the root element
+ of the JSON and creates a Record object for the entire JSON Object tree, while the Nested Field strategy defines a nested field from which to begin processing.
+ </p>
+ <p>
+ Using the Nested Field strategy, a schema corresponding to the nested JSON part should be specified. In case of schema inference, the JsonTreeReader will automatically
+ infer a schema from nested records.
+ </p>
+
+ <h3>Root Node Strategy</h3>
<p>
- As an example, consider the following JSON is read:
+ Consider the following JSON is read with the default Root Node strategy:
</p>
<code>
<pre>
-[{
+[
+ {
"id": 17,
"name": "John",
"child": {
- "id": "1"
+ "id": "1"
},
- "dob": "10-29-1982"
+ "dob": "10-29-1982",
"siblings": [
- { "name": "Jeremy", "id": 4 },
- { "name": "Julia", "id": 8}
+ {
+ "name": "Jeremy",
+ "id": 4
+ },
+ {
+ "name": "Julia",
+ "id": 8
+ }
]
},
{
"id": 98,
"name": "Jane",
"child": {
- "id": 2
+ "id": 2
},
- "dob": "08-30-1984"
+ "dob": "08-30-1984",
"gender": "F",
"siblingIds": [],
"siblings": []
- }]
+ }
+]
</pre>
</code>
<p>
- Also, consider that the schema that is configured for this JSON is as follows (assuming that the AvroSchemaRegistry
- Controller Service is chosen to denote the Schema):
+ Also, consider that the schema that is configured for this JSON is as follows (assuming that the AvroSchemaRegistry
+ Controller Service is chosen to denote the Schema):
</p>
<code>
<pre>
{
- "namespace": "nifi",
- "name": "person",
- "type": "record",
- "fields": [
- { "name": "id", "type": "int" },
- { "name": "name", "type": "string" },
- { "name": "gender", "type": "string" },
- { "name": "dob", "type": {
- "type": "int",
- "logicalType": "date"
- }},
- { "name": "siblings", "type": {
- "type": "array",
- "items": {
- "type": "record",
- "fields": [
- { "name": "name", "type": "string" }
- ]
- }
- }}
- ]
+ "namespace": "nifi",
+ "name": "person",
+ "type": "record",
+ "fields": [
+ { "name": "id", "type": "int" },
+ { "name": "name", "type": "string" },
+ { "name": "gender", "type": "string" },
+ { "name": "dob", "type": {
+ "type": "int",
+ "logicalType": "date"
+ }},
+ { "name": "siblings", "type": {
+ "type": "array",
+ "items": {
+ "type": "record",
+ "fields": [
+ { "name": "name", "type": "string" }
+ ]
+ }
+ }}
+ ]
}
</pre>
</code>
<p>
- Let us also assume that this Controller Service is configured with the "Date Format" property set to "MM-dd-yyyy", as this
- matches the date format used for our JSON data. This will result in the JSON creating two separate records, because the root
- element is a JSON array with two elements.
+ Let us also assume that this Controller Service is configured with the "Date Format" property set to "MM-dd-yyyy", as this
+ matches the date format used for our JSON data. This will result in the JSON creating two separate records, because the root
+ element is a JSON array with two elements.
</p>
<p>
- The first Record will consist of the following values:
+ The first Record will consist of the following values:
</p>
<table>
- <tr>
- <th>Field Name</th>
- <th>Field Value</th>
- </tr>
- <tr>
- <td>id</td>
- <td>17</td>
- </tr>
- <tr>
- <td>name</td>
- <td>John</td>
- </tr>
- <tr>
- <td>gender</td>
- <td><i>null</i></td>
- </tr>
- <tr>
- <td>dob</td>
- <td>11-30-1983</td>
- </tr>
- <tr>
- <td>siblings</td>
- <td>
- <i>array with two elements, each of which is itself a Record:</i>
- <br />
- <table>
- <tr>
- <th>Field Name</th>
- <th>Field Value</th>
- </tr>
- <tr>
- <td>name</td>
- <td>Jeremy</td>
- </tr>
- </table>
- <br />
- <i>and:</i>
- <br />
- <table>
- <tr>
- <th>Field Name</th>
- <th>Field Value</th>
- </tr>
- <tr>
- <td>name</td>
- <td>Julia</td>
- </tr>
- </table>
- </td>
- </tr>
+ <tr>
+ <th>Field Name</th>
+ <th>Field Value</th>
+ </tr>
+ <tr>
+ <td>id</td>
+ <td>17</td>
+ </tr>
+ <tr>
+ <td>name</td>
+ <td>John</td>
+ </tr>
+ <tr>
+ <td>gender</td>
+ <td><i>null</i></td>
+ </tr>
+ <tr>
+ <td>dob</td>
+ <td>11-30-1983</td>
+ </tr>
+ <tr>
+ <td>siblings</td>
+ <td>
+ <i>array with two elements, each of which is itself a Record:</i>
+ <br />
+ <table>
+ <tr>
+ <th>Field Name</th>
+ <th>Field Value</th>
+ </tr>
+ <tr>
+ <td>name</td>
+ <td>Jeremy</td>
+ </tr>
+ </table>
+ <br />
+ <i>and:</i>
+ <br />
+ <table>
+ <tr>
+ <th>Field Name</th>
+ <th>Field Value</th>
+ </tr>
+ <tr>
+ <td>name</td>
+ <td>Julia</td>
+ </tr>
+ </table>
+ </td>
+ </tr>
</table>
<p>
- The second Record will consist of the following values:
+ The second Record will consist of the following values:
</p>
- <table>
- <tr>
- <th>Field Name</th>
- <th>Field Value</th>
- </tr>
- <tr>
- <td>id</td>
- <td>98</td>
- </tr>
- <tr>
- <td>name</td>
- <td>Jane</td>
- </tr>
- <tr>
- <td>gender</td>
- <td>F</td>
- </tr>
- <tr>
- <td>dob</td>
- <td>08-30-1984</td>
- </tr>
- <tr>
- <td>siblings</td>
- <td><i>empty array</i></td>
- </tr>
+ <table>
+ <tr>
+ <th>Field Name</th>
+ <th>Field Value</th>
+ </tr>
+ <tr>
+ <td>id</td>
+ <td>98</td>
+ </tr>
+ <tr>
+ <td>name</td>
+ <td>Jane</td>
+ </tr>
+ <tr>
+ <td>gender</td>
+ <td>F</td>
+ </tr>
+ <tr>
+ <td>dob</td>
+ <td>08-30-1984</td>
+ </tr>
+ <tr>
+ <td>siblings</td>
+ <td><i>empty array</i></td>
+ </tr>
+ </table>
+
+ <h3>Nested Field Strategy</h3>
+
+ <p>
+ Using the Nested Field strategy, consider the same JSON where the specified Starting Field Name is
+ "siblings". The schema that is configured for this JSON is as follows:
+ </p>
+
+<code>
+<pre>
+{
+ "namespace": "nifi",
+ "name": "siblings",
+ "type": "record",
+ "fields": [
+ { "name": "name", "type": "string" },
+ { "name": "id", "type": "int" }
+ ]
+}
+</pre>
+</code>
+
+ <p>
+ The first Record will consist of the following values:
+ </p>
+
+ <table>
+ <tr>
+ <th>Field Name</th>
+ <th>Field Value</th>
+ </tr>
+ <tr>
+ <td>name</td>
+ <td>Jeremy</td>
+ </tr>
+ <tr>
+ <td>id</td>
+ <td>4</td>
+ </tr>
+ </table>
+
+ <p>
+ The second Record will consist of the following values:
+ </p>
+
+ <table>
+ <tr>
+ <th>Field Name</th>
+ <th>Field Value</th>
+ </tr>
+ <tr>
+ <td>name</td>
+ <td>Julia</td>
+ </tr>
+ <tr>
+ <td>id</td>
+ <td>8</td>
+ </tr>
</table>
</body>
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/json/TestInferJsonSchemaAccessStrategy.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/json/TestInferJsonSchemaAccessStrategy.java
index 960c7b6127..7ef2089fe2 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/json/TestInferJsonSchemaAccessStrategy.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/json/TestInferJsonSchemaAccessStrategy.java
@@ -29,6 +29,9 @@ import org.apache.nifi.serialization.record.type.ChoiceDataType;
import org.apache.nifi.serialization.record.type.RecordDataType;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
import org.mockito.Mockito;
import java.io.BufferedInputStream;
@@ -41,32 +44,31 @@ import java.nio.file.Files;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertTrue;
-public class TestInferJsonSchemaAccessStrategy {
- private final String dateFormat = RecordFieldType.DATE.getDefaultFormat();
- private final String timeFormat = RecordFieldType.TIME.getDefaultFormat();
- private final String timestampFormat = "yyyy-MM-DD'T'HH:mm:ss.SSS'Z'";
+class TestInferJsonSchemaAccessStrategy {
private final SchemaInferenceEngine<JsonNode> timestampInference = new JsonSchemaInference(new TimeValueInference("yyyy-MM-dd", "HH:mm:ss", "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"));
private final SchemaInferenceEngine<JsonNode> noTimestampInference = new JsonSchemaInference(new TimeValueInference("yyyy-MM-dd", "HH:mm:ss", "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"));
@Test
- @Disabled
- public void testPerformanceOfSchemaInferenceWithTimestamp() throws IOException {
+ @Disabled("Intended only for manual testing to determine performance before/after modifications")
+ void testPerformanceOfSchemaInferenceWithTimestamp() throws IOException {
final File file = new File("src/test/resources/json/prov-events.json");
final byte[] data = Files.readAllBytes(file.toPath());
- final ComponentLog logger = Mockito.mock(ComponentLog.class);
final byte[] manyCopies = new byte[data.length * 20];
- for (int i=0; i < 20; i++) {
+ for (int i = 0; i < 20; i++) {
System.arraycopy(data, 0, manyCopies, data.length * i, data.length);
}
- final InferSchemaAccessStrategy<?> accessStrategy = new InferSchemaAccessStrategy<>((var,content) -> new JsonRecordSource(content), timestampInference, Mockito.mock(ComponentLog.class));
+ final InferSchemaAccessStrategy<?> accessStrategy = new InferSchemaAccessStrategy<>(
+ (var, content) -> new JsonRecordSource(content), timestampInference, Mockito.mock(ComponentLog.class)
+ );
for (int j = 0; j < 10; j++) {
final long start = System.nanoTime();
@@ -83,14 +85,13 @@ public class TestInferJsonSchemaAccessStrategy {
}
@Test
- @Disabled
- public void testPerformanceOfSchemaInferenceWithoutTimestamp() throws IOException {
+ @Disabled("Intended only for manual testing to determine performance before/after modifications")
+ void testPerformanceOfSchemaInferenceWithoutTimestamp() throws IOException {
final File file = new File("src/test/resources/json/prov-events.json");
final byte[] data = Files.readAllBytes(file.toPath());
- final ComponentLog logger = Mockito.mock(ComponentLog.class);
final byte[] manyCopies = new byte[data.length * 20];
- for (int i=0; i < 20; i++) {
+ for (int i = 0; i < 20; i++) {
System.arraycopy(data, 0, manyCopies, data.length * i, data.length);
}
@@ -99,8 +100,8 @@ public class TestInferJsonSchemaAccessStrategy {
for (int i = 0; i < 10_000; i++) {
try (final InputStream in = new ByteArrayInputStream(manyCopies)) {
- final InferSchemaAccessStrategy<?> accessStrategy = new InferSchemaAccessStrategy<>((var,content) -> new JsonRecordSource(content),
- noTimestampInference, Mockito.mock(ComponentLog.class));
+ final InferSchemaAccessStrategy<?> accessStrategy = new InferSchemaAccessStrategy<>((var, content) -> new JsonRecordSource(content),
+ noTimestampInference, Mockito.mock(ComponentLog.class));
final RecordSchema schema = accessStrategy.getSchema(null, in, null);
}
@@ -112,9 +113,9 @@ public class TestInferJsonSchemaAccessStrategy {
}
@Test
- public void testInferenceIncludesAllRecords() throws IOException {
+ void testInferenceIncludesAllRecords() throws IOException {
final File file = new File("src/test/resources/json/prov-events.json");
- final RecordSchema schema = inferSchema(file);
+ final RecordSchema schema = inferSchema(file, StartingFieldStrategy.ROOT_NODE, null);
final RecordField extraField1 = schema.getField("extra field 1").get();
assertSame(RecordFieldType.STRING, extraField1.getDataType().getFieldType());
@@ -127,7 +128,7 @@ public class TestInferJsonSchemaAccessStrategy {
assertSame(RecordFieldType.RECORD, updatedAttributesDataType.getFieldType());
final List<String> expectedAttributeNames = Arrays.asList("path", "filename", "drop reason", "uuid", "reporting.task.type", "s2s.address", "schema.cache.identifier", "reporting.task.uuid",
- "record.count", "s2s.host", "reporting.task.transaction.id", "reporting.task.name", "mime.type");
+ "record.count", "s2s.host", "reporting.task.transaction.id", "reporting.task.name", "mime.type");
final RecordSchema updatedAttributesSchema = ((RecordDataType) updatedAttributesDataType).getChildSchema();
assertEquals(expectedAttributeNames.size(), updatedAttributesSchema.getFieldCount());
@@ -138,9 +139,9 @@ public class TestInferJsonSchemaAccessStrategy {
}
@Test
- public void testDateAndTimestampsInferred() throws IOException {
+ void testDateAndTimestampsInferred() throws IOException {
final File file = new File("src/test/resources/json/prov-events.json");
- final RecordSchema schema = inferSchema(file);
+ final RecordSchema schema = inferSchema(file, StartingFieldStrategy.ROOT_NODE, null);
final RecordField timestampField = schema.getField("timestamp").get();
assertEquals(RecordFieldType.TIMESTAMP.getDataType("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"), timestampField.getDataType());
@@ -167,9 +168,9 @@ public class TestInferJsonSchemaAccessStrategy {
* Test is intended to ensure that all inference rules that are explained in the readers' additionalDetails.html are correct
*/
@Test
- public void testDocsExample() throws IOException {
+ void testDocsExample() throws IOException {
final File file = new File("src/test/resources/json/docs-example.json");
- final RecordSchema schema = inferSchema(file);
+ final RecordSchema schema = inferSchema(file, StartingFieldStrategy.ROOT_NODE, null);
assertSame(RecordFieldType.STRING, schema.getDataType("name").get().getFieldType());
assertSame(RecordFieldType.CHOICE, schema.getDataType("age").get().getFieldType());
@@ -189,14 +190,70 @@ public class TestInferJsonSchemaAccessStrategy {
assertSame(RecordFieldType.STRING, schema.getDataType("nullValue").get().getFieldType());
}
- private RecordSchema inferSchema(final File file) throws IOException {
+ @ParameterizedTest(name = "{index} {2}")
+ @MethodSource("startingFieldNameArgumentProvider")
+ void testInferenceStartsFromArray(final String jsonPath, final StartingFieldStrategy strategy, final String startingFieldName, String testName) throws IOException {
+ final File file = new File(jsonPath);
+ final RecordSchema schema = inferSchema(file, strategy, startingFieldName);
+
+ assertEquals(2, schema.getFieldCount());
+
+ final RecordField field1 = schema.getField("id").get();
+ assertSame(RecordFieldType.INT, field1.getDataType().getFieldType());
+
+ final RecordField field2 = schema.getField("balance").get();
+ assertSame(RecordFieldType.DOUBLE, field2.getDataType().getFieldType());
+ }
+
+ @Test
+ void testInferenceStartsFromSimpleFieldAndNoNestedObjectOrArrayFound() throws IOException {
+ final File file = new File("src/test/resources/json/single-element-nested-array-middle.json");
+ final RecordSchema schema = inferSchema(file, StartingFieldStrategy.NESTED_FIELD, "name");
+
+ assertEquals(0, schema.getFieldCount());
+ }
+
+ @Test
+ void testInferenceStartFromNonExistentField() throws IOException {
+ final File file = new File("src/test/resources/json/single-element-nested-array.json");
+ final RecordSchema recordSchema = inferSchema(file, StartingFieldStrategy.NESTED_FIELD, "notfound");
+ assertEquals(0, recordSchema.getFieldCount());
+ }
+
+ @Test
+ void testInferenceStartFromMultipleNestedField() throws IOException {
+ final File file = new File("src/test/resources/json/multiple-nested-field.json");
+ final RecordSchema schema = inferSchema(file, StartingFieldStrategy.NESTED_FIELD, "accountIds");
+
+ final RecordField field1 = schema.getField("id").get();
+ assertSame(RecordFieldType.STRING, field1.getDataType().getFieldType());
+
+ final RecordField field2 = schema.getField("type").get();
+ assertSame(RecordFieldType.STRING, field2.getDataType().getFieldType());
+ }
+
+ private RecordSchema inferSchema(final File file, final StartingFieldStrategy strategy, final String startingFieldName) throws IOException {
try (final InputStream in = new FileInputStream(file);
final InputStream bufferedIn = new BufferedInputStream(in)) {
- final InferSchemaAccessStrategy<?> accessStrategy = new InferSchemaAccessStrategy<>((var,content) -> new JsonRecordSource(content),
- timestampInference, Mockito.mock(ComponentLog.class));
+ final InferSchemaAccessStrategy<?> accessStrategy = new InferSchemaAccessStrategy<>(
+ (var, content) -> new JsonRecordSource(content, strategy, startingFieldName),
+ timestampInference, Mockito.mock(ComponentLog.class)
+ );
return accessStrategy.getSchema(null, bufferedIn, null);
}
}
+
+ private static Stream<Arguments> startingFieldNameArgumentProvider() {
+ final StartingFieldStrategy strategy = StartingFieldStrategy.NESTED_FIELD;
+ return Stream.of(
+ Arguments.of("src/test/resources/json/single-element-nested-array.json", strategy, "accounts", "testInferenceSkipsToNestedArray"),
+ Arguments.of("src/test/resources/json/single-element-nested.json", strategy, "account", "testInferenceSkipsToNestedObject"),
+ Arguments.of("src/test/resources/json/single-element-nested-array.json", strategy, "name", "testInferenceSkipsToSimpleFieldFindsNextNestedArray"),
+ Arguments.of("src/test/resources/json/single-element-nested.json", strategy, "name", "testInferenceSkipsToSimpleFieldFindsNextNestedObject"),
+ Arguments.of("src/test/resources/json/single-element-nested-array-middle.json", strategy, "accounts", "testInferenceSkipsToNestedArrayInMiddle"),
+ Arguments.of("src/test/resources/json/nested-array-then-start-object.json", strategy, "accounts", "testInferenceSkipsToNestedThenStartObject")
+ );
+ }
}
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/json/TestJsonPathRowRecordReader.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/json/TestJsonPathRowRecordReader.java
index 904e7cd8d0..ed29e5cc5b 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/json/TestJsonPathRowRecordReader.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/json/TestJsonPathRowRecordReader.java
@@ -47,7 +47,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
-public class TestJsonPathRowRecordReader {
+class TestJsonPathRowRecordReader {
private final String dateFormat = RecordFieldType.DATE.getDefaultFormat();
private final String timeFormat = RecordFieldType.TIME.getDefaultFormat();
private final String timestampFormat = RecordFieldType.TIMESTAMP.getDefaultFormat();
@@ -87,25 +87,24 @@ public class TestJsonPathRowRecordReader {
accountFields.add(new RecordField("id", RecordFieldType.INT.getDataType()));
accountFields.add(new RecordField("balance", RecordFieldType.DOUBLE.getDataType()));
- final RecordSchema accountSchema = new SimpleRecordSchema(accountFields);
- return accountSchema;
+ return new SimpleRecordSchema(accountFields);
}
@Test
- public void testReadArray() throws IOException, MalformedRecordException {
+ void testReadArray() throws IOException, MalformedRecordException {
final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-array.json"));
final JsonPathRowRecordReader reader = new JsonPathRowRecordReader(allJsonPaths, schema, in, Mockito.mock(ComponentLog.class), dateFormat, timeFormat, timestampFormat)) {
final List<String> fieldNames = schema.getFieldNames();
- final List<String> expectedFieldNames = Arrays.asList(new String[] {"id", "name", "balance", "address", "city", "state", "zipCode", "country"});
+ final List<String> expectedFieldNames = Arrays.asList("id", "name", "balance", "address", "city", "state", "zipCode", "country");
assertEquals(expectedFieldNames, fieldNames);
- final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
- final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING,
- RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING});
+ final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(DataType::getFieldType).collect(Collectors.toList());
+ final List<RecordFieldType> expectedTypes = Arrays.asList(RecordFieldType.INT, RecordFieldType.STRING,
+ RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING);
assertEquals(expectedTypes, dataTypes);
final Object[] firstRecordValues = reader.nextRecord().getValues();
@@ -119,19 +118,19 @@ public class TestJsonPathRowRecordReader {
}
@Test
- public void testReadOneLine() throws IOException, MalformedRecordException {
+ void testReadOneLine() throws IOException, MalformedRecordException {
final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-oneline.json"));
final JsonPathRowRecordReader reader = new JsonPathRowRecordReader(allJsonPaths, schema, in, Mockito.mock(ComponentLog.class), dateFormat, timeFormat, timestampFormat)) {
final List<String> fieldNames = schema.getFieldNames();
- final List<String> expectedFieldNames = Arrays.asList(new String[] {"id", "name", "balance", "address", "city", "state", "zipCode", "country"});
+ final List<String> expectedFieldNames = Arrays.asList("id", "name", "balance", "address", "city", "state", "zipCode", "country");
assertEquals(expectedFieldNames, fieldNames);
- final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
- final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING,
- RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING});
+ final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(DataType::getFieldType).collect(Collectors.toList());
+ final List<RecordFieldType> expectedTypes = Arrays.asList(RecordFieldType.INT, RecordFieldType.STRING,
+ RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING);
assertEquals(expectedTypes, dataTypes);
final Object[] firstRecordValues = reader.nextRecord().getValues();
@@ -145,19 +144,19 @@ public class TestJsonPathRowRecordReader {
}
@Test
- public void testSingleJsonElement() throws IOException, MalformedRecordException {
+ void testSingleJsonElement() throws IOException, MalformedRecordException {
final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
try (final InputStream in = new FileInputStream(new File("src/test/resources/json/single-bank-account.json"));
final JsonPathRowRecordReader reader = new JsonPathRowRecordReader(allJsonPaths, schema, in, Mockito.mock(ComponentLog.class), dateFormat, timeFormat, timestampFormat)) {
final List<String> fieldNames = schema.getFieldNames();
- final List<String> expectedFieldNames = Arrays.asList(new String[] {"id", "name", "balance", "address", "city", "state", "zipCode", "country"});
+ final List<String> expectedFieldNames = Arrays.asList("id", "name", "balance", "address", "city", "state", "zipCode", "country");
assertEquals(expectedFieldNames, fieldNames);
- final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
- final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING,
- RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING});
+ final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(DataType::getFieldType).collect(Collectors.toList());
+ final List<RecordFieldType> expectedTypes = Arrays.asList(RecordFieldType.INT, RecordFieldType.STRING,
+ RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING);
assertEquals(expectedTypes, dataTypes);
final Object[] firstRecordValues = reader.nextRecord().getValues();
@@ -168,7 +167,7 @@ public class TestJsonPathRowRecordReader {
}
@Test
- public void testTimestampCoercedFromString() throws IOException, MalformedRecordException {
+ void testTimestampCoercedFromString() throws IOException, MalformedRecordException {
final List<RecordField> recordFields = Collections.singletonList(new RecordField("timestamp", RecordFieldType.TIMESTAMP.getDataType()));
final RecordSchema schema = new SimpleRecordSchema(recordFields);
@@ -195,7 +194,7 @@ public class TestJsonPathRowRecordReader {
@Test
- public void testElementWithNestedData() throws IOException, MalformedRecordException {
+ void testElementWithNestedData() throws IOException, MalformedRecordException {
final LinkedHashMap<String, JsonPath> jsonPaths = new LinkedHashMap<>(allJsonPaths);
jsonPaths.put("account", JsonPath.compile("$.account"));
@@ -208,12 +207,12 @@ public class TestJsonPathRowRecordReader {
final JsonPathRowRecordReader reader = new JsonPathRowRecordReader(jsonPaths, schema, in, Mockito.mock(ComponentLog.class), dateFormat, timeFormat, timestampFormat)) {
final List<String> fieldNames = schema.getFieldNames();
- final List<String> expectedFieldNames = Arrays.asList(new String[] {"id", "name", "balance", "address", "city", "state", "zipCode", "country", "account"});
+ final List<String> expectedFieldNames = Arrays.asList("id", "name", "balance", "address", "city", "state", "zipCode", "country", "account");
assertEquals(expectedFieldNames, fieldNames);
- final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
- final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING, RecordFieldType.DOUBLE,
- RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.RECORD});
+ final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(DataType::getFieldType).collect(Collectors.toList());
+ final List<RecordFieldType> expectedTypes = Arrays.asList(RecordFieldType.INT, RecordFieldType.STRING, RecordFieldType.DOUBLE,
+ RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.RECORD);
assertEquals(expectedTypes, dataTypes);
final Object[] firstRecordValues = reader.nextRecord().getValues();
@@ -231,7 +230,7 @@ public class TestJsonPathRowRecordReader {
}
@Test
- public void testElementWithNestedArray() throws IOException, MalformedRecordException {
+ void testElementWithNestedArray() throws IOException, MalformedRecordException {
final LinkedHashMap<String, JsonPath> jsonPaths = new LinkedHashMap<>(allJsonPaths);
jsonPaths.put("accounts", JsonPath.compile("$.accounts"));
@@ -246,13 +245,12 @@ public class TestJsonPathRowRecordReader {
final JsonPathRowRecordReader reader = new JsonPathRowRecordReader(jsonPaths, schema, in, Mockito.mock(ComponentLog.class), dateFormat, timeFormat, timestampFormat)) {
final List<String> fieldNames = schema.getFieldNames();
- final List<String> expectedFieldNames = Arrays.asList(new String[] {
- "id", "name", "balance", "address", "city", "state", "zipCode", "country", "accounts"});
+ final List<String> expectedFieldNames = Arrays.asList("id", "name", "balance", "address", "city", "state", "zipCode", "country", "accounts");
assertEquals(expectedFieldNames, fieldNames);
final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
- final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING, RecordFieldType.DOUBLE,
- RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.ARRAY});
+ final List<RecordFieldType> expectedTypes = Arrays.asList(RecordFieldType.INT, RecordFieldType.STRING, RecordFieldType.DOUBLE,
+ RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.ARRAY);
assertEquals(expectedTypes, dataTypes);
final Object[] firstRecordValues = reader.nextRecord().getValues();
@@ -282,19 +280,19 @@ public class TestJsonPathRowRecordReader {
}
@Test
- public void testReadArrayDifferentSchemas() throws IOException, MalformedRecordException {
+ void testReadArrayDifferentSchemas() throws IOException, MalformedRecordException {
final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-array-different-schemas.json"));
final JsonPathRowRecordReader reader = new JsonPathRowRecordReader(allJsonPaths, schema, in, Mockito.mock(ComponentLog.class), dateFormat, timeFormat, timestampFormat)) {
final List<String> fieldNames = schema.getFieldNames();
- final List<String> expectedFieldNames = Arrays.asList(new String[] {"id", "name", "balance", "address", "city", "state", "zipCode", "country"});
+ final List<String> expectedFieldNames = Arrays.asList("id", "name", "balance", "address", "city", "state", "zipCode", "country");
assertEquals(expectedFieldNames, fieldNames);
final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
- final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING,
- RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING});
+ final List<RecordFieldType> expectedTypes = Arrays.asList(RecordFieldType.INT, RecordFieldType.STRING,
+ RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING);
assertEquals(expectedTypes, dataTypes);
final Object[] firstRecordValues = reader.nextRecord().getValues();
@@ -311,7 +309,7 @@ public class TestJsonPathRowRecordReader {
}
@Test
- public void testReadArrayDifferentSchemasWithOverride() throws IOException, MalformedRecordException {
+ void testReadArrayDifferentSchemasWithOverride() throws IOException, MalformedRecordException {
final LinkedHashMap<String, JsonPath> jsonPaths = new LinkedHashMap<>(allJsonPaths);
jsonPaths.put("address2", JsonPath.compile("$.address2"));
@@ -324,12 +322,12 @@ public class TestJsonPathRowRecordReader {
final JsonPathRowRecordReader reader = new JsonPathRowRecordReader(jsonPaths, schema, in, Mockito.mock(ComponentLog.class), dateFormat, timeFormat, timestampFormat)) {
final List<String> fieldNames = schema.getFieldNames();
- final List<String> expectedFieldNames = Arrays.asList(new String[] {"id", "name", "balance", "address", "city", "state", "zipCode", "country", "address2"});
+ final List<String> expectedFieldNames = Arrays.asList("id", "name", "balance", "address", "city", "state", "zipCode", "country", "address2");
assertEquals(expectedFieldNames, fieldNames);
final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
- final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING, RecordFieldType.DOUBLE, RecordFieldType.STRING,
- RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING});
+ final List<RecordFieldType> expectedTypes = Arrays.asList(RecordFieldType.INT, RecordFieldType.STRING, RecordFieldType.DOUBLE, RecordFieldType.STRING,
+ RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING);
assertEquals(expectedTypes, dataTypes);
final Object[] firstRecordValues = reader.nextRecord().getValues();
@@ -346,7 +344,7 @@ public class TestJsonPathRowRecordReader {
}
@Test
- public void testPrimitiveTypeArrays() throws IOException, MalformedRecordException {
+ void testPrimitiveTypeArrays() throws IOException, MalformedRecordException {
final LinkedHashMap<String, JsonPath> jsonPaths = new LinkedHashMap<>(allJsonPaths);
jsonPaths.put("accountIds", JsonPath.compile("$.accountIds"));
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/json/TestJsonSchemaInference.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/json/TestJsonSchemaInference.java
index 683e884237..1eb655c117 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/json/TestJsonSchemaInference.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/json/TestJsonSchemaInference.java
@@ -35,12 +35,12 @@ import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertSame;
-public class TestJsonSchemaInference {
+class TestJsonSchemaInference {
private final TimeValueInference timestampInference = new TimeValueInference("yyyy-MM-dd", "HH:mm:ss", "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
@Test
- public void testInferenceIncludesAllRecords() throws IOException {
+ void testInferenceIncludesAllRecords() throws IOException {
final File file = new File("src/test/resources/json/data-types.json");
final RecordSchema schema;
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/json/TestJsonTreeRowRecordReader.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/json/TestJsonTreeRowRecordReader.java
index f65c0752fe..752d1a6676 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/json/TestJsonTreeRowRecordReader.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/json/TestJsonTreeRowRecordReader.java
@@ -50,7 +50,6 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -59,11 +58,12 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
-public class TestJsonTreeRowRecordReader {
+class TestJsonTreeRowRecordReader {
private final String dateFormat = RecordFieldType.DATE.getDefaultFormat();
private final String timeFormat = RecordFieldType.TIME.getDefaultFormat();
private final String timestampFormat = RecordFieldType.TIMESTAMP.getDefaultFormat();
@@ -90,13 +90,12 @@ public class TestJsonTreeRowRecordReader {
accountFields.add(new RecordField("id", RecordFieldType.INT.getDataType()));
accountFields.add(new RecordField("balance", RecordFieldType.DOUBLE.getDataType()));
- final RecordSchema accountSchema = new SimpleRecordSchema(accountFields);
- return accountSchema;
+ return new SimpleRecordSchema(accountFields);
}
@Test
- public void testReadChoiceOfStringOrArrayOfRecords() throws IOException, MalformedRecordException {
+ void testReadChoiceOfStringOrArrayOfRecords() throws IOException, MalformedRecordException {
final File schemaFile = new File("src/test/resources/json/choice-of-string-or-array-record.avsc");
final File jsonFile = new File("src/test/resources/json/choice-of-string-or-array-record.json");
@@ -129,7 +128,7 @@ public class TestJsonTreeRowRecordReader {
@Test
@Disabled("Intended only for manual testing to determine performance before/after modifications")
- public void testPerformanceOnLocalFile() throws IOException, MalformedRecordException {
+ void testPerformanceOnLocalFile() throws IOException, MalformedRecordException {
final RecordSchema schema = new SimpleRecordSchema(Collections.emptyList());
final File file = new File("/devel/nifi/nifi-assembly/target/nifi-1.2.0-SNAPSHOT-bin/nifi-1.2.0-SNAPSHOT/prov/16812193969219289");
@@ -158,7 +157,7 @@ public class TestJsonTreeRowRecordReader {
@Test
@Disabled("Intended only for manual testing to determine performance before/after modifications")
- public void testPerformanceOnIndividualMessages() throws IOException, MalformedRecordException {
+ void testPerformanceOnIndividualMessages() throws IOException, MalformedRecordException {
final RecordSchema schema = new SimpleRecordSchema(Collections.emptyList());
final File file = new File("/devel/nifi/nifi-assembly/target/nifi-1.2.0-SNAPSHOT-bin/nifi-1.2.0-SNAPSHOT/1.prov.json");
@@ -186,12 +185,12 @@ public class TestJsonTreeRowRecordReader {
}
@Test
- public void testChoiceOfRecordTypes() throws IOException, MalformedRecordException {
+ void testChoiceOfRecordTypes() throws IOException, MalformedRecordException {
final Schema avroSchema = new Schema.Parser().parse(new File("src/test/resources/json/record-choice.avsc"));
final RecordSchema recordSchema = AvroTypeUtil.createSchema(avroSchema);
- try (final InputStream in = new FileInputStream(new File("src/test/resources/json/elements-for-record-choice.json"));
- final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, mock(ComponentLog.class), recordSchema, dateFormat, timeFormat, timestampFormat)) {
+ try (final InputStream in = new FileInputStream("src/test/resources/json/elements-for-record-choice.json");
+ final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, mock(ComponentLog.class), recordSchema, dateFormat, timeFormat, timestampFormat)) {
// evaluate first record
final Record firstRecord = reader.nextRecord();
@@ -201,7 +200,7 @@ public class TestJsonTreeRowRecordReader {
assertEquals("1234", firstRecord.getValue("id"));
// record should have a schema that indicates that the 'child' is a CHOICE of 2 different record types
- assertTrue(firstOuterSchema.getDataType("child").get().getFieldType() == RecordFieldType.CHOICE);
+ assertSame(RecordFieldType.CHOICE, firstOuterSchema.getDataType("child").get().getFieldType());
final List<DataType> firstSubTypes = ((ChoiceDataType) firstOuterSchema.getDataType("child").get()).getPossibleSubTypes();
assertEquals(2, firstSubTypes.size());
assertEquals(2L, firstSubTypes.stream().filter(type -> type.getFieldType() == RecordFieldType.RECORD).count());
@@ -212,7 +211,7 @@ public class TestJsonTreeRowRecordReader {
final Record firstChildRecord = (Record) childObject;
final RecordSchema firstChildSchema = firstChildRecord.getSchema();
- assertEquals(Arrays.asList("id"), firstChildSchema.getFieldNames());
+ assertEquals(Collections.singletonList("id"), firstChildSchema.getFieldNames());
// evaluate second record
final Record secondRecord = reader.nextRecord();
@@ -223,7 +222,7 @@ public class TestJsonTreeRowRecordReader {
assertEquals("1234", secondRecord.getValue("id"));
// record should have a schema that indicates that the 'child' is a CHOICE of 2 different record types
- assertTrue(secondOuterSchema.getDataType("child").get().getFieldType() == RecordFieldType.CHOICE);
+ assertSame(RecordFieldType.CHOICE, secondOuterSchema.getDataType("child").get().getFieldType());
final List<DataType> secondSubTypes = ((ChoiceDataType) secondOuterSchema.getDataType("child").get()).getPossibleSubTypes();
assertEquals(2, secondSubTypes.size());
assertEquals(2L, secondSubTypes.stream().filter(type -> type.getFieldType() == RecordFieldType.RECORD).count());
@@ -234,7 +233,7 @@ public class TestJsonTreeRowRecordReader {
final Record secondChildRecord = (Record) secondChildObject;
final RecordSchema secondChildSchema = secondChildRecord.getSchema();
- assertEquals(Arrays.asList("name"), secondChildSchema.getFieldNames());
+ assertEquals(Collections.singletonList("name"), secondChildSchema.getFieldNames());
assertNull(reader.nextRecord());
}
@@ -242,19 +241,19 @@ public class TestJsonTreeRowRecordReader {
}
@Test
- public void testReadArray() throws IOException, MalformedRecordException {
+ void testReadArray() throws IOException, MalformedRecordException {
final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
- try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-array.json"));
- final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
+ try (final InputStream in = new FileInputStream("src/test/resources/json/bank-account-array.json");
+ final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
final List<String> fieldNames = schema.getFieldNames();
- final List<String> expectedFieldNames = Arrays.asList(new String[] {"id", "name", "balance", "address", "city", "state", "zipCode", "country"});
+ final List<String> expectedFieldNames = Arrays.asList("id", "name", "balance", "address", "city", "state", "zipCode", "country");
assertEquals(expectedFieldNames, fieldNames);
- final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
- final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING,
- RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING});
+ final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(DataType::getFieldType).collect(Collectors.toList());
+ final List<RecordFieldType> expectedTypes = Arrays.asList(RecordFieldType.INT, RecordFieldType.STRING,
+ RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING);
assertEquals(expectedTypes, dataTypes);
final Object[] firstRecordValues = reader.nextRecord().getValues();
@@ -268,19 +267,19 @@ public class TestJsonTreeRowRecordReader {
}
@Test
- public void testReadOneLinePerJSON() throws IOException, MalformedRecordException {
+ void testReadOneLinePerJSON() throws IOException, MalformedRecordException {
final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
- try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-oneline.json"));
+ try (final InputStream in = new FileInputStream("src/test/resources/json/bank-account-oneline.json");
final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
final List<String> fieldNames = schema.getFieldNames();
- final List<String> expectedFieldNames = Arrays.asList(new String[] {"id", "name", "balance", "address", "city", "state", "zipCode", "country"});
+ final List<String> expectedFieldNames = Arrays.asList("id", "name", "balance", "address", "city", "state", "zipCode", "country");
assertEquals(expectedFieldNames, fieldNames);
- final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
- final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING,
- RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING});
+ final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(DataType::getFieldType).collect(Collectors.toList());
+ final List<RecordFieldType> expectedTypes = Arrays.asList(RecordFieldType.INT, RecordFieldType.STRING,
+ RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING);
assertEquals(expectedTypes, dataTypes);
final Object[] firstRecordValues = reader.nextRecord().getValues();
@@ -294,20 +293,20 @@ public class TestJsonTreeRowRecordReader {
}
@Test
- public void testReadMultilineJSON() throws IOException, MalformedRecordException {
+ void testReadMultilineJSON() throws IOException, MalformedRecordException {
final List<RecordField> fields = getFields(RecordFieldType.DECIMAL.getDecimalDataType(30, 10));
final RecordSchema schema = new SimpleRecordSchema(fields);
- try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-multiline.json"));
+ try (final InputStream in = new FileInputStream("src/test/resources/json/bank-account-multiline.json");
final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
final List<String> fieldNames = schema.getFieldNames();
- final List<String> expectedFieldNames = Arrays.asList(new String[] {"id", "name", "balance", "address", "city", "state", "zipCode", "country"});
+ final List<String> expectedFieldNames = Arrays.asList("id", "name", "balance", "address", "city", "state", "zipCode", "country");
assertEquals(expectedFieldNames, fieldNames);
- final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
- final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING,
- RecordFieldType.DECIMAL, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING});
+ final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(DataType::getFieldType).collect(Collectors.toList());
+ final List<RecordFieldType> expectedTypes = Arrays.asList(RecordFieldType.INT, RecordFieldType.STRING,
+ RecordFieldType.DECIMAL, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING);
assertEquals(expectedTypes, dataTypes);
final Object[] firstRecordValues = reader.nextRecord().getValues();
@@ -321,19 +320,19 @@ public class TestJsonTreeRowRecordReader {
}
@Test
- public void testReadMultilineArrays() throws IOException, MalformedRecordException {
+ void testReadMultilineArrays() throws IOException, MalformedRecordException {
final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
- try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-multiarray.json"));
+ try (final InputStream in = new FileInputStream("src/test/resources/json/bank-account-multiarray.json");
final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
final List<String> fieldNames = schema.getFieldNames();
- final List<String> expectedFieldNames = Arrays.asList(new String[] {"id", "name", "balance", "address", "city", "state", "zipCode", "country"});
+ final List<String> expectedFieldNames = Arrays.asList("id", "name", "balance", "address", "city", "state", "zipCode", "country");
assertEquals(expectedFieldNames, fieldNames);
- final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
- final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING,
- RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING});
+ final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(DataType::getFieldType).collect(Collectors.toList());
+ final List<RecordFieldType> expectedTypes = Arrays.asList(RecordFieldType.INT, RecordFieldType.STRING,
+ RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING);
assertEquals(expectedTypes, dataTypes);
final Object[] firstRecordValues = reader.nextRecord().getValues();
@@ -353,19 +352,19 @@ public class TestJsonTreeRowRecordReader {
}
@Test
- public void testReadMixedJSON() throws IOException, MalformedRecordException {
+ void testReadMixedJSON() throws IOException, MalformedRecordException {
final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
- try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-mixed.json"));
+ try (final InputStream in = new FileInputStream("src/test/resources/json/bank-account-mixed.json");
final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
final List<String> fieldNames = schema.getFieldNames();
- final List<String> expectedFieldNames = Arrays.asList(new String[] {"id", "name", "balance", "address", "city", "state", "zipCode", "country"});
+ final List<String> expectedFieldNames = Arrays.asList("id", "name", "balance", "address", "city", "state", "zipCode", "country");
assertEquals(expectedFieldNames, fieldNames);
- final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
- final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING,
- RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING});
+ final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(DataType::getFieldType).collect(Collectors.toList());
+ final List<RecordFieldType> expectedTypes = Arrays.asList(RecordFieldType.INT, RecordFieldType.STRING,
+ RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING);
assertEquals(expectedTypes, dataTypes);
final Object[] firstRecordValues = reader.nextRecord().getValues();
@@ -386,14 +385,14 @@ public class TestJsonTreeRowRecordReader {
}
@Test
- public void testReadRawRecordIncludesFieldsNotInSchema() throws IOException, MalformedRecordException {
+ void testReadRawRecordIncludesFieldsNotInSchema() throws IOException, MalformedRecordException {
final List<RecordField> fields = new ArrayList<>();
fields.add(new RecordField("id", RecordFieldType.INT.getDataType()));
fields.add(new RecordField("name", RecordFieldType.STRING.getDataType()));
final RecordSchema schema = new SimpleRecordSchema(fields);
- try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-array.json"));
- final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
+ try (final InputStream in = new FileInputStream("src/test/resources/json/bank-account-array.json");
+ final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
final Record schemaValidatedRecord = reader.nextRecord(true, true);
assertEquals(1, schemaValidatedRecord.getValue("id"));
@@ -401,8 +400,8 @@ public class TestJsonTreeRowRecordReader {
assertNull(schemaValidatedRecord.getValue("balance"));
}
- try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-array.json"));
- final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
+ try (final InputStream in = new FileInputStream("src/test/resources/json/bank-account-array.json");
+ final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
final Record rawRecord = reader.nextRecord(false, false);
assertEquals(1, rawRecord.getValue("id"));
@@ -418,14 +417,14 @@ public class TestJsonTreeRowRecordReader {
@Test
- public void testReadRawRecordTypeCoercion() throws IOException, MalformedRecordException {
+ void testReadRawRecordTypeCoercion() throws IOException, MalformedRecordException {
final List<RecordField> fields = new ArrayList<>();
fields.add(new RecordField("id", RecordFieldType.STRING.getDataType()));
fields.add(new RecordField("name", RecordFieldType.STRING.getDataType()));
final RecordSchema schema = new SimpleRecordSchema(fields);
- try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-array.json"));
- final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
+ try (final InputStream in = new FileInputStream("src/test/resources/json/bank-account-array.json");
+ final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
final Record schemaValidatedRecord = reader.nextRecord(true, true);
assertEquals("1", schemaValidatedRecord.getValue("id")); // will be coerced into a STRING as per the schema
@@ -435,8 +434,8 @@ public class TestJsonTreeRowRecordReader {
assertEquals(2, schemaValidatedRecord.getRawFieldNames().size());
}
- try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-array.json"));
- final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
+ try (final InputStream in = new FileInputStream("src/test/resources/json/bank-account-array.json");
+ final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
final Record rawRecord = reader.nextRecord(false, false);
assertEquals(1, rawRecord.getValue("id")); // will return raw value of (int) 1
@@ -453,7 +452,7 @@ public class TestJsonTreeRowRecordReader {
}
@Test
- public void testDateCoercedFromString() throws IOException, MalformedRecordException {
+ void testDateCoercedFromString() throws IOException, MalformedRecordException {
final String dateField = "date";
final List<RecordField> recordFields = Collections.singletonList(new RecordField(dateField, RecordFieldType.DATE.getDataType()));
final RecordSchema schema = new SimpleRecordSchema(recordFields);
@@ -474,12 +473,12 @@ public class TestJsonTreeRowRecordReader {
}
@Test
- public void testTimestampCoercedFromString() throws IOException, MalformedRecordException {
+ void testTimestampCoercedFromString() throws IOException, MalformedRecordException {
final List<RecordField> recordFields = Collections.singletonList(new RecordField("timestamp", RecordFieldType.TIMESTAMP.getDataType()));
final RecordSchema schema = new SimpleRecordSchema(recordFields);
for (final boolean coerceTypes : new boolean[] {true, false}) {
- try (final InputStream in = new FileInputStream(new File("src/test/resources/json/timestamp.json"));
+ try (final InputStream in = new FileInputStream("src/test/resources/json/timestamp.json");
final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, mock(ComponentLog.class), schema, dateFormat, timeFormat, "yyyy/MM/dd HH:mm:ss")) {
final Record record = reader.nextRecord(coerceTypes, false);
@@ -490,19 +489,19 @@ public class TestJsonTreeRowRecordReader {
}
@Test
- public void testSingleJsonElement() throws IOException, MalformedRecordException {
+ void testSingleJsonElement() throws IOException, MalformedRecordException {
final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
- try (final InputStream in = new FileInputStream(new File("src/test/resources/json/single-bank-account.json"));
- final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
+ try (final InputStream in = new FileInputStream("src/test/resources/json/single-bank-account.json");
+ final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
final List<String> fieldNames = schema.getFieldNames();
- final List<String> expectedFieldNames = Arrays.asList(new String[] {"id", "name", "balance", "address", "city", "state", "zipCode", "country"});
+ final List<String> expectedFieldNames = Arrays.asList("id", "name", "balance", "address", "city", "state", "zipCode", "country");
assertEquals(expectedFieldNames, fieldNames);
- final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
- final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING,
- RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING});
+ final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(DataType::getFieldType).collect(Collectors.toList());
+ final List<RecordFieldType> expectedTypes = Arrays.asList(RecordFieldType.INT, RecordFieldType.STRING,
+ RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING);
assertEquals(expectedTypes, dataTypes);
final Object[] firstRecordValues = reader.nextRecord().getValues();
@@ -513,21 +512,21 @@ public class TestJsonTreeRowRecordReader {
}
@Test
- public void testSingleJsonElementWithChoiceFields() throws IOException, MalformedRecordException {
+ void testSingleJsonElementWithChoiceFields() throws IOException, MalformedRecordException {
// Wraps default fields by Choice data type to test mapping to a Choice type.
final List<RecordField> choiceFields = getDefaultFields().stream()
.map(f -> new RecordField(f.getFieldName(), RecordFieldType.CHOICE.getChoiceDataType(f.getDataType()))).collect(Collectors.toList());
final RecordSchema schema = new SimpleRecordSchema(choiceFields);
- try (final InputStream in = new FileInputStream(new File("src/test/resources/json/single-bank-account.json"));
+ try (final InputStream in = new FileInputStream("src/test/resources/json/single-bank-account.json");
final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
final List<String> fieldNames = schema.getFieldNames();
- final List<String> expectedFieldNames = Arrays.asList(new String[] {"id", "name", "balance", "address", "city", "state", "zipCode", "country"});
+ final List<String> expectedFieldNames = Arrays.asList("id", "name", "balance", "address", "city", "state", "zipCode", "country");
assertEquals(expectedFieldNames, fieldNames);
- final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING,
- RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING});
+ final List<RecordFieldType> expectedTypes = Arrays.asList(RecordFieldType.INT, RecordFieldType.STRING,
+ RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING);
final List<RecordField> fields = schema.getFields();
for (int i = 0; i < schema.getFields().size(); i++) {
assertTrue(fields.get(i).getDataType() instanceof ChoiceDataType);
@@ -543,19 +542,19 @@ public class TestJsonTreeRowRecordReader {
}
@Test
- public void testElementWithNestedData() throws IOException, MalformedRecordException {
+ void testElementWithNestedData() throws IOException, MalformedRecordException {
final DataType accountType = RecordFieldType.RECORD.getRecordDataType(getAccountSchema());
final List<RecordField> fields = getDefaultFields();
fields.add(new RecordField("account", accountType));
fields.remove(new RecordField("balance", RecordFieldType.DOUBLE.getDataType()));
final RecordSchema schema = new SimpleRecordSchema(fields);
- try (final InputStream in = new FileInputStream(new File("src/test/resources/json/single-element-nested.json"));
- final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
+ try (final InputStream in = new FileInputStream("src/test/resources/json/single-element-nested.json");
+ final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
- final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
- final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING,
- RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.RECORD});
+ final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(DataType::getFieldType).collect(Collectors.toList());
+ final List<RecordFieldType> expectedTypes = Arrays.asList(RecordFieldType.INT, RecordFieldType.STRING,
+ RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.RECORD);
assertEquals(expectedTypes, dataTypes);
final Object[] firstRecordValues = reader.nextRecord().getValues();
@@ -573,7 +572,7 @@ public class TestJsonTreeRowRecordReader {
}
@Test
- public void testElementWithNestedArray() throws IOException, MalformedRecordException {
+ void testElementWithNestedArray() throws IOException, MalformedRecordException {
final DataType accountRecordType = RecordFieldType.RECORD.getRecordDataType(getAccountSchema());
final DataType accountsType = RecordFieldType.ARRAY.getArrayDataType(accountRecordType);
@@ -582,17 +581,16 @@ public class TestJsonTreeRowRecordReader {
fields.remove(new RecordField("balance", RecordFieldType.DOUBLE.getDataType()));
final RecordSchema schema = new SimpleRecordSchema(fields);
- try (final InputStream in = new FileInputStream(new File("src/test/resources/json/single-element-nested-array.json"));
- final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
+ try (final InputStream in = new FileInputStream("src/test/resources/json/single-element-nested-array.json");
+ final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
final List<String> fieldNames = schema.getFieldNames();
- final List<String> expectedFieldNames = Arrays.asList(new String[] {
- "id", "name", "address", "city", "state", "zipCode", "country", "accounts"});
+ final List<String> expectedFieldNames = Arrays.asList("id", "name", "address", "city", "state", "zipCode", "country", "accounts");
assertEquals(expectedFieldNames, fieldNames);
- final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
- final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING,
- RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.ARRAY});
+ final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(DataType::getFieldType).collect(Collectors.toList());
+ final List<RecordFieldType> expectedTypes = Arrays.asList(RecordFieldType.INT, RecordFieldType.STRING,
+ RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.ARRAY);
assertEquals(expectedTypes, dataTypes);
final Object[] firstRecordValues = reader.nextRecord().getValues();
@@ -607,19 +605,19 @@ public class TestJsonTreeRowRecordReader {
}
@Test
- public void testReadArrayDifferentSchemas() throws IOException, MalformedRecordException {
+ void testReadArrayDifferentSchemas() throws IOException, MalformedRecordException {
final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
- try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-array-different-schemas.json"));
- final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
+ try (final InputStream in = new FileInputStream("src/test/resources/json/bank-account-array-different-schemas.json");
+ final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
final List<String> fieldNames = schema.getFieldNames();
- final List<String> expectedFieldNames = Arrays.asList(new String[] {"id", "name", "balance", "address", "city", "state", "zipCode", "country"});
+ final List<String> expectedFieldNames = Arrays.asList("id", "name", "balance", "address", "city", "state", "zipCode", "country");
assertEquals(expectedFieldNames, fieldNames);
- final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
- final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING,
- RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING});
+ final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(DataType::getFieldType).collect(Collectors.toList());
+ final List<RecordFieldType> expectedTypes = Arrays.asList(RecordFieldType.INT, RecordFieldType.STRING,
+ RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING);
assertEquals(expectedTypes, dataTypes);
final Object[] firstRecordValues = reader.nextRecord().getValues();
@@ -636,53 +634,19 @@ public class TestJsonTreeRowRecordReader {
}
@Test
- public void testReadArrayDifferentSchemasWithOverride() throws IOException, MalformedRecordException {
- final Map<String, DataType> overrides = new HashMap<>();
- overrides.put("address2", RecordFieldType.STRING.getDataType());
-
- final List<RecordField> fields = getDefaultFields();
- fields.add(new RecordField("address2", RecordFieldType.STRING.getDataType()));
- final RecordSchema schema = new SimpleRecordSchema(fields);
-
- try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-array-different-schemas.json"));
- final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
-
- final List<String> fieldNames = schema.getFieldNames();
- final List<String> expectedFieldNames = Arrays.asList(new String[] {"id", "name", "balance", "address", "city", "state", "zipCode", "country", "address2"});
- assertEquals(expectedFieldNames, fieldNames);
-
- final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
- final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING, RecordFieldType.DOUBLE, RecordFieldType.STRING,
- RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING});
- assertEquals(expectedTypes, dataTypes);
-
- final Object[] firstRecordValues = reader.nextRecord().getValues();
- assertArrayEquals(new Object[] {1, "John Doe", 4750.89, "123 My Street", "My City", "MS", "11111", "USA", null}, firstRecordValues);
-
- final Object[] secondRecordValues = reader.nextRecord().getValues();
- assertArrayEquals(new Object[] {2, "Jane Doe", 4820.09, "321 Your Street", "Your City", "NY", "33333", null, null}, secondRecordValues);
-
- final Object[] thirdRecordValues = reader.nextRecord().getValues();
- assertArrayEquals(new Object[] {3, "Jake Doe", 4751.89, "124 My Street", "My City", "MS", "11111", "USA", "Apt. #12"}, thirdRecordValues);
-
- assertNull(reader.nextRecord());
- }
- }
-
- @Test
- public void testReadArrayDifferentSchemasWithOptionalElementOverridden() throws IOException, MalformedRecordException {
+ void testReadArrayDifferentSchemasWithOptionalElementOverridden() throws IOException, MalformedRecordException {
final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
- try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-array-optional-balance.json"));
- final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
+ try (final InputStream in = new FileInputStream("src/test/resources/json/bank-account-array-optional-balance.json");
+ final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
final List<String> fieldNames = schema.getFieldNames();
- final List<String> expectedFieldNames = Arrays.asList(new String[] {"id", "name", "balance", "address", "city", "state", "zipCode", "country"});
+ final List<String> expectedFieldNames = Arrays.asList("id", "name", "balance", "address", "city", "state", "zipCode", "country");
assertEquals(expectedFieldNames, fieldNames);
- final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
- final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING, RecordFieldType.DOUBLE, RecordFieldType.STRING,
- RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING});
+ final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(DataType::getFieldType).collect(Collectors.toList());
+ final List<RecordFieldType> expectedTypes = Arrays.asList(RecordFieldType.INT, RecordFieldType.STRING, RecordFieldType.DOUBLE, RecordFieldType.STRING,
+ RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING);
assertEquals(expectedTypes, dataTypes);
final Object[] firstRecordValues = reader.nextRecord().getValues();
@@ -700,7 +664,7 @@ public class TestJsonTreeRowRecordReader {
@Test
- public void testReadUnicodeCharacters() throws IOException, MalformedRecordException {
+ void testReadUnicodeCharacters() throws IOException, MalformedRecordException {
final List<RecordField> fromFields = new ArrayList<>();
fromFields.add(new RecordField("id", RecordFieldType.LONG.getDataType()));
@@ -715,8 +679,8 @@ public class TestJsonTreeRowRecordReader {
fields.add(new RecordField("from", fromType));
final RecordSchema schema = new SimpleRecordSchema(fields);
- try (final InputStream in = new FileInputStream(new File("src/test/resources/json/json-with-unicode.json"));
- final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
+ try (final InputStream in = new FileInputStream("src/test/resources/json/json-with-unicode.json");
+ final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
final Object[] firstRecordValues = reader.nextRecord().getValues();
@@ -732,7 +696,7 @@ public class TestJsonTreeRowRecordReader {
}
@Test
- public void testIncorrectSchema() {
+ void testIncorrectSchema() {
final DataType accountType = RecordFieldType.RECORD.getRecordDataType(getAccountSchema());
final List<RecordField> fields = getDefaultFields();
fields.add(new RecordField("account", accountType));
@@ -756,7 +720,7 @@ public class TestJsonTreeRowRecordReader {
}
@Test
- public void testMergeOfSimilarRecords() throws Exception {
+ void testMergeOfSimilarRecords() throws Exception {
// GIVEN
String jsonPath = "src/test/resources/json/similar-records.json";
@@ -789,7 +753,7 @@ public class TestJsonTreeRowRecordReader {
}
@Test
- public void testChoiceOfEmbeddedSimilarRecords() throws Exception {
+ void testChoiceOfEmbeddedSimilarRecords() throws Exception {
// GIVEN
String jsonPath = "src/test/resources/json/choice-of-embedded-similar-records.json";
@@ -801,11 +765,11 @@ public class TestJsonTreeRowRecordReader {
new RecordField("integer", RecordFieldType.INT.getDataType()),
new RecordField("string", RecordFieldType.STRING.getDataType())
));
- RecordSchema expectedRecordChoiceSchema = new SimpleRecordSchema(Arrays.asList(
- new RecordField("record", RecordFieldType.CHOICE.getChoiceDataType(
- RecordFieldType.RECORD.getRecordDataType(expectedRecordSchema1),
- RecordFieldType.RECORD.getRecordDataType(expectedRecordSchema2)
- ))
+ RecordSchema expectedRecordChoiceSchema = new SimpleRecordSchema(Collections.singletonList(
+ new RecordField("record", RecordFieldType.CHOICE.getChoiceDataType(
+ RecordFieldType.RECORD.getRecordDataType(expectedRecordSchema1),
+ RecordFieldType.RECORD.getRecordDataType(expectedRecordSchema2)
+ ))
));
List<Object> expected = Arrays.asList(
@@ -829,12 +793,12 @@ public class TestJsonTreeRowRecordReader {
}
@Test
- public void testChoiceOfEmbeddedArraysAndSingleRecords() throws Exception {
+ void testChoiceOfEmbeddedArraysAndSingleRecords() throws Exception {
// GIVEN
String jsonPath = "src/test/resources/json/choice-of-embedded-arrays-and-single-records.json";
- SimpleRecordSchema expectedRecordSchema1 = new SimpleRecordSchema(Arrays.asList(
- new RecordField("integer", RecordFieldType.INT.getDataType())
+ SimpleRecordSchema expectedRecordSchema1 = new SimpleRecordSchema(Collections.singletonList(
+ new RecordField("integer", RecordFieldType.INT.getDataType())
));
SimpleRecordSchema expectedRecordSchema2 = new SimpleRecordSchema(Arrays.asList(
new RecordField("integer", RecordFieldType.INT.getDataType()),
@@ -848,13 +812,13 @@ public class TestJsonTreeRowRecordReader {
new RecordField("integer", RecordFieldType.INT.getDataType()),
new RecordField("string", RecordFieldType.STRING.getDataType())
));
- RecordSchema expectedRecordChoiceSchema = new SimpleRecordSchema(Arrays.asList(
- new RecordField("record", RecordFieldType.CHOICE.getChoiceDataType(
- RecordFieldType.RECORD.getRecordDataType(expectedRecordSchema1),
- RecordFieldType.RECORD.getRecordDataType(expectedRecordSchema3),
- RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.RECORD.getRecordDataType(expectedRecordSchema2)),
- RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.RECORD.getRecordDataType(expectedRecordSchema4))
- ))
+ RecordSchema expectedRecordChoiceSchema = new SimpleRecordSchema(Collections.singletonList(
+ new RecordField("record", RecordFieldType.CHOICE.getChoiceDataType(
+ RecordFieldType.RECORD.getRecordDataType(expectedRecordSchema1),
+ RecordFieldType.RECORD.getRecordDataType(expectedRecordSchema3),
+ RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.RECORD.getRecordDataType(expectedRecordSchema2)),
+ RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.RECORD.getRecordDataType(expectedRecordSchema4))
+ ))
));
List<Object> expected = Arrays.asList(
@@ -901,7 +865,7 @@ public class TestJsonTreeRowRecordReader {
}
@Test
- public void testChoiceOfMergedEmbeddedArraysAndSingleRecords() throws Exception {
+ void testChoiceOfMergedEmbeddedArraysAndSingleRecords() throws Exception {
// GIVEN
String jsonPath = "src/test/resources/json/choice-of-merged-embedded-arrays-and-single-records.json";
@@ -922,13 +886,13 @@ public class TestJsonTreeRowRecordReader {
new RecordField("string", RecordFieldType.STRING.getDataType()),
new RecordField("boolean", RecordFieldType.BOOLEAN.getDataType())
));
- RecordSchema expectedRecordChoiceSchema = new SimpleRecordSchema(Arrays.asList(
- new RecordField("record", RecordFieldType.CHOICE.getChoiceDataType(
- RecordFieldType.RECORD.getRecordDataType(expectedRecordSchema1),
- RecordFieldType.RECORD.getRecordDataType(expectedRecordSchema3),
- RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.RECORD.getRecordDataType(expectedRecordSchema2)),
- RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.RECORD.getRecordDataType(expectedRecordSchema4))
- ))
+ RecordSchema expectedRecordChoiceSchema = new SimpleRecordSchema(Collections.singletonList(
+ new RecordField("record", RecordFieldType.CHOICE.getChoiceDataType(
+ RecordFieldType.RECORD.getRecordDataType(expectedRecordSchema1),
+ RecordFieldType.RECORD.getRecordDataType(expectedRecordSchema3),
+ RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.RECORD.getRecordDataType(expectedRecordSchema2)),
+ RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.RECORD.getRecordDataType(expectedRecordSchema4))
+ ))
));
List<Object> expected = Arrays.asList(
@@ -980,7 +944,7 @@ public class TestJsonTreeRowRecordReader {
}
@Test
- public void testChoseSuboptimalSchemaWhenDataHasExtraFields() throws Exception {
+ void testChoseSuboptimalSchemaWhenDataHasExtraFields() throws Exception {
// GIVEN
String jsonPath = "src/test/resources/json/choice-of-different-arrays-with-extra-fields.json";
@@ -993,18 +957,18 @@ public class TestJsonTreeRowRecordReader {
new RecordField("string", RecordFieldType.STRING.getDataType())
));
- RecordSchema recordChoiceSchema = new SimpleRecordSchema(Arrays.asList(
- new RecordField("record", RecordFieldType.CHOICE.getChoiceDataType(
- RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.RECORD.getRecordDataType(recordSchema1)),
- RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.RECORD.getRecordDataType(recordSchema2))
- ))
+ RecordSchema recordChoiceSchema = new SimpleRecordSchema(Collections.singletonList(
+ new RecordField("record", RecordFieldType.CHOICE.getChoiceDataType(
+ RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.RECORD.getRecordDataType(recordSchema1)),
+ RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.RECORD.getRecordDataType(recordSchema2))
+ ))
));
- RecordSchema schema = new SimpleRecordSchema(Arrays.asList(
- new RecordField("dataCollection", RecordFieldType.ARRAY.getArrayDataType(
- RecordFieldType.RECORD.getRecordDataType(recordChoiceSchema)
- )
- )));
+ RecordSchema schema = new SimpleRecordSchema(Collections.singletonList(
+ new RecordField("dataCollection", RecordFieldType.ARRAY.getArrayDataType(
+ RecordFieldType.RECORD.getRecordDataType(recordChoiceSchema)
+ )
+ )));
SimpleRecordSchema expectedChildSchema1 = new SimpleRecordSchema(Arrays.asList(
new RecordField("integer", RecordFieldType.INT.getDataType()),
@@ -1014,11 +978,11 @@ public class TestJsonTreeRowRecordReader {
new RecordField("integer", RecordFieldType.INT.getDataType()),
new RecordField("string", RecordFieldType.STRING.getDataType())
));
- RecordSchema expectedRecordChoiceSchema = new SimpleRecordSchema(Arrays.asList(
- new RecordField("record", RecordFieldType.CHOICE.getChoiceDataType(
- RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.RECORD.getRecordDataType(expectedChildSchema1)),
- RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.RECORD.getRecordDataType(expectedChildSchema2))
- ))
+ RecordSchema expectedRecordChoiceSchema = new SimpleRecordSchema(Collections.singletonList(
+ new RecordField("record", RecordFieldType.CHOICE.getChoiceDataType(
+ RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.RECORD.getRecordDataType(expectedChildSchema1)),
+ RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.RECORD.getRecordDataType(expectedChildSchema2))
+ ))
));
// Since the actual arrays have records with either (INT, BOOLEAN, STRING) or (INT, STRING, STRING)
@@ -1062,14 +1026,119 @@ public class TestJsonTreeRowRecordReader {
testReadRecords(jsonPath, schema, expected);
}
+ @Test
+ void testStartFromNestedArray() throws IOException, MalformedRecordException {
+ String jsonPath = "src/test/resources/json/single-element-nested-array.json";
+
+ SimpleRecordSchema expectedRecordSchema = new SimpleRecordSchema(Arrays.asList(
+ new RecordField("id", RecordFieldType.INT.getDataType()),
+ new RecordField("balance", RecordFieldType.DOUBLE.getDataType())
+ ));
+
+ List<Object> expected = Arrays.asList(
+ new MapRecord(expectedRecordSchema, new HashMap<String, Object>(){{
+ put("id", 42);
+ put("balance", 4750.89);
+ }}),
+ new MapRecord(expectedRecordSchema, new HashMap<String, Object>(){{
+ put("id", 43);
+ put("balance", 48212.38);
+ }})
+ );
+
+ testReadRecords(jsonPath, expected, StartingFieldStrategy.NESTED_FIELD, "accounts");
+ }
+
+ @Test
+ void testStartsFromNestedObject() throws IOException, MalformedRecordException {
+ String jsonPath = "src/test/resources/json/single-element-nested.json";
+
+ SimpleRecordSchema expectedRecordSchema = new SimpleRecordSchema(Arrays.asList(
+ new RecordField("id", RecordFieldType.INT.getDataType()),
+ new RecordField("balance", RecordFieldType.DOUBLE.getDataType())
+ ));
+
+ List<Object> expected = Collections.singletonList(
+ new MapRecord(expectedRecordSchema, new HashMap<String, Object>() {{
+ put("id", 42);
+ put("balance", 4750.89);
+ }})
+ );
+
+ testReadRecords(jsonPath, expected, StartingFieldStrategy.NESTED_FIELD, "account");
+ }
+
+ @Test
+ void testStartsFromMultipleNestedField() throws IOException, MalformedRecordException {
+ String jsonPath = "src/test/resources/json/multiple-nested-field.json";
+
+ SimpleRecordSchema expectedRecordSchema = new SimpleRecordSchema(Arrays.asList(
+ new RecordField("id", RecordFieldType.STRING.getDataType()),
+ new RecordField("type", RecordFieldType.STRING.getDataType())
+ ));
+
+ List<Object> expected = Arrays.asList(
+ new MapRecord(expectedRecordSchema, new HashMap<String, Object>(){{
+ put("id", "n312kj3");
+ put("type", "employee");
+ }}),
+ new MapRecord(expectedRecordSchema, new HashMap<String, Object>(){{
+ put("id", "dl2kdff");
+ put("type", "security");
+ }})
+ );
+
+ testReadRecords(jsonPath, expected, StartingFieldStrategy.NESTED_FIELD, "accountIds");
+ }
+
+ @Test
+ void testStartFromSimpleFieldReturnsEmptyJson() throws IOException, MalformedRecordException {
+ String jsonPath = "src/test/resources/json/single-element-nested.json";
+
+ testReadRecords(jsonPath, Collections.emptyList(), StartingFieldStrategy.NESTED_FIELD, "name");
+ }
+
+ @Test
+ void testStartFromNonExistentFieldWithDefinedSchema() throws IOException, MalformedRecordException {
+ String jsonPath = "src/test/resources/json/single-element-nested.json";
+
+ SimpleRecordSchema expectedRecordSchema = new SimpleRecordSchema(getDefaultFields());
+ List<Object> expected = Collections.emptyList();
+
+ testReadRecords(jsonPath, expectedRecordSchema, expected, StartingFieldStrategy.NESTED_FIELD, "notfound");
+ }
+
+ @Test
+ void testStartFromNestedFieldThenStartObject() throws IOException, MalformedRecordException {
+ String jsonPath = "src/test/resources/json/nested-array-then-start-object.json";
+
+ SimpleRecordSchema expectedRecordSchema = new SimpleRecordSchema(Arrays.asList(
+ new RecordField("id", RecordFieldType.INT.getDataType()),
+ new RecordField("balance", RecordFieldType.DOUBLE.getDataType())
+ ));
+
+ List<Object> expected = Arrays.asList(
+ new MapRecord(expectedRecordSchema, new HashMap<String, Object>(){{
+ put("id", 42);
+ put("balance", 4750.89);
+ }}),
+ new MapRecord(expectedRecordSchema, new HashMap<String, Object>(){{
+ put("id", 43);
+ put("balance", 48212.38);
+ }})
+ );
+
+ testReadRecords(jsonPath, expectedRecordSchema, expected, StartingFieldStrategy.NESTED_FIELD, "accounts");
+ }
+
private void testReadRecords(String jsonPath, List<Object> expected) throws IOException, MalformedRecordException {
// GIVEN
final File jsonFile = new File(jsonPath);
try (
- InputStream jsonStream = new ByteArrayInputStream(FileUtils.readFileToByteArray(jsonFile));
+ InputStream jsonStream = new ByteArrayInputStream(FileUtils.readFileToByteArray(jsonFile))
) {
- RecordSchema schema = inferSchema(jsonStream);
+ RecordSchema schema = inferSchema(jsonStream, StartingFieldStrategy.ROOT_NODE, null);
// WHEN
// THEN
@@ -1077,25 +1146,30 @@ public class TestJsonTreeRowRecordReader {
}
}
- private void testReadRecords(String jsonPath, RecordSchema schema, List<Object> expected) throws IOException, MalformedRecordException {
- // GIVEN
+ private void testReadRecords(String jsonPath, List<Object> expected, StartingFieldStrategy strategy, String startingFieldName) throws IOException, MalformedRecordException {
final File jsonFile = new File(jsonPath);
+ try (InputStream jsonStream = new ByteArrayInputStream(FileUtils.readFileToByteArray(jsonFile))) {
+ RecordSchema schema = inferSchema(jsonStream, strategy, startingFieldName);
+ testReadRecords(jsonStream, schema, expected, strategy, startingFieldName);
+ }
+ }
- try (
- InputStream jsonStream = new ByteArrayInputStream(FileUtils.readFileToByteArray(jsonFile));
- ) {
- // WHEN
- // THEN
+ private void testReadRecords(String jsonPath, RecordSchema schema, List<Object> expected) throws IOException, MalformedRecordException {
+ final File jsonFile = new File(jsonPath);
+ try (InputStream jsonStream = new ByteArrayInputStream(FileUtils.readFileToByteArray(jsonFile))) {
testReadRecords(jsonStream, schema, expected);
}
}
+ private void testReadRecords(String jsonPath, RecordSchema schema, List<Object> expected, StartingFieldStrategy strategy, String startingFieldName) throws IOException, MalformedRecordException {
+ final File jsonFile = new File(jsonPath);
+ try (InputStream jsonStream = new ByteArrayInputStream(FileUtils.readFileToByteArray(jsonFile))) {
+ testReadRecords(jsonStream, schema, expected, strategy, startingFieldName);
+ }
+ }
+
private void testReadRecords(InputStream jsonStream, RecordSchema schema, List<Object> expected) throws IOException, MalformedRecordException {
- // GIVEN
- try (
- JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(jsonStream, mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat);
- ) {
- // WHEN
+ try (JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(jsonStream, mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
List<Object> actual = new ArrayList<>();
Record record;
while ((record = reader.nextRecord()) != null) {
@@ -1103,7 +1177,6 @@ public class TestJsonTreeRowRecordReader {
actual.addAll(dataCollection);
}
- // THEN
List<Function<Object, Object>> propertyProviders = Arrays.asList(
_object -> ((Record)_object).getSchema(),
_object -> Arrays.stream(((Record)_object).getValues()).map(value -> {
@@ -1122,9 +1195,38 @@ public class TestJsonTreeRowRecordReader {
}
}
- private RecordSchema inferSchema(InputStream jsonStream) throws IOException {
+ private void testReadRecords(InputStream jsonStream, RecordSchema schema, List<Object> expected, StartingFieldStrategy strategy, String startingFieldName)
+ throws IOException, MalformedRecordException {
+ try (JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(jsonStream, mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat,
+ strategy, startingFieldName)) {
+ List<Object> actual = new ArrayList<>();
+ Record record;
+
+ while ((record = reader.nextRecord()) != null) {
+ actual.add(record);
+ }
+
+ List<Function<Object, Object>> propertyProviders = Arrays.asList(
+ _object -> ((Record)_object).getSchema(),
+ _object -> Arrays.stream(((Record)_object).getValues()).map(value -> {
+ if (value != null && value.getClass().isArray()) {
+ return Arrays.asList((Object[]) value);
+ } else {
+ return value;
+ }
+ }).collect(Collectors.toList())
+ );
+
+ List<EqualsWrapper<Object>> wrappedExpected = EqualsWrapper.wrapList(expected, propertyProviders);
+ List<EqualsWrapper<Object>> wrappedActual = EqualsWrapper.wrapList(actual, propertyProviders);
+
+ assertEquals(wrappedExpected, wrappedActual);
+ }
+ }
+
+ private RecordSchema inferSchema(InputStream jsonStream, StartingFieldStrategy strategy, String startingFieldName) throws IOException {
RecordSchema schema = new InferSchemaAccessStrategy<>(
- (__, inputStream) -> new JsonRecordSource(inputStream),
+ (__, inputStream) -> new JsonRecordSource(inputStream, strategy, startingFieldName),
new JsonSchemaInference(new TimeValueInference(null, null, null)),
mock(ComponentLog.class)
).getSchema(Collections.emptyMap(), jsonStream, null);
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/json/TestWriteJsonResult.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/json/TestWriteJsonResult.java
index 465164b18a..84e06523cd 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/json/TestWriteJsonResult.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/json/TestWriteJsonResult.java
@@ -54,10 +54,10 @@ import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
-public class TestWriteJsonResult {
+class TestWriteJsonResult {
@Test
- public void testDataTypes() throws IOException, ParseException {
+ void testDataTypes() throws IOException, ParseException {
final List<RecordField> fields = new ArrayList<>();
for (final RecordFieldType fieldType : RecordFieldType.values()) {
if (fieldType == RecordFieldType.CHOICE) {
@@ -121,7 +121,7 @@ public class TestWriteJsonResult {
@Test
- public void testWriteSerializedForm() throws IOException {
+ void testWriteSerializedForm() throws IOException {
final List<RecordField> fields = new ArrayList<>();
fields.add(new RecordField("name", RecordFieldType.STRING.getDataType()));
fields.add(new RecordField("age", RecordFieldType.INT.getDataType()));
@@ -160,7 +160,7 @@ public class TestWriteJsonResult {
}
@Test
- public void testTimestampWithNullFormat() throws IOException {
+ void testTimestampWithNullFormat() throws IOException {
final Map<String, Object> values = new HashMap<>();
values.put("timestamp", new java.sql.Timestamp(37293723L));
values.put("time", new java.sql.Time(37293723L));
@@ -192,7 +192,7 @@ public class TestWriteJsonResult {
}
@Test
- public void testExtraFieldInWriteRecord() throws IOException {
+ void testExtraFieldInWriteRecord() throws IOException {
final List<RecordField> fields = new ArrayList<>();
fields.add(new RecordField("id", RecordFieldType.STRING.getDataType()));
final RecordSchema schema = new SimpleRecordSchema(fields);
@@ -219,7 +219,7 @@ public class TestWriteJsonResult {
}
@Test
- public void testExtraFieldInWriteRawRecord() throws IOException {
+ void testExtraFieldInWriteRawRecord() throws IOException {
final List<RecordField> fields = new ArrayList<>();
fields.add(new RecordField("id", RecordFieldType.STRING.getDataType()));
final RecordSchema schema = new SimpleRecordSchema(fields);
@@ -246,7 +246,7 @@ public class TestWriteJsonResult {
}
@Test
- public void testMissingFieldInWriteRecord() throws IOException {
+ void testMissingFieldInWriteRecord() throws IOException {
final List<RecordField> fields = new ArrayList<>();
fields.add(new RecordField("id", RecordFieldType.STRING.getDataType()));
fields.add(new RecordField("name", RecordFieldType.STRING.getDataType()));
@@ -273,7 +273,7 @@ public class TestWriteJsonResult {
}
@Test
- public void testMissingFieldInWriteRawRecord() throws IOException {
+ void testMissingFieldInWriteRawRecord() throws IOException {
final List<RecordField> fields = new ArrayList<>();
fields.add(new RecordField("id", RecordFieldType.STRING.getDataType()));
fields.add(new RecordField("name", RecordFieldType.STRING.getDataType()));
@@ -300,7 +300,7 @@ public class TestWriteJsonResult {
}
@Test
- public void testMissingAndExtraFieldInWriteRecord() throws IOException {
+ void testMissingAndExtraFieldInWriteRecord() throws IOException {
final List<RecordField> fields = new ArrayList<>();
fields.add(new RecordField("id", RecordFieldType.STRING.getDataType()));
fields.add(new RecordField("name", RecordFieldType.STRING.getDataType()));
@@ -328,7 +328,7 @@ public class TestWriteJsonResult {
}
@Test
- public void testMissingAndExtraFieldInWriteRawRecord() throws IOException {
+ void testMissingAndExtraFieldInWriteRawRecord() throws IOException {
final List<RecordField> fields = new ArrayList<>();
fields.add(new RecordField("id", RecordFieldType.STRING.getDataType()));
fields.add(new RecordField("name", RecordFieldType.STRING.getDataType()));
@@ -356,7 +356,7 @@ public class TestWriteJsonResult {
}
@Test
- public void testNullSuppression() throws IOException {
+ void testNullSuppression() throws IOException {
final List<RecordField> fields = new ArrayList<>();
fields.add(new RecordField("id", RecordFieldType.STRING.getDataType()));
fields.add(new RecordField("name", RecordFieldType.STRING.getDataType()));
@@ -378,8 +378,8 @@ public class TestWriteJsonResult {
baos.reset();
try (
- final WriteJsonResult writer = new WriteJsonResult(Mockito.mock(ComponentLog.class), schema, new SchemaNameAsAttribute(), baos, false,
- NullSuppression.ALWAYS_SUPPRESS, OutputGrouping.OUTPUT_ARRAY, null, null, null)) {
+ final WriteJsonResult writer = new WriteJsonResult(Mockito.mock(ComponentLog.class), schema, new SchemaNameAsAttribute(), baos, false,
+ NullSuppression.ALWAYS_SUPPRESS, OutputGrouping.OUTPUT_ARRAY, null, null, null)) {
writer.beginRecordSet();
writer.write(recordWithMissingName);
writer.finishRecordSet();
@@ -390,7 +390,7 @@ public class TestWriteJsonResult {
baos.reset();
try (final WriteJsonResult writer = new WriteJsonResult(Mockito.mock(ComponentLog.class), schema, new SchemaNameAsAttribute(), baos, false,
NullSuppression.SUPPRESS_MISSING, OutputGrouping.OUTPUT_ARRAY, null, null,
- null)) {
+ null)) {
writer.beginRecordSet();
writer.write(recordWithMissingName);
writer.finishRecordSet();
@@ -414,8 +414,8 @@ public class TestWriteJsonResult {
baos.reset();
try (
- final WriteJsonResult writer = new WriteJsonResult(Mockito.mock(ComponentLog.class), schema, new SchemaNameAsAttribute(), baos, false,
- NullSuppression.ALWAYS_SUPPRESS, OutputGrouping.OUTPUT_ARRAY, null, null, null)) {
+ final WriteJsonResult writer = new WriteJsonResult(Mockito.mock(ComponentLog.class), schema, new SchemaNameAsAttribute(), baos, false,
+ NullSuppression.ALWAYS_SUPPRESS, OutputGrouping.OUTPUT_ARRAY, null, null, null)) {
writer.beginRecordSet();
writer.write(recordWithNullValue);
writer.finishRecordSet();
@@ -426,7 +426,7 @@ public class TestWriteJsonResult {
baos.reset();
try (final WriteJsonResult writer = new WriteJsonResult(Mockito.mock(ComponentLog.class), schema, new SchemaNameAsAttribute(), baos, false,
NullSuppression.SUPPRESS_MISSING, OutputGrouping.OUTPUT_ARRAY, null, null,
- null)) {
+ null)) {
writer.beginRecordSet();
writer.write(recordWithNullValue);
writer.finishRecordSet();
@@ -437,7 +437,7 @@ public class TestWriteJsonResult {
}
@Test
- public void testOnelineOutput() throws IOException {
+ void testOnelineOutput() throws IOException {
final Map<String, Object> values1 = new HashMap<>();
values1.put("timestamp", new java.sql.Timestamp(37293723L));
values1.put("time", new java.sql.Time(37293723L));
@@ -480,7 +480,7 @@ public class TestWriteJsonResult {
}
@Test
- public void testChoiceArray() throws IOException {
+ void testChoiceArray() throws IOException {
final List<RecordField> fields = new ArrayList<>();
fields.add(new RecordField("path", RecordFieldType.CHOICE.getChoiceDataType(RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.STRING.getDataType()))));
final RecordSchema schema = new SimpleRecordSchema(fields);
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/multiple-nested-field.json b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/multiple-nested-field.json
new file mode 100644
index 0000000000..7b2f47d28a
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/multiple-nested-field.json
@@ -0,0 +1,22 @@
+[
+ {
+ "id": 1,
+ "name": "John Doe",
+ "balance": 4750.89,
+ "address": "123 My Street",
+ "city": "My City",
+ "state": "MS",
+ "zipCode": "11111",
+ "country": "USA",
+ "accountIds": [
+ {
+ "id": "n312kj3",
+ "type": "employee"
+ },
+ {
+ "id": "dl2kdff",
+ "type": "security"
+ }
+ ]
+ }
+]
\ No newline at end of file
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/nested-array-then-start-object.json b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/nested-array-then-start-object.json
new file mode 100644
index 0000000000..662ebb8dc8
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/nested-array-then-start-object.json
@@ -0,0 +1,20 @@
+[
+ {
+ "id": 17,
+ "name": "John",
+ "accounts": [
+ {
+ "id": 42,
+ "balance": 4750.89
+ },
+ {
+ "id": 43,
+ "balance": 48212.38
+ }
+ ]
+ },
+ {
+ "id": 98,
+ "balance": 67829.12
+ }
+]
\ No newline at end of file
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/single-element-nested-array-middle.json b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/single-element-nested-array-middle.json
new file mode 100644
index 0000000000..ccc2121ef1
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/single-element-nested-array-middle.json
@@ -0,0 +1,16 @@
+{
+ "id": 1,
+ "accounts": [{
+ "id": 42,
+ "balance": 4750.89
+ }, {
+ "id": 43,
+ "balance": 48212.38
+ }],
+ "name": "John Doe",
+ "address": "123 My Street",
+ "city": "My City",
+ "state": "MS",
+ "zipCode": "11111",
+ "country": "USA"
+}
\ No newline at end of file
[nifi] 06/09: NIFI-9934 Remove unused Groovy dependency management from nifi-graph-test-clients
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a commit to branch support/nifi-1.16
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit b16aa36681e81ffa352151478f5fe7f610c125a7
Author: exceptionfactory <ex...@apache.org>
AuthorDate: Mon Apr 18 15:08:27 2022 -0500
NIFI-9934 Remove unused Groovy dependency management from nifi-graph-test-clients
Signed-off-by: Pierre Villard <pi...@gmail.com>
This closes #5974.
---
.../nifi-graph-test-clients/pom.xml | 22 +---------------------
1 file changed, 1 insertion(+), 21 deletions(-)
diff --git a/nifi-nar-bundles/nifi-graph-bundle/nifi-graph-test-clients/pom.xml b/nifi-nar-bundles/nifi-graph-bundle/nifi-graph-test-clients/pom.xml
index 537479be7b..c22252f60e 100644
--- a/nifi-nar-bundles/nifi-graph-bundle/nifi-graph-test-clients/pom.xml
+++ b/nifi-nar-bundles/nifi-graph-bundle/nifi-graph-test-clients/pom.xml
@@ -35,27 +35,7 @@
<artifactId>commons-io</artifactId>
<version>2.10.0</version>
</dependency>
- <!-- Override Gremlin and Groovy -->
- <dependency>
- <groupId>org.codehaus.groovy</groupId>
- <artifactId>groovy</artifactId>
- <classifier>indy</classifier>
- </dependency>
- <dependency>
- <groupId>org.codehaus.groovy</groupId>
- <artifactId>groovy-json</artifactId>
- <classifier>indy</classifier>
- </dependency>
- <dependency>
- <groupId>org.codehaus.groovy</groupId>
- <artifactId>groovy-groovysh</artifactId>
- <classifier>indy</classifier>
- </dependency>
- <dependency>
- <groupId>org.codehaus.groovy</groupId>
- <artifactId>groovy-jsr223</artifactId>
- <classifier>indy</classifier>
- </dependency>
+ <!-- Override Gremlin -->
<dependency>
<groupId>org.apache.tinkerpop</groupId>
<artifactId>gremlin-core</artifactId>