[nifi] branch main updated: NIFI-8736 Adding capability to override HDFS location for NAR autoloading

[pv...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

pvillard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git


The following commit(s) were added to refs/heads/main by this push:
     new a3f54fa  NIFI-8736 Adding capability to override HDFS location for NAR autoloading
a3f54fa is described below

commit a3f54fa578f86220ff63e73f526e510de3485f41
Author: Bence Simon <si...@gmail.com>
AuthorDate: Thu Jun 24 17:50:05 2021 +0200

    NIFI-8736 Adding capability to override HDFS location for NAR autoloading
    
    Signed-off-by: Pierre Villard <pi...@gmail.com>
    
    This closes #5185.
---
 nifi-docs/src/main/asciidoc/administration-guide.adoc          |  1 +
 .../main/java/org/apache/nifi/nar/hadoop/HDFSNarProvider.java  | 10 +++++++++-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/nifi-docs/src/main/asciidoc/administration-guide.adoc b/nifi-docs/src/main/asciidoc/administration-guide.adoc
index eb5ecac..2e9e423 100644
--- a/nifi-docs/src/main/asciidoc/administration-guide.adoc
+++ b/nifi-docs/src/main/asciidoc/administration-guide.adoc
@@ -4085,6 +4085,7 @@ The value of the `nifi.nar.library.provider.<providerName>.implementation` must
 | Name                       |  Description
 | resources                  |  List of HDFS resources, separated by comma.
 | source.directory           |  The source directory of NAR files within HDFS. Note: the provider does not check for files recursively.
+| storage.location           |  Optional. If set the storage location defined in the core-site.xml will be overwritten by this value.
 | kerberos.principal         |  Optional. Kerberos principal to authenticate as.
 | kerberos.keytab            |  Optional. Kerberos keytab associated with the principal.
 | kerberos.password          |  Optional. Kerberos password associated with the principal.
diff --git a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/main/java/org/apache/nifi/nar/hadoop/HDFSNarProvider.java b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/main/java/org/apache/nifi/nar/hadoop/HDFSNarProvider.java
index 6c852b7..06714d6 100644
--- a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/main/java/org/apache/nifi/nar/hadoop/HDFSNarProvider.java
+++ b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/main/java/org/apache/nifi/nar/hadoop/HDFSNarProvider.java
@@ -56,6 +56,7 @@ public class HDFSNarProvider implements NarProvider {
 
     private static final String RESOURCES_PARAMETER = "resources";
     private static final String SOURCE_DIRECTORY_PARAMETER = "source.directory";
+    private static final String STORAGE_LOCATION = "storage.location";
     private static final String KERBEROS_PRINCIPAL_PARAMETER = "kerberos.principal";
     private static final String KERBEROS_KEYTAB_PARAMETER = "kerberos.keytab";
     private static final String KERBEROS_PASSWORD_PARAMETER = "kerberos.password";
@@ -64,9 +65,11 @@ public class HDFSNarProvider implements NarProvider {
     private static final String DELIMITER = "/";
     private static final int BUFFER_SIZE_DEFAULT = 4096;
     private static final Object RESOURCES_LOCK = new Object();
+    private static final String STORAGE_LOCATION_PROPERTY = "fs.defaultFS";
 
     private volatile List<String> resources = null;
     private volatile Path sourceDirectory = null;
+    private volatile String storageLocation = null;
 
     private volatile NarProviderInitializationContext context;
 
@@ -87,6 +90,7 @@ public class HDFSNarProvider implements NarProvider {
         }
 
         this.sourceDirectory = new Path(sourceDirectory);
+        this.storageLocation = context.getProperties().get(STORAGE_LOCATION);
 
         this.context = context;
         this.initialized = true;
@@ -100,7 +104,6 @@ public class HDFSNarProvider implements NarProvider {
 
         final HdfsResources hdfsResources = getHdfsResources();
 
-
         try {
             final FileStatus[] fileStatuses = hdfsResources.getUserGroupInformation()
                 .doAs((PrivilegedExceptionAction<FileStatus[]>) () -> hdfsResources.getFileSystem().listStatus(sourceDirectory, new ExtensionFilter(NAR_EXTENSION)));
@@ -162,6 +165,11 @@ public class HDFSNarProvider implements NarProvider {
             config.addResource(new Path(resource));
         }
 
+        // If storage location property is set, the original location will be overwritten
+        if (storageLocation != null) {
+            config.set(STORAGE_LOCATION_PROPERTY, storageLocation);
+        }
+
         // first check for timeout on HDFS connection, because FileSystem has a hard coded 15 minute timeout
         checkHdfsUriForTimeout(config);