You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by jo...@apache.org on 2018/11/21 16:37:59 UTC
nifi git commit: NIFI-5560 This closes #3000. Added Follow SYMLINK
support for listFTP & listSFTP and getFTP & getSFTP processors
Repository: nifi
Updated Branches:
refs/heads/master 286a1c880 -> 71499f774
NIFI-5560 This closes #3000. Added Follow SYMLINK support for listFTP & listSFTP and getFTP & getSFTP processors
Signed-off-by: joewitt <jo...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/71499f77
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/71499f77
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/71499f77
Branch: refs/heads/master
Commit: 71499f774d5c118c58c38be76283a512005f1e92
Parents: 286a1c8
Author: Kumara M S Hemantha <he...@nokia.com>
Authored: Thu Sep 13 02:19:32 2018 +0530
Committer: joewitt <jo...@apache.org>
Committed: Wed Nov 21 10:53:38 2018 -0500
----------------------------------------------------------------------
.../java/org/apache/nifi/processors/standard/GetFTP.java | 1 +
.../java/org/apache/nifi/processors/standard/GetSFTP.java | 1 +
.../java/org/apache/nifi/processors/standard/ListFTP.java | 1 +
.../java/org/apache/nifi/processors/standard/ListSFTP.java | 1 +
.../apache/nifi/processors/standard/util/FTPTransfer.java | 7 +++++--
.../apache/nifi/processors/standard/util/FileTransfer.java | 8 ++++++++
.../apache/nifi/processors/standard/util/SFTPTransfer.java | 8 ++++++--
7 files changed, 23 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/71499f77/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetFTP.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetFTP.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetFTP.java
index 79d7914..970eaeb 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetFTP.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetFTP.java
@@ -72,6 +72,7 @@ public class GetFTP extends GetFileTransfer {
properties.add(FTPTransfer.PATH_FILTER_REGEX);
properties.add(FTPTransfer.POLLING_INTERVAL);
properties.add(FTPTransfer.RECURSIVE_SEARCH);
+ properties.add(FTPTransfer.FOLLOW_SYMLINK);
properties.add(FTPTransfer.IGNORE_DOTTED_FILES);
properties.add(FTPTransfer.DELETE_ORIGINAL);
properties.add(FTPTransfer.CONNECTION_TIMEOUT);
http://git-wip-us.apache.org/repos/asf/nifi/blob/71499f77/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetSFTP.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetSFTP.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetSFTP.java
index 6b891c7..5a3d667 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetSFTP.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetSFTP.java
@@ -71,6 +71,7 @@ public class GetSFTP extends GetFileTransfer {
properties.add(SFTPTransfer.PATH_FILTER_REGEX);
properties.add(SFTPTransfer.POLLING_INTERVAL);
properties.add(SFTPTransfer.RECURSIVE_SEARCH);
+ properties.add(SFTPTransfer.FOLLOW_SYMLINK);
properties.add(SFTPTransfer.IGNORE_DOTTED_FILES);
properties.add(SFTPTransfer.DELETE_ORIGINAL);
properties.add(SFTPTransfer.CONNECTION_TIMEOUT);
http://git-wip-us.apache.org/repos/asf/nifi/blob/71499f77/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListFTP.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListFTP.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListFTP.java
index 2968c85..41e2e80 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListFTP.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListFTP.java
@@ -79,6 +79,7 @@ public class ListFTP extends ListFileTransfer {
properties.add(REMOTE_PATH);
properties.add(DISTRIBUTED_CACHE_SERVICE);
properties.add(FTPTransfer.RECURSIVE_SEARCH);
+ properties.add(FTPTransfer.FOLLOW_SYMLINK);
properties.add(FTPTransfer.FILE_FILTER_REGEX);
properties.add(FTPTransfer.PATH_FILTER_REGEX);
properties.add(FTPTransfer.IGNORE_DOTTED_FILES);
http://git-wip-us.apache.org/repos/asf/nifi/blob/71499f77/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListSFTP.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListSFTP.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListSFTP.java
index 2b5d47a..89012b0 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListSFTP.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListSFTP.java
@@ -83,6 +83,7 @@ public class ListSFTP extends ListFileTransfer {
properties.add(REMOTE_PATH);
properties.add(DISTRIBUTED_CACHE_SERVICE);
properties.add(SFTPTransfer.RECURSIVE_SEARCH);
+ properties.add(SFTPTransfer.FOLLOW_SYMLINK);
properties.add(SFTPTransfer.FILE_FILTER_REGEX);
properties.add(SFTPTransfer.PATH_FILTER_REGEX);
properties.add(SFTPTransfer.IGNORE_DOTTED_FILES);
http://git-wip-us.apache.org/repos/asf/nifi/blob/71499f77/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/FTPTransfer.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/FTPTransfer.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/FTPTransfer.java
index e2ddf6b..85fee40 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/FTPTransfer.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/FTPTransfer.java
@@ -204,6 +204,7 @@ public class FTPTransfer implements FileTransfer {
final boolean ignoreDottedFiles = ctx.getProperty(FileTransfer.IGNORE_DOTTED_FILES).asBoolean();
final boolean recurse = ctx.getProperty(FileTransfer.RECURSIVE_SEARCH).asBoolean();
+ final boolean symlink = ctx.getProperty(FileTransfer.FOLLOW_SYMLINK).asBoolean();
final String fileFilterRegex = ctx.getProperty(FileTransfer.FILE_FILTER_REGEX).getValue();
final Pattern pattern = (fileFilterRegex == null) ? null : Pattern.compile(fileFilterRegex);
final String pathFilterRegex = ctx.getProperty(FileTransfer.PATH_FILTER_REGEX).getValue();
@@ -255,11 +256,13 @@ public class FTPTransfer implements FileTransfer {
final File newFullPath = new File(path, filename);
final String newFullForwardPath = newFullPath.getPath().replace("\\", "/");
- if (recurse && file.isDirectory()) {
+ // if is a directory and we're supposed to recurse
+ // OR if is a link and we're supposed to follow symlink
+ if ((recurse && file.isDirectory()) || (symlink && file.isSymbolicLink())) {
try {
listing.addAll(getListing(newFullForwardPath, depth + 1, maxResults - count));
} catch (final IOException e) {
- logger.error("Unable to get listing from " + newFullForwardPath + "; skipping this subdirectory", e);
+ logger.error("Unable to get listing from " + newFullForwardPath + "; skipping", e);
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/71499f77/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/FileTransfer.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/FileTransfer.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/FileTransfer.java
index d500b9d..56dd22d 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/FileTransfer.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/FileTransfer.java
@@ -141,6 +141,14 @@ public interface FileTransfer extends Closeable {
.defaultValue("false")
.allowableValues("true", "false")
.build();
+ public static final PropertyDescriptor FOLLOW_SYMLINK = new PropertyDescriptor.Builder()
+ .name("follow-symlink")
+ .displayName("Follow symlink")
+ .description("If true, will pull even symbolic files and also nested symbolic subdirectories; otherwise, will not read symbolic files and will not traverse symbolic link subdirectories")
+ .required(true)
+ .defaultValue("false")
+ .allowableValues("true", "false")
+ .build();
public static final PropertyDescriptor FILE_FILTER_REGEX = new PropertyDescriptor.Builder()
.name("File Filter Regex")
.description("Provides a Java Regular Expression for filtering Filenames; if a filter is supplied, only files whose names match that Regular Expression will be fetched")
http://git-wip-us.apache.org/repos/asf/nifi/blob/71499f77/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/SFTPTransfer.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/SFTPTransfer.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/SFTPTransfer.java
index b418b4f..34bc0e3 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/SFTPTransfer.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/SFTPTransfer.java
@@ -188,6 +188,7 @@ public class SFTPTransfer implements FileTransfer {
final boolean ignoreDottedFiles = ctx.getProperty(FileTransfer.IGNORE_DOTTED_FILES).asBoolean();
final boolean recurse = ctx.getProperty(FileTransfer.RECURSIVE_SEARCH).asBoolean();
+ final boolean symlink = ctx.getProperty(FileTransfer.FOLLOW_SYMLINK).asBoolean();
final String fileFilterRegex = ctx.getProperty(FileTransfer.FILE_FILTER_REGEX).getValue();
final Pattern pattern = (fileFilterRegex == null) ? null : Pattern.compile(fileFilterRegex);
final String pathFilterRegex = ctx.getProperty(FileTransfer.PATH_FILTER_REGEX).getValue();
@@ -211,6 +212,7 @@ public class SFTPTransfer implements FileTransfer {
final ChannelSftp sftp = getChannel(null);
final boolean isPathMatch = pathFilterMatches;
+ //subDirs list is used for both 'sub directories' and 'symlinks'
final List<LsEntry> subDirs = new ArrayList<>();
try {
final LsEntrySelector filter = new LsEntrySelector() {
@@ -231,7 +233,8 @@ public class SFTPTransfer implements FileTransfer {
}
// if is a directory and we're supposed to recurse
- if (recurse && entry.getAttrs().isDir()) {
+ // OR if is a link and we're supposed to follow symlink
+ if ((recurse && entry.getAttrs().isDir()) || (symlink && entry.getAttrs().isLink())){
subDirs.add(entry);
return LsEntrySelector.CONTINUE;
}
@@ -278,9 +281,10 @@ public class SFTPTransfer implements FileTransfer {
try {
getListing(newFullForwardPath, depth + 1, maxResults, listing);
} catch (final IOException e) {
- logger.error("Unable to get listing from " + newFullForwardPath + "; skipping this subdirectory", e);
+ logger.error("Unable to get listing from " + newFullForwardPath + "; skipping", e);
}
}
+
}
private FileInfo newFileInfo(final LsEntry entry, String path) {