You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ma...@apache.org on 2021/01/06 17:10:31 UTC
[nifi] branch main updated: NIFI-6242 PutFileTransfer generating
incorrect provenance event
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 7214dc0 NIFI-6242 PutFileTransfer generating incorrect provenance event
7214dc0 is described below
commit 7214dc0f853e960a3803ce07d6c49237af747488
Author: tlsmith <tl...@evoforge.org>
AuthorDate: Fri Oct 23 18:10:40 2020 +0000
NIFI-6242 PutFileTransfer generating incorrect provenance event
---
.../nifi/processors/standard/PutFileTransfer.java | 6 ++-
.../apache/nifi/processors/standard/TestFTP.java | 51 ++++++++++++++++++++++
.../nifi/processors/standard/TestPutSFTP.java | 48 ++++++++++++++++++++
3 files changed, 104 insertions(+), 1 deletion(-)
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutFileTransfer.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutFileTransfer.java
index 3b9af1a..a57ea22 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutFileTransfer.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutFileTransfer.java
@@ -95,12 +95,16 @@ public abstract class PutFileTransfer<T extends FileTransfer> extends AbstractPr
}
final ComponentLog logger = getLogger();
- final String hostname = context.getProperty(FileTransfer.HOSTNAME).evaluateAttributeExpressions(flowFile).getValue();
+ String hostname = context.getProperty(FileTransfer.HOSTNAME).evaluateAttributeExpressions(flowFile).getValue();
final int maxNumberOfFiles = context.getProperty(FileTransfer.BATCH_SIZE).asInteger();
int fileCount = 0;
try (final T transfer = getFileTransfer(context)) {
do {
+ //check if hostname is regular expression requiring evaluation
+ if(context.getProperty(FileTransfer.HOSTNAME).isExpressionLanguagePresent()) {
+ hostname = context.getProperty(FileTransfer.HOSTNAME).evaluateAttributeExpressions(flowFile).getValue();
+ }
final String rootPath = context.getProperty(FileTransfer.REMOTE_PATH).evaluateAttributeExpressions(flowFile).getValue();
final String workingDirPath;
if (StringUtils.isBlank(rootPath)) {
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestFTP.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestFTP.java
index 1400fc4..5809ea4 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestFTP.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestFTP.java
@@ -146,6 +146,57 @@ public class TestFTP {
}
@Test
+ public void basicProvenanceEventTest() throws IOException {
+ TestRunner runner = TestRunners.newTestRunner(PutFTP.class);
+
+ runner.setProperty(FTPTransfer.HOSTNAME, "localhost");
+ runner.setProperty(FTPTransfer.USERNAME, username);
+ runner.setProperty(FTPTransfer.PASSWORD, password);
+ runner.setProperty(FTPTransfer.PORT, Integer.toString(ftpPort));
+
+ // Get two flowfiles to test by running data
+ try (FileInputStream fis = new FileInputStream("src/test/resources/randombytes-1")) {
+ Map<String, String> attributes = new HashMap<String, String>();
+ attributes.put(CoreAttributes.FILENAME.key(), "randombytes-1");
+ attributes.put("transfer-host", "localhost");
+ runner.enqueue(fis, attributes);
+ runner.run();
+ }
+ try (FileInputStream fis = new FileInputStream("src/test/resources/hello.txt")) {
+ Map<String, String> attributes = new HashMap<String, String>();
+ attributes.put(CoreAttributes.FILENAME.key(), "hello.txt");
+ attributes.put("transfer-host", "127.0.0.1");
+ runner.enqueue(fis, attributes);
+ runner.run();
+ }
+ runner.assertQueueEmpty();
+ runner.assertTransferCount(PutFTP.REL_SUCCESS, 2);
+
+ MockFlowFile flowFile1 = runner.getFlowFilesForRelationship(PutFileTransfer.REL_SUCCESS).get(0);
+ MockFlowFile flowFile2 = runner.getFlowFilesForRelationship(PutFileTransfer.REL_SUCCESS).get(1);
+
+ runner.clearProvenanceEvents();
+ runner.clearTransferState();
+ HashMap<String, String> map1 = new HashMap<>();
+ HashMap<String, String> map2 = new HashMap<>();
+ map1.put(CoreAttributes.FILENAME.key(), "randombytes-xx");
+ map2.put(CoreAttributes.FILENAME.key(), "randombytes-yy");
+
+ flowFile1.putAttributes(map1);
+ flowFile2.putAttributes(map2);
+ //set to derive hostname
+ runner.setProperty(FTPTransfer.HOSTNAME, "${transfer-host}");
+ runner.setThreadCount(1);
+ runner.enqueue(flowFile1);
+ runner.enqueue(flowFile2);
+ runner.run();
+
+ runner.assertTransferCount(PutFTP.REL_SUCCESS, 2);
+ assert(runner.getProvenanceEvents().get(0).getTransitUri().contains("ftp://localhost"));
+ assert(runner.getProvenanceEvents().get(1).getTransitUri().contains("ftp://127.0.0.1"));
+ }
+
+ @Test
public void basicFileGet() throws IOException {
FileSystem results = fakeFtpServer.getFileSystem();
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestPutSFTP.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestPutSFTP.java
index 194e485..36aa2ed 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestPutSFTP.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestPutSFTP.java
@@ -20,6 +20,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.nifi.processors.standard.util.FileTransfer;
import org.apache.nifi.processors.standard.util.SFTPTransfer;
import org.apache.nifi.processors.standard.util.SSHTestServer;
+import org.apache.nifi.util.MockFlowFile;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
import org.junit.AfterClass;
@@ -243,9 +244,56 @@ public class TestPutSFTP {
putSFTPRunner.clearTransferState();
}
+ @Test
+ public void testPutSFTPProvenanceTransitUri() throws IOException {
+ emptyTestDirectory();
+
+ putSFTPRunner.setProperty(SFTPTransfer.REJECT_ZERO_BYTE, "false");
+ Map<String,String> attributes = new HashMap<>();
+ attributes.put("filename", "testfile.txt");
+ attributes.put("transfer-host","localhost");
+ putSFTPRunner.enqueue(Paths.get(testFile), attributes);
+
+ attributes = new HashMap<>();
+ attributes.put("filename", "testfile1.txt");
+ attributes.put("transfer-host","127.0.0.1");
+
+ putSFTPRunner.enqueue(Paths.get(testFile), attributes);
+ putSFTPRunner.run();
+
+ putSFTPRunner.assertTransferCount(PutSFTP.REL_SUCCESS, 2);
+ putSFTPRunner.getProvenanceEvents().forEach(k->{
+ assert(k.getTransitUri().contains("sftp://localhost"));
+ });
+ //Two files in batch, should have 2 transferred to success, 0 to failure
+ putSFTPRunner.assertTransferCount(PutSFTP.REL_SUCCESS, 2);
+ putSFTPRunner.assertTransferCount(PutSFTP.REL_REJECT, 0);
+
+ MockFlowFile flowFile1 = putSFTPRunner.getFlowFilesForRelationship(PutFileTransfer.REL_SUCCESS).get(0);
+ MockFlowFile flowFile2 = putSFTPRunner.getFlowFilesForRelationship(PutFileTransfer.REL_SUCCESS).get(1);
+ putSFTPRunner.clearProvenanceEvents();
+ putSFTPRunner.clearTransferState();
+
+ //Test different destinations on flow file attributes
+ putSFTPRunner.setProperty(SFTPTransfer.HOSTNAME,"${transfer-host}"); //set to derive hostname
+
+ putSFTPRunner.setThreadCount(1);
+ putSFTPRunner.enqueue(flowFile1);
+ putSFTPRunner.enqueue(flowFile2);
+ putSFTPRunner.run();
+
+ putSFTPRunner.assertTransferCount(PutSFTP.REL_SUCCESS, 2);
+ assert(putSFTPRunner.getProvenanceEvents().get(0).getTransitUri().contains("sftp://localhost"));
+ assert(putSFTPRunner.getProvenanceEvents().get(1).getTransitUri().contains("sftp://127.0.0.1"));
+
+ putSFTPRunner.clearProvenanceEvents();
+ putSFTPRunner.clearTransferState();
+ }
+
private void emptyTestDirectory() throws IOException {
//Delete Virtual File System folder
Path dir = Paths.get(sshTestServer.getVirtualFileSystemPath());
FileUtils.cleanDirectory(dir.toFile());
}
+
}
\ No newline at end of file