You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by tk...@apache.org on 2015/11/08 03:18:32 UTC
nifi git commit: NIFI-1025 Updated aws-sdk and joda-time to resolve
AWS issues with Java 1.8 update 60. - Refactored tests - created
AbstractS3Test for common utility methods - Corrected incorrect unit test in
TestDeleteS3Object, and adjusted processor
Repository: nifi
Updated Branches:
refs/heads/master 99629646f -> 77de51df1
NIFI-1025 Updated aws-sdk and joda-time to resolve AWS issues with Java 1.8 update 60.
- Refactored tests - created AbstractS3Test for common utility methods
- Corrected incorrect unit test in TestDeleteS3Object, and adjusted processor documentation to reflect behavior
- moved aws dependency management to root pom
This closes #107
Tested, Reviewed and Amended by Tony Kurc (<tk...@apache.org>)
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/77de51df
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/77de51df
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/77de51df
Branch: refs/heads/master
Commit: 77de51df19b768ea1180d52e0e6106629161add0
Parents: 9962964
Author: Venkatesh Sellappa <vs...@outlook.com>
Authored: Sat Nov 7 21:09:18 2015 -0500
Committer: Tony Kurc <tr...@gmail.com>
Committed: Sat Nov 7 21:09:18 2015 -0500
----------------------------------------------------------------------
.../nifi/processors/aws/s3/DeleteS3Object.java | 3 +-
.../nifi/processors/aws/s3/AbstractS3Test.java | 151 +++++++++++++++++++
.../processors/aws/s3/TestDeleteS3Object.java | 118 ++++++---------
.../processors/aws/s3/TestFetchS3Object.java | 55 +++++--
.../nifi/processors/aws/s3/TestPutS3Object.java | 42 +++---
nifi-nar-bundles/nifi-aws-bundle/pom.xml | 10 --
pom.xml | 7 +-
7 files changed, 273 insertions(+), 113 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/77de51df/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/DeleteS3Object.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/DeleteS3Object.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/DeleteS3Object.java
index 836e0d8..056ff01 100644
--- a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/DeleteS3Object.java
+++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/DeleteS3Object.java
@@ -41,7 +41,7 @@ import org.apache.nifi.processor.util.StandardValidators;
@SeeAlso({PutS3Object.class})
@Tags({"Amazon", "S3", "AWS", "Archive", "Delete"})
@CapabilityDescription("Deletes FlowFiles on an Amazon S3 Bucket. " +
- "And the FlowFiles are checked if exists or not before deleting.")
+ "If attempting to delete a file that does not exist, FlowFile is routed to success.")
public class DeleteS3Object extends AbstractS3Processor {
public static final PropertyDescriptor VERSION_ID = new PropertyDescriptor.Builder()
@@ -80,6 +80,7 @@ public class DeleteS3Object extends AbstractS3Processor {
try {
if (versionId == null) {
final DeleteObjectRequest r = new DeleteObjectRequest(bucket, key);
+ // This call returns success if object doesn't exist
s3.deleteObject(r);
} else {
final DeleteVersionRequest r = new DeleteVersionRequest(bucket, key, versionId);
http://git-wip-us.apache.org/repos/asf/nifi/blob/77de51df/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/AbstractS3Test.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/AbstractS3Test.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/AbstractS3Test.java
new file mode 100644
index 0000000..167c16b
--- /dev/null
+++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/AbstractS3Test.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.aws.s3;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Iterator;
+
+import org.apache.nifi.util.file.FileUtils;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+
+import static org.junit.Assert.fail;
+
+import com.amazonaws.auth.PropertiesCredentials;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.model.AmazonS3Exception;
+import com.amazonaws.services.s3.model.CreateBucketRequest;
+import com.amazonaws.services.s3.model.DeleteBucketRequest;
+import com.amazonaws.services.s3.model.ObjectListing;
+import com.amazonaws.services.s3.model.PutObjectRequest;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+
+/**
+ * Base class for S3 tests. Establishes a bucket and helper methods for creating test scenarios
+ */
+public abstract class AbstractS3Test {
+ protected final static String CREDENTIALS_FILE = System.getProperty("user.home") + "/aws-credentials.properties";
+ protected final static String BUCKET_NAME = "test-bucket-00000000-0000-0000-0000-123456789021";
+ protected final static String SAMPLE_FILE_RESOURCE_NAME = "/hello.txt";
+ protected final static String REGION = "eu-west-1";
+
+ // Static so multiple Tests can use same client
+ protected static AmazonS3Client client;
+
+ @BeforeClass
+ public static void oneTimeSetup() {
+ // Creates a client and bucket for this test
+
+ final FileInputStream fis;
+ try {
+ fis = new FileInputStream(CREDENTIALS_FILE);
+ } catch (FileNotFoundException e1) {
+ fail("Could not open credentials file " + CREDENTIALS_FILE + ": " + e1.getLocalizedMessage());
+ return;
+ }
+ try {
+ final PropertiesCredentials credentials = new PropertiesCredentials(fis);
+ client = new AmazonS3Client(credentials);
+
+ if (client.doesBucketExist(BUCKET_NAME)) {
+ fail("Bucket " + BUCKET_NAME + " exists. Choose a different bucket name to continue test");
+ }
+
+ CreateBucketRequest request = new CreateBucketRequest(BUCKET_NAME, REGION);
+ client.createBucket(request);
+
+ } catch (final AmazonS3Exception e) {
+ fail("Can't create the key " + BUCKET_NAME + ": " + e.getLocalizedMessage());
+ } catch (final IOException e) {
+ fail("Caught IOException preparing tests: " + e.getLocalizedMessage());
+ } finally {
+ FileUtils.closeQuietly(fis);
+ }
+
+ if (!client.doesBucketExist(BUCKET_NAME)) {
+ fail("Setup incomplete, tests will fail");
+ }
+ }
+
+ @AfterClass
+ public static void oneTimeTearDown() {
+ // Empty the bucket before deleting it.
+ try {
+ ObjectListing objectListing = client.listObjects(BUCKET_NAME);
+
+ while (true) {
+ for (Iterator<?> iterator = objectListing.getObjectSummaries().iterator(); iterator.hasNext(); ) {
+ S3ObjectSummary objectSummary = (S3ObjectSummary) iterator.next();
+ client.deleteObject(BUCKET_NAME, objectSummary.getKey());
+ }
+
+ if (objectListing.isTruncated()) {
+ objectListing = client.listNextBatchOfObjects(objectListing);
+ } else {
+ break;
+ }
+ }
+
+ DeleteBucketRequest dbr = new DeleteBucketRequest(BUCKET_NAME);
+ client.deleteBucket(dbr);
+ } catch (final AmazonS3Exception e) {
+ System.err.println("Unable to delete bucket " + BUCKET_NAME + e.toString());
+ }
+
+ if (client.doesBucketExist(BUCKET_NAME)) {
+ Assert.fail("Incomplete teardown, subsequent tests might fail");
+ }
+
+ }
+
+ protected void putTestFile(String key, File file) throws AmazonS3Exception {
+ PutObjectRequest putRequest = new PutObjectRequest(BUCKET_NAME, key, file);
+
+ client.putObject(putRequest);
+ }
+
+ protected Path getResourcePath(String resourceName) {
+ Path path = null;
+
+ try {
+ path = Paths.get(getClass().getResource(resourceName).toURI());
+ } catch (URISyntaxException e) {
+ Assert.fail("Resource: " + resourceName + " does not exist" + e.getLocalizedMessage());
+ }
+
+ return path;
+ }
+
+ protected File getFileFromResourceName(String resourceName) {
+ URI uri = null;
+ try {
+ uri = this.getClass().getResource(resourceName).toURI();
+ } catch (URISyntaxException e) {
+ Assert.fail("Cannot proceed without File : " + resourceName);
+ }
+
+ return new File(uri);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/77de51df/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestDeleteS3Object.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestDeleteS3Object.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestDeleteS3Object.java
index 509a31f..643aa11 100644
--- a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestDeleteS3Object.java
+++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestDeleteS3Object.java
@@ -16,97 +16,74 @@
*/
package org.apache.nifi.processors.aws.s3;
-import java.io.File;
-import java.io.FileInputStream;
import java.io.IOException;
-import java.net.URL;
import java.util.HashMap;
import java.util.Map;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
+
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
import org.junit.Ignore;
import org.junit.Test;
-import com.amazonaws.auth.PropertiesCredentials;
-import com.amazonaws.services.s3.AmazonS3Client;
-import com.amazonaws.services.s3.model.AmazonS3Exception;
-import com.amazonaws.services.s3.model.CreateBucketRequest;
-import com.amazonaws.services.s3.model.DeleteBucketRequest;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.PutObjectResult;
+@Ignore("For local testing only - interacts with S3 so the credentials file must be configured and all necessary buckets created")
+public class TestDeleteS3Object extends AbstractS3Test {
-import org.apache.nifi.util.TestRunner;
-import org.apache.nifi.util.TestRunners;
+ @Test
+ public void testSimpleDelete() throws IOException {
+ // Prepares for this test
+ putTestFile("delete-me", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));
+ final TestRunner runner = TestRunners.newTestRunner(new DeleteS3Object());
-@Ignore("For local testing only - interacts with S3 so the credentials file must be configured and all necessary buckets created")
-public class TestDeleteS3Object {
-
- private static final String CREDENTIALS_FILE = System.getProperty("user.home") + "/aws-credentials.properties";
-
- // When you want to test this, you should create a bucket on Amazon S3 as follows.
- private static final String TEST_REGION = "ap-northeast-1";
- private static final String TEST_BUCKET = "test-bucket-00000000-0000-0000-0000-1234567890123";
-
- @BeforeClass
- public static void oneTimeSetUp() {
- // Creates a new bucket for this test
- try {
- PropertiesCredentials credentials = new PropertiesCredentials(new FileInputStream(CREDENTIALS_FILE));
- AmazonS3Client client = new AmazonS3Client(credentials);
- CreateBucketRequest request = new CreateBucketRequest(TEST_BUCKET, TEST_REGION);
- client.createBucket(request);
- } catch (final AmazonS3Exception e) {
- System.out.println("Can't create the key " + TEST_BUCKET + ":" + e.toString());
- } catch (final IOException e) {
- System.out.println(CREDENTIALS_FILE + " doesn't exist.");
- }
- }
+ runner.setProperty(DeleteS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
+ runner.setProperty(DeleteS3Object.REGION, REGION);
+ runner.setProperty(DeleteS3Object.BUCKET, BUCKET_NAME);
+
+ final Map<String, String> attrs = new HashMap<>();
+ attrs.put("filename", "delete-me");
+ runner.enqueue(new byte[0], attrs);
- @AfterClass
- public static void oneTimeTearDown() throws IOException {
- // Delete a bucket for this test
- PropertiesCredentials credentials = new PropertiesCredentials(new FileInputStream(CREDENTIALS_FILE));
- AmazonS3Client client = new AmazonS3Client(credentials);
- DeleteBucketRequest dbr = new DeleteBucketRequest(TEST_BUCKET);
- client.deleteBucket(dbr);
+ runner.run(1);
+
+ runner.assertAllFlowFilesTransferred(DeleteS3Object.REL_SUCCESS, 1);
}
@Test
- public void testSimpleDelete() throws IOException {
+ public void testDeleteFolder() throws IOException {
// Prepares for this test
- uploadTestFile("hello.txt");
+ putTestFile("folder/delete-me", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));
+
+ final TestRunner runner = TestRunners.newTestRunner(new DeleteS3Object());
- DeleteS3Object deleter = new DeleteS3Object();
- final TestRunner runner = TestRunners.newTestRunner(deleter);
runner.setProperty(DeleteS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
- runner.setProperty(DeleteS3Object.REGION, TEST_REGION);
- runner.setProperty(DeleteS3Object.BUCKET, TEST_BUCKET);
- runner.setProperty(DeleteS3Object.KEY, "hello.txt");
+ runner.setProperty(DeleteS3Object.REGION, REGION);
+ runner.setProperty(DeleteS3Object.BUCKET, BUCKET_NAME);
final Map<String, String> attrs = new HashMap<>();
- attrs.put("filename", "hello.txt");
+ attrs.put("filename", "folder/delete-me");
runner.enqueue(new byte[0], attrs);
+
runner.run(1);
runner.assertAllFlowFilesTransferred(DeleteS3Object.REL_SUCCESS, 1);
}
@Test
- public void testDeleteFolder() throws IOException {
+ public void testDeleteFolderNoExpressionLanguage() throws IOException {
// Prepares for this test
- uploadTestFile("folder/1.txt");
+ putTestFile("folder/delete-me", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));
+
+ final TestRunner runner = TestRunners.newTestRunner(new DeleteS3Object());
- DeleteS3Object deleter = new DeleteS3Object();
- final TestRunner runner = TestRunners.newTestRunner(deleter);
runner.setProperty(DeleteS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
- runner.setProperty(DeleteS3Object.REGION, TEST_REGION);
- runner.setProperty(DeleteS3Object.BUCKET, TEST_BUCKET);
- runner.setProperty(DeleteS3Object.KEY, "folder/1.txt");
+ runner.setProperty(DeleteS3Object.REGION, REGION);
+ runner.setProperty(DeleteS3Object.BUCKET, BUCKET_NAME);
+ runner.setProperty(DeleteS3Object.KEY, "folder/delete-me");
final Map<String, String> attrs = new HashMap<>();
- attrs.put("filename", "hello.txt");
+ attrs.put("filename", "a-different-name");
runner.enqueue(new byte[0], attrs);
+
runner.run(1);
runner.assertAllFlowFilesTransferred(DeleteS3Object.REL_SUCCESS, 1);
@@ -114,28 +91,19 @@ public class TestDeleteS3Object {
@Test
public void testTryToDeleteNotExistingFile() throws IOException {
- DeleteS3Object deleter = new DeleteS3Object();
- final TestRunner runner = TestRunners.newTestRunner(deleter);
+ final TestRunner runner = TestRunners.newTestRunner(new DeleteS3Object());
+
runner.setProperty(DeleteS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
- runner.setProperty(DeleteS3Object.REGION, TEST_REGION);
- runner.setProperty(DeleteS3Object.BUCKET, TEST_BUCKET);
- runner.setProperty(DeleteS3Object.BUCKET, "no-such-a-key");
+ runner.setProperty(DeleteS3Object.REGION, REGION);
+ runner.setProperty(DeleteS3Object.BUCKET, BUCKET_NAME);
final Map<String, String> attrs = new HashMap<>();
attrs.put("filename", "no-such-a-file");
runner.enqueue(new byte[0], attrs);
+
runner.run(1);
- runner.assertAllFlowFilesTransferred(DeleteS3Object.REL_FAILURE, 1);
+ runner.assertAllFlowFilesTransferred(DeleteS3Object.REL_SUCCESS, 1);
}
- // Uploads a test file
- private void uploadTestFile(String key) throws IOException {
- PropertiesCredentials credentials = new PropertiesCredentials(new FileInputStream(CREDENTIALS_FILE));
- AmazonS3Client client = new AmazonS3Client(credentials);
- URL fileURL = this.getClass().getClassLoader().getResource("hello.txt");
- File file = new File(fileURL.getPath());
- PutObjectRequest putRequest = new PutObjectRequest(TEST_BUCKET, key, file);
- PutObjectResult result = client.putObject(putRequest);
- }
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/77de51df/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestFetchS3Object.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestFetchS3Object.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestFetchS3Object.java
index 0321514..e7d3e8b 100644
--- a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestFetchS3Object.java
+++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestFetchS3Object.java
@@ -18,7 +18,6 @@ package org.apache.nifi.processors.aws.s3;
import java.io.IOException;
import java.nio.file.Files;
-import java.nio.file.Paths;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -30,32 +29,70 @@ import org.junit.Ignore;
import org.junit.Test;
@Ignore("For local testing only - interacts with S3 so the credentials file must be configured and all necessary buckets created")
-public class TestFetchS3Object {
+public class TestFetchS3Object extends AbstractS3Test {
+ @Test
+ public void testSimpleGet() throws IOException {
+ putTestFile("test-file", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));
+
+ final TestRunner runner = TestRunners.newTestRunner(new FetchS3Object());
+
+ runner.setProperty(FetchS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
+ runner.setProperty(FetchS3Object.REGION, REGION);
+ runner.setProperty(FetchS3Object.BUCKET, BUCKET_NAME);
- private final String CREDENTIALS_FILE = System.getProperty("user.home") + "/aws-credentials.properties";
+ final Map<String, String> attrs = new HashMap<>();
+ attrs.put("filename", "test-file");
+ runner.enqueue(new byte[0], attrs);
+
+ runner.run(1);
+
+ runner.assertAllFlowFilesTransferred(FetchS3Object.REL_SUCCESS, 1);
+ }
@Test
- public void testGet() throws IOException {
+ public void testTryToFetchNotExistingFile() throws IOException {
final TestRunner runner = TestRunners.newTestRunner(new FetchS3Object());
- runner.setProperty(FetchS3Object.BUCKET, "anonymous-test-bucket-00000000");
+
runner.setProperty(FetchS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
- runner.setProperty(FetchS3Object.KEY, "folder/1.txt");
+ runner.setProperty(FetchS3Object.REGION, REGION);
+ runner.setProperty(FetchS3Object.BUCKET, BUCKET_NAME);
final Map<String, String> attrs = new HashMap<>();
- attrs.put("start", "0");
+ attrs.put("filename", "no-such-a-file");
+ runner.enqueue(new byte[0], attrs);
+ runner.run(1);
+
+ runner.assertAllFlowFilesTransferred(FetchS3Object.REL_FAILURE, 1);
+ }
+
+ @Test
+ public void testContentsOfFileRetrieved() throws IOException {
+ String key = "folder/1.txt";
+ putTestFile(key, getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));
+
+ final TestRunner runner = TestRunners.newTestRunner(new FetchS3Object());
+
+ runner.setProperty(FetchS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
+ runner.setProperty(FetchS3Object.REGION, REGION);
+ runner.setProperty(FetchS3Object.BUCKET, BUCKET_NAME);
+
+ final Map<String, String> attrs = new HashMap<>();
+ attrs.put("filename", key);
runner.enqueue(new byte[0], attrs);
+
runner.run(1);
runner.assertAllFlowFilesTransferred(FetchS3Object.REL_SUCCESS, 1);
+
final List<MockFlowFile> ffs = runner.getFlowFilesForRelationship(FetchS3Object.REL_SUCCESS);
final MockFlowFile out = ffs.iterator().next();
- final byte[] expectedBytes = Files.readAllBytes(Paths.get("src/test/resources/hello.txt"));
+ final byte[] expectedBytes = Files.readAllBytes(getResourcePath(SAMPLE_FILE_RESOURCE_NAME));
out.assertContentEquals(new String(expectedBytes));
+
for (final Map.Entry<String, String> entry : out.getAttributes().entrySet()) {
System.out.println(entry.getKey() + " : " + entry.getValue());
}
}
-
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/77de51df/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestPutS3Object.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestPutS3Object.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestPutS3Object.java
index de7816d..4dac0d4 100644
--- a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestPutS3Object.java
+++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestPutS3Object.java
@@ -17,7 +17,6 @@
package org.apache.nifi.processors.aws.s3;
import java.io.IOException;
-import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Map;
@@ -30,22 +29,22 @@ import org.junit.Test;
import com.amazonaws.services.s3.model.StorageClass;
@Ignore("For local testing only - interacts with S3 so the credentials file must be configured and all necessary buckets created")
-public class TestPutS3Object {
-
- private final String CREDENTIALS_FILE = System.getProperty("user.home") + "/aws-credentials.properties";
+public class TestPutS3Object extends AbstractS3Test {
@Test
public void testSimplePut() throws IOException {
final TestRunner runner = TestRunners.newTestRunner(new PutS3Object());
+
runner.setProperty(PutS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
- runner.setProperty(PutS3Object.BUCKET, "test-bucket-00000000-0000-0000-0000-123456789012");
- runner.setProperty(PutS3Object.EXPIRATION_RULE_ID, "Expire Quickly");
+ runner.setProperty(PutS3Object.REGION, REGION);
+ runner.setProperty(PutS3Object.BUCKET, BUCKET_NAME);
+
Assert.assertTrue(runner.setProperty("x-custom-prop", "hello").isValid());
for (int i = 0; i < 3; i++) {
final Map<String, String> attrs = new HashMap<>();
attrs.put("filename", String.valueOf(i) + ".txt");
- runner.enqueue(Paths.get("src/test/resources/hello.txt"), attrs);
+ runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME), attrs);
}
runner.run(3);
@@ -55,14 +54,17 @@ public class TestPutS3Object {
@Test
public void testPutInFolder() throws IOException {
final TestRunner runner = TestRunners.newTestRunner(new PutS3Object());
- runner.setProperty(PutS3Object.BUCKET, "test-bucket-00000000-0000-0000-0000-123456789012");
+
runner.setProperty(PutS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
- runner.setProperty(PutS3Object.EXPIRATION_RULE_ID, "Expire Quickly");
+ runner.setProperty(PutS3Object.REGION, REGION);
+ runner.setProperty(PutS3Object.BUCKET, BUCKET_NAME);
+
Assert.assertTrue(runner.setProperty("x-custom-prop", "hello").isValid());
final Map<String, String> attrs = new HashMap<>();
attrs.put("filename", "folder/1.txt");
- runner.enqueue(Paths.get("src/test/resources/hello.txt"), attrs);
+ runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME), attrs);
+
runner.run();
runner.assertAllFlowFilesTransferred(PutS3Object.REL_SUCCESS, 1);
@@ -71,14 +73,18 @@ public class TestPutS3Object {
@Test
public void testStorageClass() throws IOException {
final TestRunner runner = TestRunners.newTestRunner(new PutS3Object());
- runner.setProperty(PutS3Object.BUCKET, "test-bucket-00000000-0000-0000-0000-123456789012");
+
runner.setProperty(PutS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
+ runner.setProperty(PutS3Object.REGION, REGION);
+ runner.setProperty(PutS3Object.BUCKET, BUCKET_NAME);
runner.setProperty(PutS3Object.STORAGE_CLASS, StorageClass.ReducedRedundancy.name());
+
Assert.assertTrue(runner.setProperty("x-custom-prop", "hello").isValid());
final Map<String, String> attrs = new HashMap<>();
attrs.put("filename", "folder/2.txt");
- runner.enqueue(Paths.get("src/test/resources/hello.txt"), attrs);
+ runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME), attrs);
+
runner.run();
runner.assertAllFlowFilesTransferred(PutS3Object.REL_SUCCESS, 1);
@@ -87,16 +93,18 @@ public class TestPutS3Object {
@Test
public void testPermissions() throws IOException {
final TestRunner runner = TestRunners.newTestRunner(new PutS3Object());
- runner.setProperty(PutS3Object.BUCKET, "test-bucket-00000000-0000-0000-0000-123456789012");
+
runner.setProperty(PutS3Object.CREDENTAILS_FILE, CREDENTIALS_FILE);
- runner.setProperty(PutS3Object.FULL_CONTROL_USER_LIST, "28545acd76c35c7e91f8409b95fd1aa0c0914bfa1ac60975d9f48bc3c5e090b5");
+ runner.setProperty(PutS3Object.BUCKET, BUCKET_NAME);
+ runner.setProperty(PutS3Object.FULL_CONTROL_USER_LIST,"28545acd76c35c7e91f8409b95fd1aa0c0914bfa1ac60975d9f48bc3c5e090b5");
+ runner.setProperty(PutS3Object.REGION, REGION);
final Map<String, String> attrs = new HashMap<>();
attrs.put("filename", "folder/4.txt");
- runner.enqueue(Paths.get("src/test/resources/hello.txt"), attrs);
+ runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME), attrs);
+
runner.run();
runner.assertAllFlowFilesTransferred(PutS3Object.REL_SUCCESS, 1);
}
-
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/77de51df/nifi-nar-bundles/nifi-aws-bundle/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-aws-bundle/pom.xml b/nifi-nar-bundles/nifi-aws-bundle/pom.xml
index a43bb80..2470e41 100644
--- a/nifi-nar-bundles/nifi-aws-bundle/pom.xml
+++ b/nifi-nar-bundles/nifi-aws-bundle/pom.xml
@@ -30,14 +30,4 @@
<module>nifi-aws-nar</module>
</modules>
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk</artifactId>
- <version>1.9.24</version>
- </dependency>
- </dependencies>
- </dependencyManagement>
-
</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/77de51df/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ded7257..08a227f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -492,7 +492,7 @@
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
- <version>2.8</version>
+ <version>2.8.2</version>
</dependency>
<dependency>
<groupId>com.yammer.metrics</groupId>
@@ -968,6 +968,11 @@
<artifactId>derby</artifactId>
<version>10.11.1.1</version>
</dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk</artifactId>
+ <version>1.10.32</version>
+ </dependency>
</dependencies>
</dependencyManagement>
<dependencies>