You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by mf...@apache.org on 2014/09/24 19:46:40 UTC
[1/2] git commit: STREAMS-166 | Modified the S3PeristReader to check
for object summaries in the listing response. If there are any object
summaries then we can assume that the path is pointing to a directory and we
need to iterate through all files that
Repository: incubator-streams
Updated Branches:
refs/heads/master 8c45159b0 -> a9f1b2fed
STREAMS-166 | Modified the S3PeristReader to check for object summaries in the listing response. If there are any object summaries then we can assume that the path is pointing to a directory and we need to iterate through all files that it contains.
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/8fb07eb2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/8fb07eb2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/8fb07eb2
Branch: refs/heads/master
Commit: 8fb07eb2f3ebfae42f612ad169c06115914d8a4f
Parents: 3ee5175
Author: Robert Douglas <rd...@w2odigital.com>
Authored: Tue Sep 9 15:32:50 2014 -0500
Committer: Robert Douglas <rd...@w2odigital.com>
Committed: Tue Sep 9 15:32:50 2014 -0500
----------------------------------------------------------------------
.../org/apache/streams/s3/S3PersistReader.java | 19 +++++++++++++++----
1 file changed, 15 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/8fb07eb2/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3PersistReader.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3PersistReader.java b/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3PersistReader.java
index 1123dbb..4f62a06 100644
--- a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3PersistReader.java
+++ b/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3PersistReader.java
@@ -25,6 +25,7 @@ import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.S3ClientOptions;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Queues;
import org.apache.streams.core.*;
@@ -116,7 +117,7 @@ public class S3PersistReader implements StreamsPersistReader, DatumStatusCountab
final ListObjectsRequest request = new ListObjectsRequest()
.withBucketName(this.s3ReaderConfiguration.getBucket())
.withPrefix(s3ReaderConfiguration.getReaderPath())
- .withMaxKeys(50);
+ .withMaxKeys(500);
ObjectListing listing = this.amazonS3Client.listObjects(request);
@@ -128,12 +129,22 @@ public class S3PersistReader implements StreamsPersistReader, DatumStatusCountab
* if you cannot list files that are in this path, then you are most likely dealing with
* a simple file.
*/
- if(listing.getCommonPrefixes().size() > 0) {
+ boolean hasCommonPrefixes = listing.getCommonPrefixes().size() > 0 ? true : false;
+ boolean hasObjectSummaries = listing.getObjectSummaries().size() > 0 ? true : false;
+
+ if(hasCommonPrefixes || hasObjectSummaries) {
// Handle the 'directory' use case
do
{
- for (String file : listing.getCommonPrefixes())
- this.files.add(file);
+ if(hasCommonPrefixes) {
+ for (String file : listing.getCommonPrefixes()) {
+ this.files.add(file);
+ }
+ } else {
+ for(final S3ObjectSummary objectSummary : listing.getObjectSummaries()) {
+ this.files.add(objectSummary.getKey());
+ }
+ }
// get the next batch.
listing = this.amazonS3Client.listNextBatchOfObjects(listing);
[2/2] git commit: Merge PR#80 from 'robdouglas/STREAMS-166'
Posted by mf...@apache.org.
Merge PR#80 from 'robdouglas/STREAMS-166'
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/a9f1b2fe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/a9f1b2fe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/a9f1b2fe
Branch: refs/heads/master
Commit: a9f1b2fed2c3a67b8cb8cd3887153d9cf171128b
Parents: 8c45159 8fb07eb
Author: Matt Franklin <mf...@apache.org>
Authored: Wed Sep 24 13:35:05 2014 -0400
Committer: Matt Franklin <mf...@apache.org>
Committed: Wed Sep 24 13:35:05 2014 -0400
----------------------------------------------------------------------
.../org/apache/streams/s3/S3PersistReader.java | 19 +++++++++++++++----
1 file changed, 15 insertions(+), 4 deletions(-)
----------------------------------------------------------------------