You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by ac...@apache.org on 2021/04/13 14:50:26 UTC
[camel] branch master updated: CAMEL-16495 - Camel-AWS-S3 -
Streaming upload: While restarting the listObject operation should be
paginated
This is an automated email from the ASF dual-hosted git repository.
acosentino pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/master by this push:
new eaa2d59 CAMEL-16495 - Camel-AWS-S3 - Streaming upload: While restarting the listObject operation should be paginated
eaa2d59 is described below
commit eaa2d595f7c6baaa4447abbee38428ebd4997d1e
Author: Andrea Cosentino <an...@gmail.com>
AuthorDate: Tue Apr 13 16:49:34 2021 +0200
CAMEL-16495 - Camel-AWS-S3 - Streaming upload: While restarting the listObject operation should be paginated
---
.../aws2/s3/stream/AWS2S3StreamUploadProducer.java | 21 +++++----------------
1 file changed, 5 insertions(+), 16 deletions(-)
diff --git a/components/camel-aws/camel-aws2-s3/src/main/java/org/apache/camel/component/aws2/s3/stream/AWS2S3StreamUploadProducer.java b/components/camel-aws/camel-aws2-s3/src/main/java/org/apache/camel/component/aws2/s3/stream/AWS2S3StreamUploadProducer.java
index e1121fc..934e734 100644
--- a/components/camel-aws/camel-aws2-s3/src/main/java/org/apache/camel/component/aws2/s3/stream/AWS2S3StreamUploadProducer.java
+++ b/components/camel-aws/camel-aws2-s3/src/main/java/org/apache/camel/component/aws2/s3/stream/AWS2S3StreamUploadProducer.java
@@ -48,11 +48,11 @@ import software.amazon.awssdk.services.s3.model.CompletedPart;
import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
-import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
import software.amazon.awssdk.services.s3.model.ObjectCannedACL;
import software.amazon.awssdk.services.s3.model.S3Object;
import software.amazon.awssdk.services.s3.model.ServerSideEncryption;
import software.amazon.awssdk.services.s3.model.UploadPartRequest;
+import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable;
import software.amazon.awssdk.utils.IoUtils;
/**
@@ -290,21 +290,10 @@ public class AWS2S3StreamUploadProducer extends DefaultProducer {
ArrayList<S3Object> list = new ArrayList<>();
ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(getConfiguration().getBucketName())
.prefix(AWS2S3Utils.determineFileName(getConfiguration().getKeyName())).build();
- boolean done = false;
- while (!done) {
- ListObjectsV2Response listObjResponse = getEndpoint().getS3Client().listObjectsV2(request);
- for (S3Object content : listObjResponse.contents()) {
- list.add(content);
- }
-
- if (listObjResponse.nextContinuationToken() == null) {
- done = true;
- }
-
- request = request.toBuilder()
- .continuationToken(listObjResponse.nextContinuationToken())
- .build();
- }
+ ListObjectsV2Iterable listRes = getEndpoint().getS3Client().listObjectsV2Paginator(request);
+ listRes.stream()
+ .flatMap(r -> r.contents().stream())
+ .forEach(content -> list.add(content));
if (list.size() > 0) {
list.sort(Comparator.comparing(S3Object::lastModified));
int listSize = list.size();