You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by da...@apache.org on 2025/07/30 06:52:50 UTC

(camel) 01/01: CAMEL-22291: camel-aws-s3: Consumer should use next marker if batch is truncated.

This is an automated email from the ASF dual-hosted git repository.

davsclaus pushed a commit to branch s3net
in repository https://gitbox.apache.org/repos/asf/camel.git

commit a6f8d6cec468eee7ca0dc18340270de4cb5d12e9
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Wed Jul 30 08:52:28 2025 +0200

    CAMEL-22291: camel-aws-s3: Consumer should use next marker if batch is truncated.
---
 .../camel/component/aws2/s3/AWS2S3Consumer.java    |  19 +++-
 .../S3ConsumerMaxMessagesPerPollIT.java            | 110 +++++++++++++++++++++
 2 files changed, 126 insertions(+), 3 deletions(-)

diff --git a/components/camel-aws/camel-aws2-s3/src/main/java/org/apache/camel/component/aws2/s3/AWS2S3Consumer.java b/components/camel-aws/camel-aws2-s3/src/main/java/org/apache/camel/component/aws2/s3/AWS2S3Consumer.java
index bc94c3e9262..7da4ead6ac2 100644
--- a/components/camel-aws/camel-aws2-s3/src/main/java/org/apache/camel/component/aws2/s3/AWS2S3Consumer.java
+++ b/components/camel-aws/camel-aws2-s3/src/main/java/org/apache/camel/component/aws2/s3/AWS2S3Consumer.java
@@ -132,8 +132,12 @@ public class AWS2S3Consumer extends ScheduledBatchPollingConsumer {
 
             ListObjectsRequest.Builder listObjectsRequest = ListObjectsRequest.builder();
             listObjectsRequest.bucket(bucketName);
-            listObjectsRequest.prefix(getConfiguration().getPrefix());
-            listObjectsRequest.delimiter(getConfiguration().getDelimiter());
+            if (ObjectHelper.isNotEmpty(getConfiguration().getPrefix())) {
+                listObjectsRequest.prefix(getConfiguration().getPrefix());
+            }
+            if (ObjectHelper.isNotEmpty(getConfiguration().getDelimiter())) {
+                listObjectsRequest.delimiter(getConfiguration().getDelimiter());
+            }
 
             if (maxMessagesPerPoll > 0) {
                 listObjectsRequest.maxKeys(maxMessagesPerPoll);
@@ -148,7 +152,16 @@ public class AWS2S3Consumer extends ScheduledBatchPollingConsumer {
             ListObjectsResponse listObjects = getAmazonS3Client().listObjects(listObjectsRequest.build());
 
             if (Boolean.TRUE.equals(listObjects.isTruncated())) {
-                marker = listObjects.nextMarker();
+                String next = listObjects.nextMarker();
+                if (next == null && listObjects.hasContents() && ObjectHelper.isEmpty(listObjects.prefix())) {
+                    // fallback to use last key from the returned list of objects
+                    int size = listObjects.contents().size();
+                    if (size > 0) {
+                        S3Object last = listObjects.contents().get(size - 1);
+                        next = last.key();
+                    }
+                }
+                marker = next;
                 LOG.trace("Returned list is truncated, so setting next marker: {}", marker);
             } else {
                 // no more data so clear marker
diff --git a/components/camel-aws/camel-aws2-s3/src/test/java/org/apache/camel/component/aws2/s3/integration/S3ConsumerMaxMessagesPerPollIT.java b/components/camel-aws/camel-aws2-s3/src/test/java/org/apache/camel/component/aws2/s3/integration/S3ConsumerMaxMessagesPerPollIT.java
new file mode 100644
index 00000000000..d342c7f051d
--- /dev/null
+++ b/components/camel-aws/camel-aws2-s3/src/test/java/org/apache/camel/component/aws2/s3/integration/S3ConsumerMaxMessagesPerPollIT.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.aws2.s3.integration;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.camel.EndpointInject;
+import org.apache.camel.Exchange;
+import org.apache.camel.Processor;
+import org.apache.camel.ProducerTemplate;
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.component.aws2.s3.AWS2S3Constants;
+import org.apache.camel.component.aws2.s3.AWS2S3Endpoint;
+import org.apache.camel.component.mock.MockEndpoint;
+import org.awaitility.Awaitility;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class S3ConsumerMaxMessagesPerPollIT extends Aws2S3Base {
+
+    @EndpointInject
+    private ProducerTemplate template;
+
+    @EndpointInject("mock:result")
+    private MockEndpoint result;
+
+    @Test
+    public void sendIn() throws Exception {
+        result.expectedMessageCount(3);
+
+        template.send("direct:putObject", new Processor() {
+
+            @Override
+            public void process(Exchange exchange) {
+                exchange.getIn().setHeader(AWS2S3Constants.KEY, "test.txt");
+                exchange.getIn().setBody("Test");
+            }
+        });
+
+        template.send("direct:putObject", new Processor() {
+
+            @Override
+            public void process(Exchange exchange) {
+                exchange.getIn().setHeader(AWS2S3Constants.KEY, "test1.txt");
+                exchange.getIn().setBody("Test1");
+            }
+        });
+
+        template.send("direct:putObject", new Processor() {
+
+            @Override
+            public void process(Exchange exchange) {
+                exchange.getIn().setHeader(AWS2S3Constants.KEY, "test2.txt");
+                exchange.getIn().setBody("Test2");
+            }
+        });
+
+        Awaitility.await().atMost(10, TimeUnit.SECONDS)
+                .untilAsserted(() -> MockEndpoint.assertIsSatisfied(context));
+
+        // in-progress should remove keys when complete
+        AWS2S3Endpoint s3 = (AWS2S3Endpoint) context.getRoute("s3consumer").getEndpoint();
+        Awaitility.await().atMost(5, TimeUnit.SECONDS)
+                .untilAsserted(() -> {
+                    Assertions.assertFalse(s3.getInProgressRepository().contains("test.txt"));
+                    Assertions.assertFalse(s3.getInProgressRepository().contains("test1.txt"));
+                    Assertions.assertFalse(s3.getInProgressRepository().contains("test2.txt"));
+                });
+    }
+
+    @Override
+    protected RouteBuilder createRouteBuilder() {
+        return new RouteBuilder() {
+            @Override
+            public void configure() {
+                String awsEndpoint = "aws2-s3://" + name.get() + "?autoCreateBucket=true";
+
+                from("direct:putObject").startupOrder(1).to(awsEndpoint);
+
+                from("aws2-s3://" + name.get()
+                     + "?maxMessagesPerPoll=2&moveAfterRead=true&destinationBucket=camel-kafka-connector&autoCreateBucket=true&destinationBucketPrefix=RAW(movedPrefix)&destinationBucketSuffix=RAW(movedSuffix)")
+                        .routeId("s3consumer")
+                        .startupOrder(2).log("${body}")
+                        .process(e -> {
+                            String key = e.getMessage().getHeader(AWS2S3Constants.KEY, String.class);
+                            log.info("Processing S3Object: {}", key);
+                            // should be in-progress
+                            AWS2S3Endpoint s3 = (AWS2S3Endpoint) context.getRoute("s3consumer").getEndpoint();
+                            Assertions.assertTrue(s3.getInProgressRepository().contains(key));
+                        })
+                        .to("mock:result");
+
+            }
+        };
+    }
+}