You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by tu...@apache.org on 2026/03/09 14:21:28 UTC
(nifi) branch main updated: NIFI-15685 - ListS3 V1 pagination fails when delimiter is not set causing infinite loop for buckets with more than 1000 objects
This is an automated email from the ASF dual-hosted git repository.
turcsanyi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 75184260be1 NIFI-15685 - ListS3 V1 pagination fails when delimiter is not set causing infinite loop for buckets with more than 1000 objects
75184260be1 is described below
commit 75184260be137931c198690dcef32a971581c940
Author: Pierre Villard <pi...@gmail.com>
AuthorDate: Mon Mar 9 12:41:33 2026 +0100
NIFI-15685 - ListS3 V1 pagination fails when delimiter is not set causing infinite loop for buckets with more than 1000 objects
This closes #10983.
Signed-off-by: Peter Turcsanyi <tu...@apache.org>
---
.../org/apache/nifi/processors/aws/s3/ListS3.java | 10 ++-
.../apache/nifi/processors/aws/s3/TestListS3.java | 92 ++++++++++++++++++++++
2 files changed, 101 insertions(+), 1 deletion(-)
diff --git a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/ListS3.java b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/ListS3.java
index 3f6ec56f15b..b81aac8822b 100644
--- a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/ListS3.java
+++ b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/ListS3.java
@@ -900,7 +900,15 @@ public class ListS3 extends AbstractS3Processor implements VerifiableProcessor {
@Override
public void setNextMarker() {
- listObjectsRequestBuilder.marker(listObjectsResponse.nextMarker());
+ final String nextMarker = listObjectsResponse.nextMarker();
+ if (nextMarker != null) {
+ listObjectsRequestBuilder.marker(nextMarker);
+ } else {
+ final List<S3Object> contents = listObjectsResponse.contents();
+ if (!contents.isEmpty()) {
+ listObjectsRequestBuilder.marker(contents.get(contents.size() - 1).key());
+ }
+ }
}
@Override
diff --git a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestListS3.java b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestListS3.java
index 93fde82b0c4..6160c19db20 100644
--- a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestListS3.java
+++ b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestListS3.java
@@ -849,6 +849,98 @@ public class TestListS3 {
when(mockS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(response);
}
+ @Test
+ public void testListV1PaginationWithoutDelimiter() {
+ final List<String> firstPageKeys = List.of("key-0001", "key-0002", "key-0003");
+ final List<String> secondPageKeys = List.of("key-0004", "key-0005", "key-0006");
+ setupV1PaginationMocks(firstPageKeys, null, secondPageKeys, null);
+
+ runner.run();
+
+ final List<ListObjectsRequest> requests = captureListObjectsRequests(2);
+ assertNull(requests.get(0).marker());
+ assertEquals("key-0003", requests.get(1).marker());
+
+ runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 6);
+ }
+
+ @Test
+ public void testListV1PaginationWithDelimiter() {
+ runner.setProperty(ListS3.DELIMITER, "/");
+
+ final List<String> firstPageKeys = List.of("key-0001", "key-0002");
+ final List<String> secondPageKeys = List.of("key-0003", "key-0004");
+ setupV1PaginationMocks(firstPageKeys, "key-0002", secondPageKeys, null);
+
+ runner.run();
+
+ final List<ListObjectsRequest> requests = captureListObjectsRequests(2);
+ assertNull(requests.get(0).marker());
+ assertEquals("key-0002", requests.get(1).marker());
+
+ runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 4);
+ }
+
+ @Test
+ public void testListV1PaginationWithEmptyContentsAndNoNextMarker() {
+ final ListObjectsResponse firstPageResponse = ListObjectsResponse.builder()
+ .contents(Collections.emptyList())
+ .isTruncated(true)
+ .build();
+
+ final List<String> secondPageKeys = List.of("key-0001", "key-0002");
+ final ListObjectsResponse secondPageResponse = buildListObjectsResponse(secondPageKeys, false, null);
+
+ when(mockS3Client.listObjects(any(ListObjectsRequest.class)))
+ .thenReturn(firstPageResponse)
+ .thenReturn(secondPageResponse);
+
+ runner.setProperty(RegionUtil.REGION, "eu-west-1");
+ runner.setProperty(ListS3.BUCKET_WITHOUT_DEFAULT_VALUE, "test-bucket");
+ runner.setProperty(ListS3.LIST_TYPE, "1");
+
+ runner.run();
+
+ final List<ListObjectsRequest> requests = captureListObjectsRequests(2);
+ assertNull(requests.get(0).marker());
+ assertNull(requests.get(1).marker());
+
+ runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 2);
+ }
+
+ private void setupV1PaginationMocks(final List<String> firstPageKeys, final String firstPageNextMarker,
+ final List<String> secondPageKeys, final String secondPageNextMarker) {
+ runner.setProperty(RegionUtil.REGION, "eu-west-1");
+ runner.setProperty(ListS3.BUCKET_WITHOUT_DEFAULT_VALUE, "test-bucket");
+ runner.setProperty(ListS3.LIST_TYPE, "1");
+
+ final ListObjectsResponse firstPageResponse = buildListObjectsResponse(firstPageKeys, true, firstPageNextMarker);
+ final ListObjectsResponse secondPageResponse = buildListObjectsResponse(secondPageKeys, false, secondPageNextMarker);
+
+ when(mockS3Client.listObjects(any(ListObjectsRequest.class)))
+ .thenReturn(firstPageResponse)
+ .thenReturn(secondPageResponse);
+ }
+
+ private ListObjectsResponse buildListObjectsResponse(final List<String> keys, final boolean isTruncated, final String nextMarker) {
+ final Instant lastModified = Instant.now();
+ final List<S3Object> objects = keys.stream()
+ .map(key -> S3Object.builder().key(key).lastModified(lastModified).build())
+ .toList();
+
+ return ListObjectsResponse.builder()
+ .contents(objects)
+ .isTruncated(isTruncated)
+ .nextMarker(nextMarker)
+ .build();
+ }
+
+ private List<ListObjectsRequest> captureListObjectsRequests(final int expectedCount) {
+ final ArgumentCaptor<ListObjectsRequest> captureRequest = ArgumentCaptor.forClass(ListObjectsRequest.class);
+ verify(mockS3Client, Mockito.times(expectedCount)).listObjects(captureRequest.capture());
+ return captureRequest.getAllValues();
+ }
+
@Test
void testMigration() {
final PropertyMigrationResult propertyMigrationResult = runner.migrateProperties();