You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by an...@apache.org on 2025/05/30 02:49:00 UTC
(hadoop) branch branch-3.4 updated: HADOOP-19580. [ABFS][BugFix][Backport 3.4] IsNonEmptyDirectory Check should loop on listing using updated continuation token (#7719)
This is an automated email from the ASF dual-hosted git repository.
anujmodi pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new 83dffcc1ab8 HADOOP-19580. [ABFS][BugFix][Backport 3.4] IsNonEmptyDirectory Check should loop on listing using updated continuation token (#7719)
83dffcc1ab8 is described below
commit 83dffcc1ab8f4a4dd983157fc88237a4051d7fca
Author: Anuj Modi <an...@gmail.com>
AuthorDate: Fri May 30 08:18:51 2025 +0530
HADOOP-19580. [ABFS][BugFix][Backport 3.4] IsNonEmptyDirectory Check should loop on listing using updated continuation token (#7719)
Contributed by Anuj Modi
Reviewed by Sneha Vijayarajan
---
.../fs/azurebfs/services/AbfsBlobClient.java | 2 +-
.../ITestAzureBlobFileSystemListStatus.java | 44 ++++++++++++++++++----
.../fs/azurebfs/services/ITestAbfsClient.java | 42 +++++++++++++++++++--
3 files changed, 76 insertions(+), 12 deletions(-)
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java
index 5d8c2efb94e..a0567da97b2 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java
@@ -2037,7 +2037,7 @@ public boolean isNonEmptyDirectory(String path,
List<FileStatus> fileStatusList = new ArrayList<>();
// We need to loop on continuation token until we get an entry or continuation token becomes null.
do {
- ListResponseData listResponseData = listPath(path, false, 1, null, tracingContext, null);
+ ListResponseData listResponseData = listPath(path, false, 1, continuationToken, tracingContext, null);
fileStatusList.addAll(listResponseData.getFileStatusList());
continuationToken = listResponseData.getContinuationToken();
} while (StringUtils.isNotEmpty(continuationToken) && fileStatusList.isEmpty());
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java
index a533f352a97..2959a321994 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java
@@ -391,9 +391,9 @@ public void testEmptyListingInSubsequentCall() throws IOException {
true, 2, 0);
testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN, true, EMPTY_STRING,
false, 2, 1);
- testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN, true, TEST_CONTINUATION_TOKEN,
+ testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN + 1, true, TEST_CONTINUATION_TOKEN + 2,
true, 3, 0);
- testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN, true, TEST_CONTINUATION_TOKEN,
+ testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN + 1, true, TEST_CONTINUATION_TOKEN + 2,
false, 3, 1);
testEmptyListingInSubsequentCallInternal(EMPTY_STRING, false, EMPTY_STRING,
@@ -409,9 +409,9 @@ public void testEmptyListingInSubsequentCall() throws IOException {
true, 2, 1);
testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN, false, EMPTY_STRING,
false, 2, 2);
- testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN, false, TEST_CONTINUATION_TOKEN,
+ testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN + 1, false, TEST_CONTINUATION_TOKEN + 2,
true, 3, 1);
- testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN, false, TEST_CONTINUATION_TOKEN,
+ testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN + 1, false, TEST_CONTINUATION_TOKEN + 2,
false, 3, 2);
}
@@ -453,9 +453,23 @@ private void testEmptyListingInSubsequentCallInternal(String firstCT,
listResponseData3.setFileStatusList(new ArrayList<>());
listResponseData3.setOp(Mockito.mock(AbfsRestOperation.class));
- Mockito.doReturn(listResponseData1).doReturn(listResponseData2).doReturn(listResponseData3)
- .when(spiedClient).listPath(eq("/testPath"), eq(false), eq(1),
- any(), any(), any());
+ final int[] itr = new int[1];
+ final String[] continuationTokenUsed = new String[3];
+
+ Mockito.doAnswer(invocationOnMock -> {
+ if (itr[0] == 0) {
+ itr[0]++;
+ continuationTokenUsed[0] = invocationOnMock.getArgument(3);
+ return listResponseData1;
+ } else if (itr[0] == 1) {
+ itr[0]++;
+ continuationTokenUsed[1] = invocationOnMock.getArgument(3);
+ return listResponseData2;
+ }
+ continuationTokenUsed[2] = invocationOnMock.getArgument(3);
+ return listResponseData3;
+ }).when(spiedClient).listPath(eq("/testPath"), eq(false),
+ eq(1), any(), any(TracingContext.class), any());
FileStatus[] list = spiedFs.listStatus(new Path("/testPath"));
@@ -473,6 +487,22 @@ private void testEmptyListingInSubsequentCallInternal(String firstCT,
Mockito.verify(spiedClient, times(0))
.getPathStatus(eq("/testPath"), any(), eq(null), eq(false));
}
+
+ Assertions.assertThat(continuationTokenUsed[0])
+ .describedAs("First continuation token used is not as expected")
+ .isNull();
+
+ if (expectedInvocations > 1) {
+ Assertions.assertThat(continuationTokenUsed[1])
+ .describedAs("Second continuation token used is not as expected")
+ .isEqualTo(firstCT);
+ }
+
+ if (expectedInvocations > 2) {
+ Assertions.assertThat(continuationTokenUsed[2])
+ .describedAs("Third continuation token used is not as expected")
+ .isEqualTo(secondCT);
+ }
}
/**
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClient.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClient.java
index 1083df3ba57..cba1643478f 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClient.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClient.java
@@ -745,18 +745,18 @@ public void testIsNonEmptyDirectory() throws IOException {
true, 2, false);
testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN, true, EMPTY_STRING,
false, 2, true);
- testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN, true, TEST_CONTINUATION_TOKEN,
+ testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN + 1, true, TEST_CONTINUATION_TOKEN + 2,
true, 3, false);
- testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN, true, TEST_CONTINUATION_TOKEN,
+ testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN + 1, true, TEST_CONTINUATION_TOKEN + 2,
false, 2, true);
testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN, false, EMPTY_STRING,
true, 1, true);
testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN, false, EMPTY_STRING,
false, 1, true);
- testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN, false, TEST_CONTINUATION_TOKEN,
+ testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN + 1, false, TEST_CONTINUATION_TOKEN + 2,
true, 1, true);
- testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN, false, TEST_CONTINUATION_TOKEN,
+ testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN + 1, false, TEST_CONTINUATION_TOKEN + 2,
false, 1, true);
}
@@ -801,11 +801,45 @@ private void testIsNonEmptyDirectoryInternal(String firstCT,
.when(spiedClient).listPath(eq("/testPath"), eq(false), eq(1),
any(), any(), any());
+ final int[] itr = new int[1];
+ final String[] continuationTokenUsed = new String[3];
+
+ Mockito.doAnswer(invocationOnMock -> {
+ if (itr[0] == 0) {
+ itr[0]++;
+ continuationTokenUsed[0] = invocationOnMock.getArgument(3);
+ return listResponseData1;
+ } else if (itr[0] == 1) {
+ itr[0]++;
+ continuationTokenUsed[1] = invocationOnMock.getArgument(3);
+ return listResponseData2;
+ }
+ continuationTokenUsed[2] = invocationOnMock.getArgument(3);
+ return listResponseData3;
+ }).when(spiedClient).listPath(eq("/testPath"), eq(false), eq(1),
+ any(), any(TracingContext.class), any());
+
Assertions.assertThat(spiedClient.isNonEmptyDirectory("/testPath",
Mockito.mock(TracingContext.class)))
.describedAs("isNonEmptyDirectory in client giving unexpected results")
.isEqualTo(isNonEmpty);
+ Assertions.assertThat(continuationTokenUsed[0])
+ .describedAs("First continuation token used is not as expected")
+ .isNull();
+
+ if (expectedInvocations > 1) {
+ Assertions.assertThat(continuationTokenUsed[1])
+ .describedAs("Second continuation token used is not as expected")
+ .isEqualTo(firstCT);
+ }
+
+ if (expectedInvocations > 2) {
+ Assertions.assertThat(continuationTokenUsed[2])
+ .describedAs("Third continuation token used is not as expected")
+ .isEqualTo(secondCT);
+ }
+
Mockito.verify(spiedClient, times(expectedInvocations))
.listPath(eq("/testPath"), eq(false), eq(1),
any(), any(TracingContext.class), any());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org