(hudi) branch master updated: perf: Reduce memory fetched to driver for getAllPartitions API by removing unused objects (#17947)

[si...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

sivabalan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new 9db8dcf7205f perf: Reduce memory fetched to driver for getAllPartitions API by removing unused objects (#17947)
9db8dcf7205f is described below

commit 9db8dcf7205f455a90c084e07c5b90f918fa9c33
Author: Surya Prasanna <sy...@uber.com>
AuthorDate: Wed Jan 28 17:38:37 2026 -0800

    perf: Reduce memory fetched to driver for getAllPartitions API by removing unused objects (#17947)
---
 .../hudi/metadata/FileSystemBackedTableMetadata.java    | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/FileSystemBackedTableMetadata.java b/hudi-common/src/main/java/org/apache/hudi/metadata/FileSystemBackedTableMetadata.java
index f01b926522cd..5b1878ccd53e 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/FileSystemBackedTableMetadata.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/FileSystemBackedTableMetadata.java
@@ -164,9 +164,10 @@ public class FileSystemBackedTableMetadata extends AbstractHoodieTableMetadata {
           "Listing all partitions on " + this.tableName
               + " with prefix " + relativePathPrefix);
       // Need to use serializable file status here, see HUDI-5936
-      List<StoragePathInfo> dirToFileListing = engineContext.flatMap(pathsToList, path -> {
+      List<Pair<StoragePath, Boolean>> dirToFileListingPairs = engineContext.flatMap(pathsToList, path -> {
         try {
-          return getStorage().listDirectEntries(path).stream();
+          return getStorage().listDirectEntries(path).stream()
+              .map(storagePathInfo -> Pair.of(storagePathInfo.getPath(), storagePathInfo.isDirectory()));
         } catch (FileNotFoundException e) {
           // The partition may have been cleaned.
           return Stream.empty();
@@ -177,16 +178,16 @@ public class FileSystemBackedTableMetadata extends AbstractHoodieTableMetadata {
 
       // if current dictionary contains PartitionMetadata, add it to result
       // if current dictionary does not contain PartitionMetadata, add it to queue to be processed.
-      int fileListingParallelism = Math.min(DEFAULT_LISTING_PARALLELISM, dirToFileListing.size());
-      if (!dirToFileListing.isEmpty()) {
+      int fileListingParallelism = Math.min(DEFAULT_LISTING_PARALLELISM, dirToFileListingPairs.size());
+      if (!dirToFileListingPairs.isEmpty()) {
         // result below holds a list of pair. first entry in the pair optionally holds the deduced list of partitions.
         // and second entry holds optionally a directory path to be processed further.
         engineContext.setJobStatus(this.getClass().getSimpleName(), "Processing listed partitions");
         List<Pair<Option<String>, Option<StoragePath>>> result =
-            engineContext.map(dirToFileListing,
-                fileInfo -> {
-                  StoragePath path = fileInfo.getPath();
-                  if (fileInfo.isDirectory()) {
+            engineContext.map(dirToFileListingPairs,
+                fileInfoPair -> {
+                  StoragePath path = fileInfoPair.getKey();
+                  if (fileInfoPair.getValue()) {
                     if (HoodiePartitionMetadata.hasPartitionMetadata(getStorage(), path)) {
                       return Pair.of(
                           Option.of(FSUtils.getRelativePartitionPath(dataBasePath,