You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by zh...@apache.org on 2024/05/28 01:43:42 UTC

(flink) 02/03: [FLINK-33892][runtime] Support JobMaster fetch and retain partition metrics on TaskManagers during job recovery.

This is an automated email from the ASF dual-hosted git repository.

zhuzh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 3206659db5b7c4ce645072f11f091e0e9e92b0ce
Author: JunRuiLee <jr...@gmail.com>
AuthorDate: Sat May 11 12:24:16 2024 +0800

    [FLINK-33892][runtime] Support JobMaster fetch and retain partition metrics on TaskManagers during job recovery.
---
 .../apache/flink/runtime/jobmaster/JobMaster.java  | 197 ++++++++++++++----
 .../flink/runtime/jobmaster/JobMasterGateway.java  |  22 +-
 .../flink/runtime/shuffle/JobShuffleContext.java   |  18 +-
 .../runtime/shuffle/JobShuffleContextImpl.java     |  13 +-
 .../flink/runtime/shuffle/NettyShuffleMaster.java  |  13 +-
 .../flink/runtime/shuffle/ShuffleMaster.java       |  24 ++-
 .../flink/runtime/jobmaster/JobMasterTest.java     | 231 +++++++++++++++++----
 .../jobmaster/utils/TestingJobMasterGateway.java   |  25 ++-
 .../utils/TestingJobMasterGatewayBuilder.java      |  23 +-
 9 files changed, 469 insertions(+), 97 deletions(-)

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
index 859caa53920..ee5765d198f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
@@ -18,6 +18,7 @@
 
 package org.apache.flink.runtime.jobmaster;
 
+import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.api.common.JobStatus;
 import org.apache.flink.api.common.functions.AggregateFunction;
@@ -91,6 +92,7 @@ import org.apache.flink.runtime.scheduler.SchedulerNG;
 import org.apache.flink.runtime.shuffle.JobShuffleContext;
 import org.apache.flink.runtime.shuffle.JobShuffleContextImpl;
 import org.apache.flink.runtime.shuffle.PartitionWithMetrics;
+import org.apache.flink.runtime.shuffle.ShuffleDescriptor;
 import org.apache.flink.runtime.shuffle.ShuffleMaster;
 import org.apache.flink.runtime.slots.ResourceRequirement;
 import org.apache.flink.runtime.state.KeyGroupRange;
@@ -116,11 +118,11 @@ import javax.annotation.Nullable;
 
 import java.io.IOException;
 import java.net.InetSocketAddress;
+import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
@@ -130,6 +132,7 @@ import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
 import java.util.concurrent.Executor;
 import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.stream.Collectors;
 
@@ -229,6 +232,22 @@ public class JobMaster extends FencedRpcEndpoint<JobMasterId>
 
     private final BlocklistHandler blocklistHandler;
 
+    private final Map<ResultPartitionID, PartitionWithMetrics> fetchedPartitionsWithMetrics =
+            new HashMap<>();
+
+    /**
+     * A flag that indicates whether to fetch and retain partitions on task managers. This will
+     * apply to future TaskManager registrations as well as already registered TaskManagers. The
+     * flag will be set to true when starting batch job recovery and set to false after all required
+     * partitions, as defined in {@code requireToFetchPartitions}, are either fetched or when a
+     * timeout occurs.
+     */
+    private boolean fetchAndRetainPartitions = false;
+
+    private Set<ResultPartitionID> partitionsToFetch;
+
+    private CompletableFuture<Collection<PartitionWithMetrics>> fetchPartitionsFuture;
+
     // ------------------------------------------------------------------------
 
     public JobMaster(
@@ -797,30 +816,41 @@ public class JobMaster extends FencedRpcEndpoint<JobMasterId>
             }
         }
 
-        return getRpcService()
-                .connect(
-                        taskManagerRegistrationInformation.getTaskManagerRpcAddress(),
-                        TaskExecutorGateway.class)
-                .handleAsync(
-                        (TaskExecutorGateway taskExecutorGateway, Throwable throwable) -> {
-                            if (throwable != null) {
-                                return new RegistrationResponse.Failure(throwable);
-                            }
-
-                            slotPoolService.registerTaskManager(taskManagerId);
-                            registeredTaskManagers.put(
-                                    taskManagerId,
-                                    TaskManagerRegistration.create(
-                                            taskManagerLocation, taskExecutorGateway, sessionId));
-
-                            // monitor the task manager as heartbeat target
-                            taskManagerHeartbeatManager.monitorTarget(
-                                    taskManagerId,
-                                    new TaskExecutorHeartbeatSender(taskExecutorGateway));
+        CompletableFuture<RegistrationResponse> registrationResponseFuture =
+                getRpcService()
+                        .connect(
+                                taskManagerRegistrationInformation.getTaskManagerRpcAddress(),
+                                TaskExecutorGateway.class)
+                        .handleAsync(
+                                (TaskExecutorGateway taskExecutorGateway, Throwable throwable) -> {
+                                    if (throwable != null) {
+                                        return new RegistrationResponse.Failure(throwable);
+                                    }
+
+                                    slotPoolService.registerTaskManager(taskManagerId);
+                                    registeredTaskManagers.put(
+                                            taskManagerId,
+                                            TaskManagerRegistration.create(
+                                                    taskManagerLocation,
+                                                    taskExecutorGateway,
+                                                    sessionId));
+
+                                    // monitor the task manager as heartbeat target
+                                    taskManagerHeartbeatManager.monitorTarget(
+                                            taskManagerId,
+                                            new TaskExecutorHeartbeatSender(taskExecutorGateway));
+
+                                    return new JMTMRegistrationSuccess(resourceId);
+                                },
+                                getMainThreadExecutor());
+
+        if (fetchAndRetainPartitions) {
+            registrationResponseFuture.whenComplete(
+                    (ignored, throwable) ->
+                            fetchAndRetainPartitionWithMetricsOnTaskManager(taskManagerId));
+        }
 
-                            return new JMTMRegistrationSuccess(resourceId);
-                        },
-                        getMainThreadExecutor());
+        return registrationResponseFuture;
     }
 
     @Nonnull
@@ -961,25 +991,108 @@ public class JobMaster extends FencedRpcEndpoint<JobMasterId>
     }
 
     @Override
-    public CompletableFuture<Collection<PartitionWithMetrics>>
-            getAllPartitionWithMetricsOnTaskManagers() {
-        final List<CompletableFuture<Collection<PartitionWithMetrics>>> allFutures =
-                new ArrayList<>();
+    public CompletableFuture<Collection<PartitionWithMetrics>> getPartitionWithMetrics(
+            Duration timeout, Set<ResultPartitionID> expectedPartitions) {
+        this.partitionsToFetch = expectedPartitions;
+        this.fetchPartitionsFuture = new CompletableFuture<>();
+
+        // check already fetched partitions
+        checkPartitionOnTaskManagerReportFinished();
+
+        return FutureUtils.orTimeout(
+                        fetchPartitionsFuture, timeout.toMillis(), TimeUnit.MILLISECONDS, null)
+                .handleAsync(
+                        (metrics, throwable) -> {
+                            stopFetchAndRetainPartitionWithMetricsOnTaskManager();
+
+                            if (throwable != null) {
+                                if (throwable instanceof TimeoutException) {
+                                    log.warn(
+                                            "Timeout occurred after {} ms "
+                                                    + "while fetching partition(s) ({}) from task managers.",
+                                            timeout.toMillis(),
+                                            expectedPartitions);
+
+                                    return new ArrayList<>(fetchedPartitionsWithMetrics.values());
+                                }
+                                throw new CompletionException(throwable);
+                            }
+
+                            return new ArrayList<>(fetchedPartitionsWithMetrics.values());
+                        },
+                        getMainThreadExecutor());
+    }
+
+    @VisibleForTesting
+    Map<ResultPartitionID, PartitionWithMetrics> getPartitionWithMetricsOnTaskManagers() {
+        return fetchedPartitionsWithMetrics;
+    }
+
+    @Override
+    public void startFetchAndRetainPartitionWithMetricsOnTaskManager() {
+        fetchAndRetainPartitions = true;
+
+        // process all already registered task managers
         registeredTaskManagers
-                .values()
-                .forEach(
-                        taskManager ->
-                                allFutures.add(
-                                        taskManager
-                                                .getTaskExecutorGateway()
-                                                .getAndRetainPartitionWithMetrics(
-                                                        jobGraph.getJobID())));
-        return FutureUtils.combineAll(allFutures)
-                .thenApply(
-                        partitions ->
-                                partitions.stream()
-                                        .flatMap(Collection::stream)
-                                        .collect(Collectors.toList()));
+                .keySet()
+                .forEach(this::fetchAndRetainPartitionWithMetricsOnTaskManager);
+    }
+
+    private void fetchAndRetainPartitionWithMetricsOnTaskManager(ResourceID resourceId) {
+        TaskManagerRegistration taskManager = registeredTaskManagers.get(resourceId);
+        checkNotNull(taskManager);
+
+        taskManager
+                .getTaskExecutorGateway()
+                .getAndRetainPartitionWithMetrics(jobGraph.getJobID())
+                .thenAccept(
+                        partitionWithMetrics -> {
+                            if (fetchAndRetainPartitions) {
+                                for (PartitionWithMetrics partitionWithMetric :
+                                        partitionWithMetrics) {
+                                    log.debug(
+                                            "Received partition metrics for {} from Task Manager {}.",
+                                            partitionWithMetric
+                                                    .getPartition()
+                                                    .getResultPartitionID(),
+                                            resourceId);
+                                    fetchedPartitionsWithMetrics.put(
+                                            partitionWithMetric
+                                                    .getPartition()
+                                                    .getResultPartitionID(),
+                                            partitionWithMetric);
+                                }
+                                checkPartitionOnTaskManagerReportFinished();
+                            } else {
+                                log.info(
+                                        "Received late report of partition metrics from {}. Release the partitions.",
+                                        resourceId);
+
+                                taskManager
+                                        .getTaskExecutorGateway()
+                                        .releasePartitions(
+                                                jobGraph.getJobID(),
+                                                partitionWithMetrics.stream()
+                                                        .map(PartitionWithMetrics::getPartition)
+                                                        .map(
+                                                                ShuffleDescriptor
+                                                                        ::getResultPartitionID)
+                                                        .collect(Collectors.toSet()));
+                            }
+                        });
+    }
+
+    private void stopFetchAndRetainPartitionWithMetricsOnTaskManager() {
+        fetchAndRetainPartitions = false;
+    }
+
+    private void checkPartitionOnTaskManagerReportFinished() {
+        if (fetchPartitionsFuture != null) {
+            if (fetchedPartitionsWithMetrics.keySet().containsAll(partitionsToFetch)
+                    && !fetchPartitionsFuture.isDone()) {
+                fetchPartitionsFuture.complete(fetchedPartitionsWithMetrics.values());
+            }
+        }
     }
 
     @Override
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
index 994492807ff..38772fbadae 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
@@ -55,8 +55,10 @@ import org.apache.flink.util.SerializedValue;
 
 import javax.annotation.Nullable;
 
+import java.time.Duration;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 
 /** {@link JobMaster} rpc gateway interface. */
@@ -303,11 +305,27 @@ public interface JobMasterGateway
     CompletableFuture<?> stopTrackingAndReleasePartitions(
             Collection<ResultPartitionID> partitionIds);
 
-    default CompletableFuture<Collection<PartitionWithMetrics>>
-            getAllPartitionWithMetricsOnTaskManagers() {
+    /**
+     * Get specified partitions and their metrics (identified by {@code expectedPartitions}), the
+     * metrics include sizes of sub-partitions in a result partition.
+     *
+     * @param timeout The timeout used for retrieve the specified partitions.
+     * @param expectedPartitions The set of identifiers for the result partitions whose metrics are
+     *     to be fetched.
+     * @return A future will contain a collection of the partitions with their metrics that could be
+     *     retrieved from the expected partitions within the specified timeout period.
+     */
+    default CompletableFuture<Collection<PartitionWithMetrics>> getPartitionWithMetrics(
+            Duration timeout, Set<ResultPartitionID> expectedPartitions) {
         return CompletableFuture.completedFuture(Collections.emptyList());
     }
 
+    /**
+     * Notify jobMaster to fetch and retain partitions on task managers. It will process for future
+     * TaskManager registrations and already registered TaskManagers.
+     */
+    default void startFetchAndRetainPartitionWithMetricsOnTaskManager() {}
+
     /**
      * Read current {@link JobResourceRequirements job resource requirements}.
      *
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/JobShuffleContext.java b/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/JobShuffleContext.java
index 585b73806d9..5cd59046381 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/JobShuffleContext.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/JobShuffleContext.java
@@ -21,7 +21,9 @@ package org.apache.flink.runtime.shuffle;
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
 
+import java.time.Duration;
 import java.util.Collection;
+import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 
 /**
@@ -41,8 +43,18 @@ public interface JobShuffleContext {
             Collection<ResultPartitionID> partitionIds);
 
     /**
-     * Retrieves a collection containing descriptions and metrics of existing result partitions from
-     * all TaskManagers.
+     * Retrieves specified partitions and their metrics (identified by {@code expectedPartitions}),
+     * the metrics include sizes of sub-partitions in a result partition.
+     *
+     * @param timeout The timeout used for retrieve the specified partitions.
+     * @param expectedPartitions The set of identifiers for the result partitions whose metrics are
+     *     to be fetched.
+     * @return A future will contain a collection of the partitions with their metrics that could be
+     *     retrieved from the expected partitions within the specified timeout period.
      */
-    CompletableFuture<Collection<PartitionWithMetrics>> getAllPartitionWithMetricsOnTaskManagers();
+    CompletableFuture<Collection<PartitionWithMetrics>> getPartitionWithMetrics(
+            Duration timeout, Set<ResultPartitionID> expectedPartitions);
+
+    /** Notifies that the recovery process of result partitions has started. */
+    void notifyPartitionRecoveryStarted();
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/JobShuffleContextImpl.java b/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/JobShuffleContextImpl.java
index 546cc3025b4..51fd013788c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/JobShuffleContextImpl.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/JobShuffleContextImpl.java
@@ -22,7 +22,9 @@ import org.apache.flink.api.common.JobID;
 import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
 import org.apache.flink.runtime.jobmaster.JobMasterGateway;
 
+import java.time.Duration;
 import java.util.Collection;
+import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 
 import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -51,8 +53,13 @@ public class JobShuffleContextImpl implements JobShuffleContext {
     }
 
     @Override
-    public CompletableFuture<Collection<PartitionWithMetrics>>
-            getAllPartitionWithMetricsOnTaskManagers() {
-        return jobMasterGateway.getAllPartitionWithMetricsOnTaskManagers();
+    public CompletableFuture<Collection<PartitionWithMetrics>> getPartitionWithMetrics(
+            Duration timeout, Set<ResultPartitionID> expectedPartitions) {
+        return jobMasterGateway.getPartitionWithMetrics(timeout, expectedPartitions);
+    }
+
+    @Override
+    public void notifyPartitionRecoveryStarted() {
+        jobMasterGateway.startFetchAndRetainPartitionWithMetricsOnTaskManager();
     }
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/NettyShuffleMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/NettyShuffleMaster.java
index 461457a29b6..df544fdce47 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/NettyShuffleMaster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/NettyShuffleMaster.java
@@ -31,10 +31,12 @@ import org.apache.flink.runtime.util.ConfigurationParserUtils;
 
 import javax.annotation.Nullable;
 
+import java.time.Duration;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 
 import static org.apache.flink.api.common.BatchShuffleMode.ALL_EXCHANGES_HYBRID_FULL;
@@ -172,10 +174,10 @@ public class NettyShuffleMaster implements ShuffleMaster<NettyShuffleDescriptor>
     }
 
     @Override
-    public CompletableFuture<Collection<PartitionWithMetrics>> getAllPartitionWithMetrics(
-            JobID jobId) {
+    public CompletableFuture<Collection<PartitionWithMetrics>> getPartitionWithMetrics(
+            JobID jobId, Duration timeout, Set<ResultPartitionID> expectedPartitions) {
         return checkNotNull(jobShuffleContexts.get(jobId))
-                .getAllPartitionWithMetricsOnTaskManagers();
+                .getPartitionWithMetrics(timeout, expectedPartitions);
     }
 
     @Override
@@ -199,4 +201,9 @@ public class NettyShuffleMaster implements ShuffleMaster<NettyShuffleDescriptor>
             ShuffleMasterSnapshotContext context) {
         snapshotFuture.complete(EmptyShuffleMasterSnapshot.getInstance());
     }
+
+    @Override
+    public void notifyPartitionRecoveryStarted(JobID jobId) {
+        checkNotNull(jobShuffleContexts.get(jobId)).notifyPartitionRecoveryStarted();
+    }
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/ShuffleMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/ShuffleMaster.java
index f2ef66ea13d..09950d4e3f1 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/ShuffleMaster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/ShuffleMaster.java
@@ -20,10 +20,13 @@ package org.apache.flink.runtime.shuffle;
 
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.configuration.MemorySize;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
 
+import java.time.Duration;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 
 /**
@@ -112,14 +115,18 @@ public interface ShuffleMaster<T extends ShuffleDescriptor> extends AutoCloseabl
     }
 
     /**
-     * Get all partitions and their metrics, the metrics include sizes of sub-partitions in a result
-     * partition.
+     * Retrieves specified partitions and their metrics (identified by {@code expectedPartitions}),
+     * the metrics include sizes of sub-partitions in a result partition.
      *
      * @param jobId ID of the target job
-     * @return All partitions belong to the target job and their metrics
+     * @param timeout The timeout used for retrieve the specified partitions.
+     * @param expectedPartitions The set of identifiers for the result partitions whose metrics are
+     *     to be fetched.
+     * @return A future will contain a collection of the partitions with their metrics that could be
+     *     retrieved from the expected partitions within the specified timeout period.
      */
-    default CompletableFuture<Collection<PartitionWithMetrics>> getAllPartitionWithMetrics(
-            JobID jobId) {
+    default CompletableFuture<Collection<PartitionWithMetrics>> getPartitionWithMetrics(
+            JobID jobId, Duration timeout, Set<ResultPartitionID> expectedPartitions) {
         return CompletableFuture.completedFuture(Collections.emptyList());
     }
 
@@ -140,4 +147,11 @@ public interface ShuffleMaster<T extends ShuffleDescriptor> extends AutoCloseabl
 
     /** Restores the state of the shuffle master from the provided snapshots. */
     default void restoreState(List<ShuffleMasterSnapshot> snapshots) {}
+
+    /**
+     * Notifies that the recovery process of result partitions has started.
+     *
+     * @param jobId ID of the target job
+     */
+    default void notifyPartitionRecoveryStarted(JobID jobId) {}
 }
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTest.java
index bc03aa604f7..e1f60398a3a 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTest.java
@@ -26,6 +26,7 @@ import org.apache.flink.api.common.io.DefaultInputSplitAssigner;
 import org.apache.flink.api.common.restartstrategy.RestartStrategies;
 import org.apache.flink.api.common.time.Time;
 import org.apache.flink.api.java.ClosureCleaner;
+import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.api.java.tuple.Tuple3;
 import org.apache.flink.configuration.BlobServerOptions;
 import org.apache.flink.configuration.Configuration;
@@ -152,11 +153,13 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Queue;
+import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.BlockingQueue;
@@ -1909,7 +1912,7 @@ class JobMasterTest {
     }
 
     @Test
-    void testGetAllPartitionWithMetrics() throws Exception {
+    void testGetPartitionWithMetrics() throws Exception {
         JobVertex jobVertex = new JobVertex("jobVertex");
         jobVertex.setInvokableClass(NoOpInvokable.class);
         jobVertex.setParallelism(1);
@@ -1929,57 +1932,211 @@ class JobMasterTest {
             final JobMasterGateway jobMasterGateway =
                     jobMaster.getSelfGateway(JobMasterGateway.class);
 
-            NettyShuffleDescriptor shuffleDescriptor =
-                    NettyShuffleDescriptorBuilder.newBuilder().buildLocal();
-            DefaultShuffleMetrics shuffleMetrics =
+            DefaultShuffleMetrics shuffleMetrics1 =
                     new DefaultShuffleMetrics(new ResultPartitionBytes(new long[] {1, 2, 3}));
-            Collection<PartitionWithMetrics> defaultPartitionWithMetrics =
+            NettyShuffleDescriptor descriptor1 =
+                    NettyShuffleDescriptorBuilder.newBuilder().buildLocal();
+            Collection<PartitionWithMetrics> defaultPartitionWithMetrics1 =
                     Collections.singletonList(
-                            new DefaultPartitionWithMetrics(shuffleDescriptor, shuffleMetrics));
-            final TestingTaskExecutorGateway taskExecutorGateway =
+                            new DefaultPartitionWithMetrics(descriptor1, shuffleMetrics1));
+
+            DefaultShuffleMetrics shuffleMetrics2 =
+                    new DefaultShuffleMetrics(new ResultPartitionBytes(new long[] {4, 5, 6}));
+            NettyShuffleDescriptor descriptor2 =
+                    NettyShuffleDescriptorBuilder.newBuilder().buildLocal();
+            Collection<PartitionWithMetrics> defaultPartitionWithMetrics2 =
+                    Collections.singletonList(
+                            new DefaultPartitionWithMetrics(descriptor2, shuffleMetrics2));
+
+            DefaultShuffleMetrics shuffleMetrics3 =
+                    new DefaultShuffleMetrics(new ResultPartitionBytes(new long[] {7, 8, 9}));
+            NettyShuffleDescriptor descriptor3 =
+                    NettyShuffleDescriptorBuilder.newBuilder().buildLocal();
+            Collection<PartitionWithMetrics> defaultPartitionWithMetrics3 =
+                    Collections.singletonList(
+                            new DefaultPartitionWithMetrics(descriptor3, shuffleMetrics3));
+
+            DefaultShuffleMetrics shuffleMetrics4 =
+                    new DefaultShuffleMetrics(new ResultPartitionBytes(new long[] {10, 11}));
+            NettyShuffleDescriptor descriptor4 =
+                    NettyShuffleDescriptorBuilder.newBuilder().buildLocal();
+            Collection<PartitionWithMetrics> defaultPartitionWithMetrics4 =
+                    Collections.singletonList(
+                            new DefaultPartitionWithMetrics(descriptor4, shuffleMetrics4));
+
+            // start fetch and retain partitions and then register tm1
+            final TestingTaskExecutorGateway taskExecutorGateway1 =
                     new TestingTaskExecutorGatewayBuilder()
                             .setRequestPartitionWithMetricsFunction(
                                     ignored ->
                                             CompletableFuture.completedFuture(
-                                                    defaultPartitionWithMetrics))
+                                                    defaultPartitionWithMetrics1))
+                            .setAddress("tm1")
                             .createTestingTaskExecutorGateway();
 
-            final LocalUnresolvedTaskManagerLocation taskManagerUnresolvedLocation =
-                    new LocalUnresolvedTaskManagerLocation();
-            final Collection<SlotOffer> slotOffers =
-                    registerSlotsAtJobMaster(
-                            1,
-                            jobMasterGateway,
-                            jobGraph.getJobID(),
-                            taskExecutorGateway,
-                            taskManagerUnresolvedLocation);
-            assertThat(slotOffers).hasSize(1);
+            registerSlotsAtJobMaster(
+                    1,
+                    jobMasterGateway,
+                    jobGraph.getJobID(),
+                    taskExecutorGateway1,
+                    new LocalUnresolvedTaskManagerLocation());
 
-            waitUntilAllExecutionsAreScheduledOrDeployed(jobMasterGateway);
+            jobMaster.startFetchAndRetainPartitionWithMetricsOnTaskManager();
 
-            PartitionWithMetrics metrics =
-                    jobMasterGateway
-                            .getAllPartitionWithMetricsOnTaskManagers()
-                            .get()
-                            .iterator()
-                            .next();
-            PartitionWithMetrics expectedMetrics = defaultPartitionWithMetrics.iterator().next();
+            verifyPartitionMetrics(
+                    jobMaster.getPartitionWithMetricsOnTaskManagers(),
+                    defaultPartitionWithMetrics1);
+
+            // register tm2
+            TestingTaskExecutorGateway taskExecutorGateway2 =
+                    new TestingTaskExecutorGatewayBuilder()
+                            .setRequestPartitionWithMetricsFunction(
+                                    ignored ->
+                                            CompletableFuture.completedFuture(
+                                                    defaultPartitionWithMetrics2))
+                            .setAddress("tm2")
+                            .createTestingTaskExecutorGateway();
+            registerSlotsAtJobMaster(
+                    1,
+                    jobMasterGateway,
+                    jobGraph.getJobID(),
+                    taskExecutorGateway2,
+                    new LocalUnresolvedTaskManagerLocation());
+
+            Collection<PartitionWithMetrics> expectedMetrics =
+                    new ArrayList<>(defaultPartitionWithMetrics1);
+            expectedMetrics.addAll(defaultPartitionWithMetrics2);
+            verifyPartitionMetrics(
+                    jobMaster.getPartitionWithMetricsOnTaskManagers(), expectedMetrics);
 
-            assertThat(metrics.getPartitionMetrics().getPartitionBytes().getSubpartitionBytes())
+            // register tm3 which received fetch request but not response on time.
+            CompletableFuture<Tuple2<JobID, Set<ResultPartitionID>>> releaseRequest =
+                    new CompletableFuture<>();
+            CompletableFuture<Collection<PartitionWithMetrics>> fetchPartitionsFuture =
+                    new CompletableFuture<>();
+            TestingTaskExecutorGateway taskExecutorGateway3 =
+                    new TestingTaskExecutorGatewayBuilder()
+                            .setRequestPartitionWithMetricsFunction(
+                                    ignored -> fetchPartitionsFuture)
+                            .setReleasePartitionsConsumer(
+                                    (id, partitions) ->
+                                            releaseRequest.complete(Tuple2.of(id, partitions)))
+                            .setAddress("tm3")
+                            .createTestingTaskExecutorGateway();
+
+            registerSlotsAtJobMaster(
+                    1,
+                    jobMasterGateway,
+                    jobGraph.getJobID(),
+                    taskExecutorGateway3,
+                    new LocalUnresolvedTaskManagerLocation());
+
+            // register tm4 which is not included in expected partitions
+            TestingTaskExecutorGateway taskExecutorGateway4 =
+                    new TestingTaskExecutorGatewayBuilder()
+                            .setRequestPartitionWithMetricsFunction(
+                                    ignored ->
+                                            CompletableFuture.completedFuture(
+                                                    defaultPartitionWithMetrics4))
+                            .setAddress("tm4")
+                            .createTestingTaskExecutorGateway();
+            registerSlotsAtJobMaster(
+                    1,
+                    jobMasterGateway,
+                    jobGraph.getJobID(),
+                    taskExecutorGateway4,
+                    new LocalUnresolvedTaskManagerLocation());
+
+            Duration timeout = Duration.ofSeconds(10);
+            Set<ResultPartitionID> expectedResultPartitions = new HashSet<>();
+            expectedResultPartitions.add(descriptor1.getResultPartitionID());
+            expectedResultPartitions.add(descriptor2.getResultPartitionID());
+            expectedResultPartitions.add(descriptor4.getResultPartitionID());
+            CompletableFuture<Collection<PartitionWithMetrics>> future =
+                    jobMasterGateway.getPartitionWithMetrics(timeout, expectedResultPartitions);
+
+            expectedMetrics = new ArrayList<>(defaultPartitionWithMetrics1);
+            expectedMetrics.addAll(defaultPartitionWithMetrics2);
+            expectedMetrics.addAll(defaultPartitionWithMetrics4);
+
+            assertThat(future).succeedsWithin(timeout);
+            verifyPartitionMetrics(
+                    future.get().stream()
+                            .collect(
+                                    Collectors.toMap(
+                                            metrics ->
+                                                    metrics.getPartition().getResultPartitionID(),
+                                            metrics -> metrics)),
+                    expectedMetrics);
+
+            // fetch partition after timeout
+            fetchPartitionsFuture.complete(defaultPartitionWithMetrics3);
+            assertThat(releaseRequest.get())
                     .isEqualTo(
-                            expectedMetrics
-                                    .getPartitionMetrics()
-                                    .getPartitionBytes()
-                                    .getSubpartitionBytes());
-            assertThat(metrics.getPartition().getResultPartitionID())
-                    .isEqualTo(expectedMetrics.getPartition().getResultPartitionID());
-            assertThat(metrics.getPartition().isUnknown())
-                    .isEqualTo(expectedMetrics.getPartition().isUnknown());
-            assertThat(metrics.getPartition().storesLocalResourcesOn())
-                    .isEqualTo(expectedMetrics.getPartition().storesLocalResourcesOn());
+                            Tuple2.of(
+                                    jobGraph.getJobID(),
+                                    Collections.singleton(descriptor3.getResultPartitionID())));
+
+            // after partitions fetching finished and then register tm5
+            CompletableFuture<Collection<PartitionWithMetrics>> requestFuture =
+                    new CompletableFuture<>();
+            TestingTaskExecutorGateway taskExecutorGateway5 =
+                    new TestingTaskExecutorGatewayBuilder()
+                            .setRequestPartitionWithMetricsFunction(
+                                    ignored -> {
+                                        requestFuture.complete(null);
+                                        return requestFuture;
+                                    })
+                            .setAddress("tm5")
+                            .createTestingTaskExecutorGateway();
+
+            registerSlotsAtJobMaster(
+                    1,
+                    jobMasterGateway,
+                    jobGraph.getJobID(),
+                    taskExecutorGateway5,
+                    new LocalUnresolvedTaskManagerLocation());
+
+            assertThatThrownBy(() -> requestFuture.get(timeout.toMillis(), TimeUnit.MILLISECONDS))
+                    .isInstanceOf(TimeoutException.class);
         }
     }
 
+    private static void verifyPartitionMetrics(
+            Map<ResultPartitionID, PartitionWithMetrics> actualMap,
+            Collection<PartitionWithMetrics> expectedMetrics) {
+
+        Map<ResultPartitionID, PartitionWithMetrics> expectedMap =
+                expectedMetrics.stream()
+                        .collect(
+                                Collectors.toMap(
+                                        metrics -> metrics.getPartition().getResultPartitionID(),
+                                        metrics -> metrics));
+
+        assertThat(actualMap).hasSameSizeAs(expectedMap);
+
+        actualMap.forEach(
+                (k, v) -> {
+                    PartitionWithMetrics metrics = expectedMap.get(k);
+
+                    assertThat(metrics).isNotNull();
+                    assertThat(
+                                    metrics.getPartitionMetrics()
+                                            .getPartitionBytes()
+                                            .getSubpartitionBytes())
+                            .isEqualTo(
+                                    v.getPartitionMetrics()
+                                            .getPartitionBytes()
+                                            .getSubpartitionBytes());
+
+                    assertThat(metrics.getPartition().isUnknown())
+                            .isEqualTo(v.getPartition().isUnknown());
+
+                    assertThat(metrics.getPartition().storesLocalResourcesOn())
+                            .isEqualTo(v.getPartition().storesLocalResourcesOn());
+                });
+    }
+
     @Test
     void testBlockResourcesWillTriggerReleaseFreeSlots() throws Exception {
         JobVertex jobVertex = new JobVertex("jobVertex");
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/TestingJobMasterGateway.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/TestingJobMasterGateway.java
index ecdc78f45a9..384ba882f97 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/TestingJobMasterGateway.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/TestingJobMasterGateway.java
@@ -55,6 +55,7 @@ import org.apache.flink.runtime.query.KvStateLocation;
 import org.apache.flink.runtime.registration.RegistrationResponse;
 import org.apache.flink.runtime.resourcemanager.ResourceManagerId;
 import org.apache.flink.runtime.scheduler.ExecutionGraphInfo;
+import org.apache.flink.runtime.shuffle.PartitionWithMetrics;
 import org.apache.flink.runtime.slots.ResourceRequirement;
 import org.apache.flink.runtime.state.KeyGroupRange;
 import org.apache.flink.runtime.taskexecutor.TaskExecutorToJobManagerHeartbeatPayload;
@@ -68,7 +69,9 @@ import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
 import java.net.InetSocketAddress;
+import java.time.Duration;
 import java.util.Collection;
+import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.function.BiFunction;
 import java.util.function.Consumer;
@@ -173,6 +176,13 @@ public class TestingJobMasterGateway implements JobMasterGateway {
                     CompletableFuture<Acknowledge>>
             notifyKvStateUnregisteredFunction;
 
+    @Nonnull
+    private final BiFunction<
+                    Duration,
+                    Set<ResultPartitionID>,
+                    CompletableFuture<Collection<PartitionWithMetrics>>>
+            getPartitionWithMetricsFunction;
+
     @Nonnull TriFunction<String, Object, byte[], CompletableFuture<Object>> updateAggregateFunction;
 
     @Nonnull
@@ -312,7 +322,13 @@ public class TestingJobMasterGateway implements JobMasterGateway {
                             requestJobResourceRequirementsSupplier,
             @Nonnull
                     Function<JobResourceRequirements, CompletableFuture<Acknowledge>>
-                            updateJobResourceRequirementsFunction) {
+                            updateJobResourceRequirementsFunction,
+            @Nonnull
+                    BiFunction<
+                                    Duration,
+                                    Set<ResultPartitionID>,
+                                    CompletableFuture<Collection<PartitionWithMetrics>>>
+                            getPartitionWithMetricsFunction) {
         this.address = address;
         this.hostname = hostname;
         this.cancelFunction = cancelFunction;
@@ -345,6 +361,13 @@ public class TestingJobMasterGateway implements JobMasterGateway {
         this.notifyNewBlockedNodesFunction = notifyNewBlockedNodesFunction;
         this.requestJobResourceRequirementsSupplier = requestJobResourceRequirementsSupplier;
         this.updateJobResourceRequirementsFunction = updateJobResourceRequirementsFunction;
+        this.getPartitionWithMetricsFunction = getPartitionWithMetricsFunction;
+    }
+
+    @Override
+    public CompletableFuture<Collection<PartitionWithMetrics>> getPartitionWithMetrics(
+            Duration timeout, Set<ResultPartitionID> expectedPartitions) {
+        return getPartitionWithMetricsFunction.apply(timeout, expectedPartitions);
     }
 
     @Override
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/TestingJobMasterGatewayBuilder.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/TestingJobMasterGatewayBuilder.java
index 6fbc367f800..4b5c2104571 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/TestingJobMasterGatewayBuilder.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/TestingJobMasterGatewayBuilder.java
@@ -54,6 +54,7 @@ import org.apache.flink.runtime.query.UnknownKvStateLocation;
 import org.apache.flink.runtime.registration.RegistrationResponse;
 import org.apache.flink.runtime.resourcemanager.ResourceManagerId;
 import org.apache.flink.runtime.scheduler.ExecutionGraphInfo;
+import org.apache.flink.runtime.shuffle.PartitionWithMetrics;
 import org.apache.flink.runtime.slots.ResourceRequirement;
 import org.apache.flink.runtime.state.KeyGroupRange;
 import org.apache.flink.runtime.taskexecutor.TaskExecutorToJobManagerHeartbeatPayload;
@@ -65,9 +66,11 @@ import org.apache.flink.util.function.TriConsumer;
 import org.apache.flink.util.function.TriFunction;
 
 import java.net.InetSocketAddress;
+import java.time.Duration;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Map;
+import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 import java.util.function.BiFunction;
@@ -188,6 +191,13 @@ public class TestingJobMasterGatewayBuilder {
             requestJobResourceRequirementsSupplier =
                     () -> CompletableFuture.completedFuture(JobResourceRequirements.empty());
 
+    private BiFunction<
+                    Duration,
+                    Set<ResultPartitionID>,
+                    CompletableFuture<Collection<PartitionWithMetrics>>>
+            getPartitionWithMetricsFunction =
+                    (timeout, set) -> CompletableFuture.completedFuture(Collections.emptyList());
+
     private Function<JobResourceRequirements, CompletableFuture<Acknowledge>>
             updateJobResourceRequirementsFunction =
                     ignored -> CompletableFuture.completedFuture(Acknowledge.get());
@@ -424,6 +434,16 @@ public class TestingJobMasterGatewayBuilder {
         return this;
     }
 
+    public TestingJobMasterGatewayBuilder setGetPartitionWithMetricsFunction(
+            BiFunction<
+                            Duration,
+                            Set<ResultPartitionID>,
+                            CompletableFuture<Collection<PartitionWithMetrics>>>
+                    getPartitionWithMetricsFunction) {
+        this.getPartitionWithMetricsFunction = getPartitionWithMetricsFunction;
+        return this;
+    }
+
     public TestingJobMasterGatewayBuilder setUpdateJobResourceRequirementsFunction(
             Function<JobResourceRequirements, CompletableFuture<Acknowledge>>
                     updateJobResourceRequirementsFunction) {
@@ -464,6 +484,7 @@ public class TestingJobMasterGatewayBuilder {
                 notifyNotEnoughResourcesConsumer,
                 notifyNewBlockedNodesFunction,
                 requestJobResourceRequirementsSupplier,
-                updateJobResourceRequirementsFunction);
+                updateJobResourceRequirementsFunction,
+                getPartitionWithMetricsFunction);
     }
 }