You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by zh...@apache.org on 2024/05/28 01:43:42 UTC
(flink) 02/03: [FLINK-33892][runtime] Support JobMaster fetch and retain partition metrics on TaskManagers during job recovery.
This is an automated email from the ASF dual-hosted git repository.
zhuzh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 3206659db5b7c4ce645072f11f091e0e9e92b0ce
Author: JunRuiLee <jr...@gmail.com>
AuthorDate: Sat May 11 12:24:16 2024 +0800
[FLINK-33892][runtime] Support JobMaster fetch and retain partition metrics on TaskManagers during job recovery.
---
.../apache/flink/runtime/jobmaster/JobMaster.java | 197 ++++++++++++++----
.../flink/runtime/jobmaster/JobMasterGateway.java | 22 +-
.../flink/runtime/shuffle/JobShuffleContext.java | 18 +-
.../runtime/shuffle/JobShuffleContextImpl.java | 13 +-
.../flink/runtime/shuffle/NettyShuffleMaster.java | 13 +-
.../flink/runtime/shuffle/ShuffleMaster.java | 24 ++-
.../flink/runtime/jobmaster/JobMasterTest.java | 231 +++++++++++++++++----
.../jobmaster/utils/TestingJobMasterGateway.java | 25 ++-
.../utils/TestingJobMasterGatewayBuilder.java | 23 +-
9 files changed, 469 insertions(+), 97 deletions(-)
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
index 859caa53920..ee5765d198f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
@@ -18,6 +18,7 @@
package org.apache.flink.runtime.jobmaster;
+import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.JobStatus;
import org.apache.flink.api.common.functions.AggregateFunction;
@@ -91,6 +92,7 @@ import org.apache.flink.runtime.scheduler.SchedulerNG;
import org.apache.flink.runtime.shuffle.JobShuffleContext;
import org.apache.flink.runtime.shuffle.JobShuffleContextImpl;
import org.apache.flink.runtime.shuffle.PartitionWithMetrics;
+import org.apache.flink.runtime.shuffle.ShuffleDescriptor;
import org.apache.flink.runtime.shuffle.ShuffleMaster;
import org.apache.flink.runtime.slots.ResourceRequirement;
import org.apache.flink.runtime.state.KeyGroupRange;
@@ -116,11 +118,11 @@ import javax.annotation.Nullable;
import java.io.IOException;
import java.net.InetSocketAddress;
+import java.time.Duration;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
@@ -130,6 +132,7 @@ import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.concurrent.Executor;
import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.stream.Collectors;
@@ -229,6 +232,22 @@ public class JobMaster extends FencedRpcEndpoint<JobMasterId>
private final BlocklistHandler blocklistHandler;
+ private final Map<ResultPartitionID, PartitionWithMetrics> fetchedPartitionsWithMetrics =
+ new HashMap<>();
+
+ /**
+ * A flag that indicates whether to fetch and retain partitions on task managers. This will
+ * apply to future TaskManager registrations as well as already registered TaskManagers. The
+ * flag will be set to true when starting batch job recovery and set to false after all required
+ * partitions, as defined in {@code requireToFetchPartitions}, are either fetched or when a
+ * timeout occurs.
+ */
+ private boolean fetchAndRetainPartitions = false;
+
+ private Set<ResultPartitionID> partitionsToFetch;
+
+ private CompletableFuture<Collection<PartitionWithMetrics>> fetchPartitionsFuture;
+
// ------------------------------------------------------------------------
public JobMaster(
@@ -797,30 +816,41 @@ public class JobMaster extends FencedRpcEndpoint<JobMasterId>
}
}
- return getRpcService()
- .connect(
- taskManagerRegistrationInformation.getTaskManagerRpcAddress(),
- TaskExecutorGateway.class)
- .handleAsync(
- (TaskExecutorGateway taskExecutorGateway, Throwable throwable) -> {
- if (throwable != null) {
- return new RegistrationResponse.Failure(throwable);
- }
-
- slotPoolService.registerTaskManager(taskManagerId);
- registeredTaskManagers.put(
- taskManagerId,
- TaskManagerRegistration.create(
- taskManagerLocation, taskExecutorGateway, sessionId));
-
- // monitor the task manager as heartbeat target
- taskManagerHeartbeatManager.monitorTarget(
- taskManagerId,
- new TaskExecutorHeartbeatSender(taskExecutorGateway));
+ CompletableFuture<RegistrationResponse> registrationResponseFuture =
+ getRpcService()
+ .connect(
+ taskManagerRegistrationInformation.getTaskManagerRpcAddress(),
+ TaskExecutorGateway.class)
+ .handleAsync(
+ (TaskExecutorGateway taskExecutorGateway, Throwable throwable) -> {
+ if (throwable != null) {
+ return new RegistrationResponse.Failure(throwable);
+ }
+
+ slotPoolService.registerTaskManager(taskManagerId);
+ registeredTaskManagers.put(
+ taskManagerId,
+ TaskManagerRegistration.create(
+ taskManagerLocation,
+ taskExecutorGateway,
+ sessionId));
+
+ // monitor the task manager as heartbeat target
+ taskManagerHeartbeatManager.monitorTarget(
+ taskManagerId,
+ new TaskExecutorHeartbeatSender(taskExecutorGateway));
+
+ return new JMTMRegistrationSuccess(resourceId);
+ },
+ getMainThreadExecutor());
+
+ if (fetchAndRetainPartitions) {
+ registrationResponseFuture.whenComplete(
+ (ignored, throwable) ->
+ fetchAndRetainPartitionWithMetricsOnTaskManager(taskManagerId));
+ }
- return new JMTMRegistrationSuccess(resourceId);
- },
- getMainThreadExecutor());
+ return registrationResponseFuture;
}
@Nonnull
@@ -961,25 +991,108 @@ public class JobMaster extends FencedRpcEndpoint<JobMasterId>
}
@Override
- public CompletableFuture<Collection<PartitionWithMetrics>>
- getAllPartitionWithMetricsOnTaskManagers() {
- final List<CompletableFuture<Collection<PartitionWithMetrics>>> allFutures =
- new ArrayList<>();
+ public CompletableFuture<Collection<PartitionWithMetrics>> getPartitionWithMetrics(
+ Duration timeout, Set<ResultPartitionID> expectedPartitions) {
+ this.partitionsToFetch = expectedPartitions;
+ this.fetchPartitionsFuture = new CompletableFuture<>();
+
+ // check already fetched partitions
+ checkPartitionOnTaskManagerReportFinished();
+
+ return FutureUtils.orTimeout(
+ fetchPartitionsFuture, timeout.toMillis(), TimeUnit.MILLISECONDS, null)
+ .handleAsync(
+ (metrics, throwable) -> {
+ stopFetchAndRetainPartitionWithMetricsOnTaskManager();
+
+ if (throwable != null) {
+ if (throwable instanceof TimeoutException) {
+ log.warn(
+ "Timeout occurred after {} ms "
+ + "while fetching partition(s) ({}) from task managers.",
+ timeout.toMillis(),
+ expectedPartitions);
+
+ return new ArrayList<>(fetchedPartitionsWithMetrics.values());
+ }
+ throw new CompletionException(throwable);
+ }
+
+ return new ArrayList<>(fetchedPartitionsWithMetrics.values());
+ },
+ getMainThreadExecutor());
+ }
+
+ @VisibleForTesting
+ Map<ResultPartitionID, PartitionWithMetrics> getPartitionWithMetricsOnTaskManagers() {
+ return fetchedPartitionsWithMetrics;
+ }
+
+ @Override
+ public void startFetchAndRetainPartitionWithMetricsOnTaskManager() {
+ fetchAndRetainPartitions = true;
+
+ // process all already registered task managers
registeredTaskManagers
- .values()
- .forEach(
- taskManager ->
- allFutures.add(
- taskManager
- .getTaskExecutorGateway()
- .getAndRetainPartitionWithMetrics(
- jobGraph.getJobID())));
- return FutureUtils.combineAll(allFutures)
- .thenApply(
- partitions ->
- partitions.stream()
- .flatMap(Collection::stream)
- .collect(Collectors.toList()));
+ .keySet()
+ .forEach(this::fetchAndRetainPartitionWithMetricsOnTaskManager);
+ }
+
+ private void fetchAndRetainPartitionWithMetricsOnTaskManager(ResourceID resourceId) {
+ TaskManagerRegistration taskManager = registeredTaskManagers.get(resourceId);
+ checkNotNull(taskManager);
+
+ taskManager
+ .getTaskExecutorGateway()
+ .getAndRetainPartitionWithMetrics(jobGraph.getJobID())
+ .thenAccept(
+ partitionWithMetrics -> {
+ if (fetchAndRetainPartitions) {
+ for (PartitionWithMetrics partitionWithMetric :
+ partitionWithMetrics) {
+ log.debug(
+ "Received partition metrics for {} from Task Manager {}.",
+ partitionWithMetric
+ .getPartition()
+ .getResultPartitionID(),
+ resourceId);
+ fetchedPartitionsWithMetrics.put(
+ partitionWithMetric
+ .getPartition()
+ .getResultPartitionID(),
+ partitionWithMetric);
+ }
+ checkPartitionOnTaskManagerReportFinished();
+ } else {
+ log.info(
+ "Received late report of partition metrics from {}. Release the partitions.",
+ resourceId);
+
+ taskManager
+ .getTaskExecutorGateway()
+ .releasePartitions(
+ jobGraph.getJobID(),
+ partitionWithMetrics.stream()
+ .map(PartitionWithMetrics::getPartition)
+ .map(
+ ShuffleDescriptor
+ ::getResultPartitionID)
+ .collect(Collectors.toSet()));
+ }
+ });
+ }
+
+ private void stopFetchAndRetainPartitionWithMetricsOnTaskManager() {
+ fetchAndRetainPartitions = false;
+ }
+
+ private void checkPartitionOnTaskManagerReportFinished() {
+ if (fetchPartitionsFuture != null) {
+ if (fetchedPartitionsWithMetrics.keySet().containsAll(partitionsToFetch)
+ && !fetchPartitionsFuture.isDone()) {
+ fetchPartitionsFuture.complete(fetchedPartitionsWithMetrics.values());
+ }
+ }
}
@Override
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
index 994492807ff..38772fbadae 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
@@ -55,8 +55,10 @@ import org.apache.flink.util.SerializedValue;
import javax.annotation.Nullable;
+import java.time.Duration;
import java.util.Collection;
import java.util.Collections;
+import java.util.Set;
import java.util.concurrent.CompletableFuture;
/** {@link JobMaster} rpc gateway interface. */
@@ -303,11 +305,27 @@ public interface JobMasterGateway
CompletableFuture<?> stopTrackingAndReleasePartitions(
Collection<ResultPartitionID> partitionIds);
- default CompletableFuture<Collection<PartitionWithMetrics>>
- getAllPartitionWithMetricsOnTaskManagers() {
+ /**
+ * Get specified partitions and their metrics (identified by {@code expectedPartitions}), the
+ * metrics include sizes of sub-partitions in a result partition.
+ *
+ * @param timeout The timeout used for retrieve the specified partitions.
+ * @param expectedPartitions The set of identifiers for the result partitions whose metrics are
+ * to be fetched.
+ * @return A future will contain a collection of the partitions with their metrics that could be
+ * retrieved from the expected partitions within the specified timeout period.
+ */
+ default CompletableFuture<Collection<PartitionWithMetrics>> getPartitionWithMetrics(
+ Duration timeout, Set<ResultPartitionID> expectedPartitions) {
return CompletableFuture.completedFuture(Collections.emptyList());
}
+ /**
+ * Notify jobMaster to fetch and retain partitions on task managers. It will process for future
+ * TaskManager registrations and already registered TaskManagers.
+ */
+ default void startFetchAndRetainPartitionWithMetricsOnTaskManager() {}
+
/**
* Read current {@link JobResourceRequirements job resource requirements}.
*
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/JobShuffleContext.java b/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/JobShuffleContext.java
index 585b73806d9..5cd59046381 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/JobShuffleContext.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/JobShuffleContext.java
@@ -21,7 +21,9 @@ package org.apache.flink.runtime.shuffle;
import org.apache.flink.api.common.JobID;
import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
+import java.time.Duration;
import java.util.Collection;
+import java.util.Set;
import java.util.concurrent.CompletableFuture;
/**
@@ -41,8 +43,18 @@ public interface JobShuffleContext {
Collection<ResultPartitionID> partitionIds);
/**
- * Retrieves a collection containing descriptions and metrics of existing result partitions from
- * all TaskManagers.
+ * Retrieves specified partitions and their metrics (identified by {@code expectedPartitions}),
+ * the metrics include sizes of sub-partitions in a result partition.
+ *
+ * @param timeout The timeout used for retrieve the specified partitions.
+ * @param expectedPartitions The set of identifiers for the result partitions whose metrics are
+ * to be fetched.
+ * @return A future will contain a collection of the partitions with their metrics that could be
+ * retrieved from the expected partitions within the specified timeout period.
*/
- CompletableFuture<Collection<PartitionWithMetrics>> getAllPartitionWithMetricsOnTaskManagers();
+ CompletableFuture<Collection<PartitionWithMetrics>> getPartitionWithMetrics(
+ Duration timeout, Set<ResultPartitionID> expectedPartitions);
+
+ /** Notifies that the recovery process of result partitions has started. */
+ void notifyPartitionRecoveryStarted();
}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/JobShuffleContextImpl.java b/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/JobShuffleContextImpl.java
index 546cc3025b4..51fd013788c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/JobShuffleContextImpl.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/JobShuffleContextImpl.java
@@ -22,7 +22,9 @@ import org.apache.flink.api.common.JobID;
import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
import org.apache.flink.runtime.jobmaster.JobMasterGateway;
+import java.time.Duration;
import java.util.Collection;
+import java.util.Set;
import java.util.concurrent.CompletableFuture;
import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -51,8 +53,13 @@ public class JobShuffleContextImpl implements JobShuffleContext {
}
@Override
- public CompletableFuture<Collection<PartitionWithMetrics>>
- getAllPartitionWithMetricsOnTaskManagers() {
- return jobMasterGateway.getAllPartitionWithMetricsOnTaskManagers();
+ public CompletableFuture<Collection<PartitionWithMetrics>> getPartitionWithMetrics(
+ Duration timeout, Set<ResultPartitionID> expectedPartitions) {
+ return jobMasterGateway.getPartitionWithMetrics(timeout, expectedPartitions);
+ }
+
+ @Override
+ public void notifyPartitionRecoveryStarted() {
+ jobMasterGateway.startFetchAndRetainPartitionWithMetricsOnTaskManager();
}
}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/NettyShuffleMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/NettyShuffleMaster.java
index 461457a29b6..df544fdce47 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/NettyShuffleMaster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/NettyShuffleMaster.java
@@ -31,10 +31,12 @@ import org.apache.flink.runtime.util.ConfigurationParserUtils;
import javax.annotation.Nullable;
+import java.time.Duration;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
+import java.util.Set;
import java.util.concurrent.CompletableFuture;
import static org.apache.flink.api.common.BatchShuffleMode.ALL_EXCHANGES_HYBRID_FULL;
@@ -172,10 +174,10 @@ public class NettyShuffleMaster implements ShuffleMaster<NettyShuffleDescriptor>
}
@Override
- public CompletableFuture<Collection<PartitionWithMetrics>> getAllPartitionWithMetrics(
- JobID jobId) {
+ public CompletableFuture<Collection<PartitionWithMetrics>> getPartitionWithMetrics(
+ JobID jobId, Duration timeout, Set<ResultPartitionID> expectedPartitions) {
return checkNotNull(jobShuffleContexts.get(jobId))
- .getAllPartitionWithMetricsOnTaskManagers();
+ .getPartitionWithMetrics(timeout, expectedPartitions);
}
@Override
@@ -199,4 +201,9 @@ public class NettyShuffleMaster implements ShuffleMaster<NettyShuffleDescriptor>
ShuffleMasterSnapshotContext context) {
snapshotFuture.complete(EmptyShuffleMasterSnapshot.getInstance());
}
+
+ @Override
+ public void notifyPartitionRecoveryStarted(JobID jobId) {
+ checkNotNull(jobShuffleContexts.get(jobId)).notifyPartitionRecoveryStarted();
+ }
}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/ShuffleMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/ShuffleMaster.java
index f2ef66ea13d..09950d4e3f1 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/ShuffleMaster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/ShuffleMaster.java
@@ -20,10 +20,13 @@ package org.apache.flink.runtime.shuffle;
import org.apache.flink.api.common.JobID;
import org.apache.flink.configuration.MemorySize;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
+import java.time.Duration;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.CompletableFuture;
/**
@@ -112,14 +115,18 @@ public interface ShuffleMaster<T extends ShuffleDescriptor> extends AutoCloseabl
}
/**
- * Get all partitions and their metrics, the metrics include sizes of sub-partitions in a result
- * partition.
+ * Retrieves specified partitions and their metrics (identified by {@code expectedPartitions}),
+ * the metrics include sizes of sub-partitions in a result partition.
*
* @param jobId ID of the target job
- * @return All partitions belong to the target job and their metrics
+ * @param timeout The timeout used for retrieve the specified partitions.
+ * @param expectedPartitions The set of identifiers for the result partitions whose metrics are
+ * to be fetched.
+ * @return A future will contain a collection of the partitions with their metrics that could be
+ * retrieved from the expected partitions within the specified timeout period.
*/
- default CompletableFuture<Collection<PartitionWithMetrics>> getAllPartitionWithMetrics(
- JobID jobId) {
+ default CompletableFuture<Collection<PartitionWithMetrics>> getPartitionWithMetrics(
+ JobID jobId, Duration timeout, Set<ResultPartitionID> expectedPartitions) {
return CompletableFuture.completedFuture(Collections.emptyList());
}
@@ -140,4 +147,11 @@ public interface ShuffleMaster<T extends ShuffleDescriptor> extends AutoCloseabl
/** Restores the state of the shuffle master from the provided snapshots. */
default void restoreState(List<ShuffleMasterSnapshot> snapshots) {}
+
+ /**
+ * Notifies that the recovery process of result partitions has started.
+ *
+ * @param jobId ID of the target job
+ */
+ default void notifyPartitionRecoveryStarted(JobID jobId) {}
}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTest.java
index bc03aa604f7..e1f60398a3a 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTest.java
@@ -26,6 +26,7 @@ import org.apache.flink.api.common.io.DefaultInputSplitAssigner;
import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.api.java.ClosureCleaner;
+import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.configuration.BlobServerOptions;
import org.apache.flink.configuration.Configuration;
@@ -152,11 +153,13 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Queue;
+import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
@@ -1909,7 +1912,7 @@ class JobMasterTest {
}
@Test
- void testGetAllPartitionWithMetrics() throws Exception {
+ void testGetPartitionWithMetrics() throws Exception {
JobVertex jobVertex = new JobVertex("jobVertex");
jobVertex.setInvokableClass(NoOpInvokable.class);
jobVertex.setParallelism(1);
@@ -1929,57 +1932,211 @@ class JobMasterTest {
final JobMasterGateway jobMasterGateway =
jobMaster.getSelfGateway(JobMasterGateway.class);
- NettyShuffleDescriptor shuffleDescriptor =
- NettyShuffleDescriptorBuilder.newBuilder().buildLocal();
- DefaultShuffleMetrics shuffleMetrics =
+ DefaultShuffleMetrics shuffleMetrics1 =
new DefaultShuffleMetrics(new ResultPartitionBytes(new long[] {1, 2, 3}));
- Collection<PartitionWithMetrics> defaultPartitionWithMetrics =
+ NettyShuffleDescriptor descriptor1 =
+ NettyShuffleDescriptorBuilder.newBuilder().buildLocal();
+ Collection<PartitionWithMetrics> defaultPartitionWithMetrics1 =
Collections.singletonList(
- new DefaultPartitionWithMetrics(shuffleDescriptor, shuffleMetrics));
- final TestingTaskExecutorGateway taskExecutorGateway =
+ new DefaultPartitionWithMetrics(descriptor1, shuffleMetrics1));
+
+ DefaultShuffleMetrics shuffleMetrics2 =
+ new DefaultShuffleMetrics(new ResultPartitionBytes(new long[] {4, 5, 6}));
+ NettyShuffleDescriptor descriptor2 =
+ NettyShuffleDescriptorBuilder.newBuilder().buildLocal();
+ Collection<PartitionWithMetrics> defaultPartitionWithMetrics2 =
+ Collections.singletonList(
+ new DefaultPartitionWithMetrics(descriptor2, shuffleMetrics2));
+
+ DefaultShuffleMetrics shuffleMetrics3 =
+ new DefaultShuffleMetrics(new ResultPartitionBytes(new long[] {7, 8, 9}));
+ NettyShuffleDescriptor descriptor3 =
+ NettyShuffleDescriptorBuilder.newBuilder().buildLocal();
+ Collection<PartitionWithMetrics> defaultPartitionWithMetrics3 =
+ Collections.singletonList(
+ new DefaultPartitionWithMetrics(descriptor3, shuffleMetrics3));
+
+ DefaultShuffleMetrics shuffleMetrics4 =
+ new DefaultShuffleMetrics(new ResultPartitionBytes(new long[] {10, 11}));
+ NettyShuffleDescriptor descriptor4 =
+ NettyShuffleDescriptorBuilder.newBuilder().buildLocal();
+ Collection<PartitionWithMetrics> defaultPartitionWithMetrics4 =
+ Collections.singletonList(
+ new DefaultPartitionWithMetrics(descriptor4, shuffleMetrics4));
+
+ // start fetch and retain partitions and then register tm1
+ final TestingTaskExecutorGateway taskExecutorGateway1 =
new TestingTaskExecutorGatewayBuilder()
.setRequestPartitionWithMetricsFunction(
ignored ->
CompletableFuture.completedFuture(
- defaultPartitionWithMetrics))
+ defaultPartitionWithMetrics1))
+ .setAddress("tm1")
.createTestingTaskExecutorGateway();
- final LocalUnresolvedTaskManagerLocation taskManagerUnresolvedLocation =
- new LocalUnresolvedTaskManagerLocation();
- final Collection<SlotOffer> slotOffers =
- registerSlotsAtJobMaster(
- 1,
- jobMasterGateway,
- jobGraph.getJobID(),
- taskExecutorGateway,
- taskManagerUnresolvedLocation);
- assertThat(slotOffers).hasSize(1);
+ registerSlotsAtJobMaster(
+ 1,
+ jobMasterGateway,
+ jobGraph.getJobID(),
+ taskExecutorGateway1,
+ new LocalUnresolvedTaskManagerLocation());
- waitUntilAllExecutionsAreScheduledOrDeployed(jobMasterGateway);
+ jobMaster.startFetchAndRetainPartitionWithMetricsOnTaskManager();
- PartitionWithMetrics metrics =
- jobMasterGateway
- .getAllPartitionWithMetricsOnTaskManagers()
- .get()
- .iterator()
- .next();
- PartitionWithMetrics expectedMetrics = defaultPartitionWithMetrics.iterator().next();
+ verifyPartitionMetrics(
+ jobMaster.getPartitionWithMetricsOnTaskManagers(),
+ defaultPartitionWithMetrics1);
+
+ // register tm2
+ TestingTaskExecutorGateway taskExecutorGateway2 =
+ new TestingTaskExecutorGatewayBuilder()
+ .setRequestPartitionWithMetricsFunction(
+ ignored ->
+ CompletableFuture.completedFuture(
+ defaultPartitionWithMetrics2))
+ .setAddress("tm2")
+ .createTestingTaskExecutorGateway();
+ registerSlotsAtJobMaster(
+ 1,
+ jobMasterGateway,
+ jobGraph.getJobID(),
+ taskExecutorGateway2,
+ new LocalUnresolvedTaskManagerLocation());
+
+ Collection<PartitionWithMetrics> expectedMetrics =
+ new ArrayList<>(defaultPartitionWithMetrics1);
+ expectedMetrics.addAll(defaultPartitionWithMetrics2);
+ verifyPartitionMetrics(
+ jobMaster.getPartitionWithMetricsOnTaskManagers(), expectedMetrics);
- assertThat(metrics.getPartitionMetrics().getPartitionBytes().getSubpartitionBytes())
+ // register tm3 which received fetch request but not response on time.
+ CompletableFuture<Tuple2<JobID, Set<ResultPartitionID>>> releaseRequest =
+ new CompletableFuture<>();
+ CompletableFuture<Collection<PartitionWithMetrics>> fetchPartitionsFuture =
+ new CompletableFuture<>();
+ TestingTaskExecutorGateway taskExecutorGateway3 =
+ new TestingTaskExecutorGatewayBuilder()
+ .setRequestPartitionWithMetricsFunction(
+ ignored -> fetchPartitionsFuture)
+ .setReleasePartitionsConsumer(
+ (id, partitions) ->
+ releaseRequest.complete(Tuple2.of(id, partitions)))
+ .setAddress("tm3")
+ .createTestingTaskExecutorGateway();
+
+ registerSlotsAtJobMaster(
+ 1,
+ jobMasterGateway,
+ jobGraph.getJobID(),
+ taskExecutorGateway3,
+ new LocalUnresolvedTaskManagerLocation());
+
+ // register tm4 which is not included in expected partitions
+ TestingTaskExecutorGateway taskExecutorGateway4 =
+ new TestingTaskExecutorGatewayBuilder()
+ .setRequestPartitionWithMetricsFunction(
+ ignored ->
+ CompletableFuture.completedFuture(
+ defaultPartitionWithMetrics4))
+ .setAddress("tm4")
+ .createTestingTaskExecutorGateway();
+ registerSlotsAtJobMaster(
+ 1,
+ jobMasterGateway,
+ jobGraph.getJobID(),
+ taskExecutorGateway4,
+ new LocalUnresolvedTaskManagerLocation());
+
+ Duration timeout = Duration.ofSeconds(10);
+ Set<ResultPartitionID> expectedResultPartitions = new HashSet<>();
+ expectedResultPartitions.add(descriptor1.getResultPartitionID());
+ expectedResultPartitions.add(descriptor2.getResultPartitionID());
+ expectedResultPartitions.add(descriptor4.getResultPartitionID());
+ CompletableFuture<Collection<PartitionWithMetrics>> future =
+ jobMasterGateway.getPartitionWithMetrics(timeout, expectedResultPartitions);
+
+ expectedMetrics = new ArrayList<>(defaultPartitionWithMetrics1);
+ expectedMetrics.addAll(defaultPartitionWithMetrics2);
+ expectedMetrics.addAll(defaultPartitionWithMetrics4);
+
+ assertThat(future).succeedsWithin(timeout);
+ verifyPartitionMetrics(
+ future.get().stream()
+ .collect(
+ Collectors.toMap(
+ metrics ->
+ metrics.getPartition().getResultPartitionID(),
+ metrics -> metrics)),
+ expectedMetrics);
+
+ // fetch partition after timeout
+ fetchPartitionsFuture.complete(defaultPartitionWithMetrics3);
+ assertThat(releaseRequest.get())
.isEqualTo(
- expectedMetrics
- .getPartitionMetrics()
- .getPartitionBytes()
- .getSubpartitionBytes());
- assertThat(metrics.getPartition().getResultPartitionID())
- .isEqualTo(expectedMetrics.getPartition().getResultPartitionID());
- assertThat(metrics.getPartition().isUnknown())
- .isEqualTo(expectedMetrics.getPartition().isUnknown());
- assertThat(metrics.getPartition().storesLocalResourcesOn())
- .isEqualTo(expectedMetrics.getPartition().storesLocalResourcesOn());
+ Tuple2.of(
+ jobGraph.getJobID(),
+ Collections.singleton(descriptor3.getResultPartitionID())));
+
+ // after partitions fetching finished and then register tm5
+ CompletableFuture<Collection<PartitionWithMetrics>> requestFuture =
+ new CompletableFuture<>();
+ TestingTaskExecutorGateway taskExecutorGateway5 =
+ new TestingTaskExecutorGatewayBuilder()
+ .setRequestPartitionWithMetricsFunction(
+ ignored -> {
+ requestFuture.complete(null);
+ return requestFuture;
+ })
+ .setAddress("tm5")
+ .createTestingTaskExecutorGateway();
+
+ registerSlotsAtJobMaster(
+ 1,
+ jobMasterGateway,
+ jobGraph.getJobID(),
+ taskExecutorGateway5,
+ new LocalUnresolvedTaskManagerLocation());
+
+ assertThatThrownBy(() -> requestFuture.get(timeout.toMillis(), TimeUnit.MILLISECONDS))
+ .isInstanceOf(TimeoutException.class);
}
}
+ private static void verifyPartitionMetrics(
+ Map<ResultPartitionID, PartitionWithMetrics> actualMap,
+ Collection<PartitionWithMetrics> expectedMetrics) {
+
+ Map<ResultPartitionID, PartitionWithMetrics> expectedMap =
+ expectedMetrics.stream()
+ .collect(
+ Collectors.toMap(
+ metrics -> metrics.getPartition().getResultPartitionID(),
+ metrics -> metrics));
+
+ assertThat(actualMap).hasSameSizeAs(expectedMap);
+
+ actualMap.forEach(
+ (k, v) -> {
+ PartitionWithMetrics metrics = expectedMap.get(k);
+
+ assertThat(metrics).isNotNull();
+ assertThat(
+ metrics.getPartitionMetrics()
+ .getPartitionBytes()
+ .getSubpartitionBytes())
+ .isEqualTo(
+ v.getPartitionMetrics()
+ .getPartitionBytes()
+ .getSubpartitionBytes());
+
+ assertThat(metrics.getPartition().isUnknown())
+ .isEqualTo(v.getPartition().isUnknown());
+
+ assertThat(metrics.getPartition().storesLocalResourcesOn())
+ .isEqualTo(v.getPartition().storesLocalResourcesOn());
+ });
+ }
+
@Test
void testBlockResourcesWillTriggerReleaseFreeSlots() throws Exception {
JobVertex jobVertex = new JobVertex("jobVertex");
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/TestingJobMasterGateway.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/TestingJobMasterGateway.java
index ecdc78f45a9..384ba882f97 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/TestingJobMasterGateway.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/TestingJobMasterGateway.java
@@ -55,6 +55,7 @@ import org.apache.flink.runtime.query.KvStateLocation;
import org.apache.flink.runtime.registration.RegistrationResponse;
import org.apache.flink.runtime.resourcemanager.ResourceManagerId;
import org.apache.flink.runtime.scheduler.ExecutionGraphInfo;
+import org.apache.flink.runtime.shuffle.PartitionWithMetrics;
import org.apache.flink.runtime.slots.ResourceRequirement;
import org.apache.flink.runtime.state.KeyGroupRange;
import org.apache.flink.runtime.taskexecutor.TaskExecutorToJobManagerHeartbeatPayload;
@@ -68,7 +69,9 @@ import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.net.InetSocketAddress;
+import java.time.Duration;
import java.util.Collection;
+import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.function.BiFunction;
import java.util.function.Consumer;
@@ -173,6 +176,13 @@ public class TestingJobMasterGateway implements JobMasterGateway {
CompletableFuture<Acknowledge>>
notifyKvStateUnregisteredFunction;
+ @Nonnull
+ private final BiFunction<
+ Duration,
+ Set<ResultPartitionID>,
+ CompletableFuture<Collection<PartitionWithMetrics>>>
+ getPartitionWithMetricsFunction;
+
@Nonnull TriFunction<String, Object, byte[], CompletableFuture<Object>> updateAggregateFunction;
@Nonnull
@@ -312,7 +322,13 @@ public class TestingJobMasterGateway implements JobMasterGateway {
requestJobResourceRequirementsSupplier,
@Nonnull
Function<JobResourceRequirements, CompletableFuture<Acknowledge>>
- updateJobResourceRequirementsFunction) {
+ updateJobResourceRequirementsFunction,
+ @Nonnull
+ BiFunction<
+ Duration,
+ Set<ResultPartitionID>,
+ CompletableFuture<Collection<PartitionWithMetrics>>>
+ getPartitionWithMetricsFunction) {
this.address = address;
this.hostname = hostname;
this.cancelFunction = cancelFunction;
@@ -345,6 +361,13 @@ public class TestingJobMasterGateway implements JobMasterGateway {
this.notifyNewBlockedNodesFunction = notifyNewBlockedNodesFunction;
this.requestJobResourceRequirementsSupplier = requestJobResourceRequirementsSupplier;
this.updateJobResourceRequirementsFunction = updateJobResourceRequirementsFunction;
+ this.getPartitionWithMetricsFunction = getPartitionWithMetricsFunction;
+ }
+
+ @Override
+ public CompletableFuture<Collection<PartitionWithMetrics>> getPartitionWithMetrics(
+ Duration timeout, Set<ResultPartitionID> expectedPartitions) {
+ return getPartitionWithMetricsFunction.apply(timeout, expectedPartitions);
}
@Override
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/TestingJobMasterGatewayBuilder.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/TestingJobMasterGatewayBuilder.java
index 6fbc367f800..4b5c2104571 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/TestingJobMasterGatewayBuilder.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/TestingJobMasterGatewayBuilder.java
@@ -54,6 +54,7 @@ import org.apache.flink.runtime.query.UnknownKvStateLocation;
import org.apache.flink.runtime.registration.RegistrationResponse;
import org.apache.flink.runtime.resourcemanager.ResourceManagerId;
import org.apache.flink.runtime.scheduler.ExecutionGraphInfo;
+import org.apache.flink.runtime.shuffle.PartitionWithMetrics;
import org.apache.flink.runtime.slots.ResourceRequirement;
import org.apache.flink.runtime.state.KeyGroupRange;
import org.apache.flink.runtime.taskexecutor.TaskExecutorToJobManagerHeartbeatPayload;
@@ -65,9 +66,11 @@ import org.apache.flink.util.function.TriConsumer;
import org.apache.flink.util.function.TriFunction;
import java.net.InetSocketAddress;
+import java.time.Duration;
import java.util.Collection;
import java.util.Collections;
import java.util.Map;
+import java.util.Set;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.function.BiFunction;
@@ -188,6 +191,13 @@ public class TestingJobMasterGatewayBuilder {
requestJobResourceRequirementsSupplier =
() -> CompletableFuture.completedFuture(JobResourceRequirements.empty());
+ private BiFunction<
+ Duration,
+ Set<ResultPartitionID>,
+ CompletableFuture<Collection<PartitionWithMetrics>>>
+ getPartitionWithMetricsFunction =
+ (timeout, set) -> CompletableFuture.completedFuture(Collections.emptyList());
+
private Function<JobResourceRequirements, CompletableFuture<Acknowledge>>
updateJobResourceRequirementsFunction =
ignored -> CompletableFuture.completedFuture(Acknowledge.get());
@@ -424,6 +434,16 @@ public class TestingJobMasterGatewayBuilder {
return this;
}
+ public TestingJobMasterGatewayBuilder setGetPartitionWithMetricsFunction(
+ BiFunction<
+ Duration,
+ Set<ResultPartitionID>,
+ CompletableFuture<Collection<PartitionWithMetrics>>>
+ getPartitionWithMetricsFunction) {
+ this.getPartitionWithMetricsFunction = getPartitionWithMetricsFunction;
+ return this;
+ }
+
public TestingJobMasterGatewayBuilder setUpdateJobResourceRequirementsFunction(
Function<JobResourceRequirements, CompletableFuture<Acknowledge>>
updateJobResourceRequirementsFunction) {
@@ -464,6 +484,7 @@ public class TestingJobMasterGatewayBuilder {
notifyNotEnoughResourcesConsumer,
notifyNewBlockedNodesFunction,
requestJobResourceRequirementsSupplier,
- updateJobResourceRequirementsFunction);
+ updateJobResourceRequirementsFunction,
+ getPartitionWithMetricsFunction);
}
}