You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2020/01/17 14:35:52 UTC
[flink] branch release-1.10 updated: [FLINK-14163][runtime] Enforce
synchronous registration of produced partitions
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a commit to branch release-1.10
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/release-1.10 by this push:
new a16f7c7 [FLINK-14163][runtime] Enforce synchronous registration of produced partitions
a16f7c7 is described below
commit a16f7c7cf9c5e0d210591b93690784d593ff0910
Author: Yuan Mei <yu...@gmail.com>
AuthorDate: Thu Jan 16 12:12:56 2020 +0800
[FLINK-14163][runtime] Enforce synchronous registration of produced partitions
---
.../flink/runtime/executiongraph/Execution.java | 24 ++++++++++
.../flink/runtime/shuffle/ShuffleMaster.java | 2 +
.../runtime/executiongraph/ExecutionTest.java | 53 ++++++++++++++++++++++
3 files changed, 79 insertions(+)
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
index 3f9e631..3e11ac3 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
@@ -53,6 +53,7 @@ import org.apache.flink.runtime.jobmaster.LogicalSlot;
import org.apache.flink.runtime.jobmaster.SlotRequestId;
import org.apache.flink.runtime.messages.Acknowledge;
import org.apache.flink.runtime.messages.TaskBackPressureResponse;
+import org.apache.flink.runtime.shuffle.NettyShuffleMaster;
import org.apache.flink.runtime.shuffle.PartitionDescriptor;
import org.apache.flink.runtime.shuffle.ProducerDescriptor;
import org.apache.flink.runtime.shuffle.ShuffleDescriptor;
@@ -609,6 +610,26 @@ public class Execution implements AccessExecution, Archiveable<ArchivedExecution
});
}
+ /**
+ * Register producedPartitions to {@link ShuffleMaster}
+ *
+ * <p>HACK: Please notice that this method simulates asynchronous registration in a synchronous way
+ * by making sure the returned {@link CompletableFuture} from {@link ShuffleMaster#registerPartitionWithProducer}
+ * is completed immediately.
+ *
+ * <p>{@link Execution#producedPartitions} are registered through an asynchronous interface
+ * {@link ShuffleMaster#registerPartitionWithProducer} to {@link ShuffleMaster}, however they are not always
+ * accessed through callbacks. So, it is possible that {@link Execution#producedPartitions}
+ * have not been available yet when accessed (in {@link Execution#deploy} for example).
+ *
+ * <p>Since the only implementation of {@link ShuffleMaster} is {@link NettyShuffleMaster},
+ * which indeed registers producedPartition in a synchronous way, this method enforces
+ * synchronous registration under an asynchronous interface for now.
+ *
+ * <p>TODO: If asynchronous registration is needed in the future, use callbacks to access {@link Execution#producedPartitions}.
+ *
+ * @return completed future of partition deployment descriptors.
+ */
@VisibleForTesting
static CompletableFuture<Map<IntermediateResultPartitionID, ResultPartitionDeploymentDescriptor>> registerProducedPartitions(
ExecutionVertex vertex,
@@ -630,6 +651,9 @@ public class Execution implements AccessExecution, Archiveable<ArchivedExecution
.getShuffleMaster()
.registerPartitionWithProducer(partitionDescriptor, producerDescriptor);
+ // temporary hack; the scheduler does not handle incomplete futures properly
+ Preconditions.checkState(shuffleDescriptorFuture.isDone(), "ShuffleDescriptor future is incomplete.");
+
CompletableFuture<ResultPartitionDeploymentDescriptor> partitionRegistration = shuffleDescriptorFuture
.thenApply(shuffleDescriptor -> new ResultPartitionDeploymentDescriptor(
partitionDescriptor,
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/ShuffleMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/ShuffleMaster.java
index 9f729c9..439e927 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/ShuffleMaster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/ShuffleMaster.java
@@ -31,6 +31,8 @@ public interface ShuffleMaster<T extends ShuffleDescriptor> {
/**
* Asynchronously register a partition and its producer with the shuffle service.
*
+ * <p>IMPORTANT: the returned future must be completed due to limitations in the default scheduler.
+ *
* <p>The returned shuffle descriptor is an internal handle
* which identifies the partition internally within the shuffle service.
* The descriptor should provide enough information to read from or write data to the partition.
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionTest.java
index 3be8657..42f0f48 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionTest.java
@@ -25,8 +25,10 @@ import org.apache.flink.runtime.concurrent.ComponentMainThreadExecutorServiceAda
import org.apache.flink.runtime.execution.ExecutionState;
import org.apache.flink.runtime.executiongraph.restart.NoRestartStrategy;
import org.apache.flink.runtime.executiongraph.utils.SimpleAckingTaskManagerGateway;
+import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.jobgraph.JobVertex;
import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
import org.apache.flink.runtime.jobmanager.scheduler.LocationPreferenceConstraint;
import org.apache.flink.runtime.jobmanager.scheduler.SchedulerTestUtils;
import org.apache.flink.runtime.jobmaster.LogicalSlot;
@@ -36,6 +38,10 @@ import org.apache.flink.runtime.jobmaster.TestingLogicalSlot;
import org.apache.flink.runtime.jobmaster.TestingLogicalSlotBuilder;
import org.apache.flink.runtime.jobmaster.slotpool.SingleLogicalSlot;
import org.apache.flink.runtime.jobmaster.slotpool.SlotProvider;
+import org.apache.flink.runtime.shuffle.PartitionDescriptor;
+import org.apache.flink.runtime.shuffle.ProducerDescriptor;
+import org.apache.flink.runtime.shuffle.ShuffleDescriptor;
+import org.apache.flink.runtime.shuffle.ShuffleMaster;
import org.apache.flink.runtime.taskmanager.LocalTaskManagerLocation;
import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
import org.apache.flink.runtime.testtasks.NoOpInvokable;
@@ -55,6 +61,8 @@ import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
+import static org.apache.flink.runtime.io.network.partition.ResultPartitionType.PIPELINED;
+import static org.apache.flink.runtime.jobgraph.DistributionPattern.POINTWISE;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
@@ -540,6 +548,51 @@ public class ExecutionTest extends TestLogger {
});
}
+ /**
+ * Tests that incomplete futures returned by {@link ShuffleMaster#registerPartitionWithProducer} are rejected.
+ */
+ @Test
+ public void testIncompletePartitionRegistrationFutureIsRejected() throws Exception {
+ final ShuffleMaster<ShuffleDescriptor> shuffleMaster = new TestingShuffleMaster();
+ final JobGraph jobGraph = new JobGraph("job graph");
+ final JobVertex source = new JobVertex("source");
+ final JobVertex target = new JobVertex("target");
+
+ source.setInvokableClass(AbstractInvokable.class);
+ target.setInvokableClass(AbstractInvokable.class);
+ target.connectNewDataSetAsInput(source, POINTWISE, PIPELINED);
+ jobGraph.addVertex(source);
+ jobGraph.addVertex(target);
+ ExecutionGraph executionGraph = TestingExecutionGraphBuilder
+ .newBuilder()
+ .setJobGraph(jobGraph)
+ .setShuffleMaster(shuffleMaster)
+ .build();
+
+ final ExecutionVertex sourceVertex = executionGraph.getAllVertices().get(source.getID()).getTaskVertices()[0];
+
+ boolean incompletePartitionRegistrationRejected = false;
+ try {
+ Execution.registerProducedPartitions(sourceVertex, new LocalTaskManagerLocation(), new ExecutionAttemptID(), false);
+ } catch (IllegalStateException e) {
+ incompletePartitionRegistrationRejected = true;
+ }
+
+ assertTrue(incompletePartitionRegistrationRejected);
+ }
+
+ private static class TestingShuffleMaster implements ShuffleMaster<ShuffleDescriptor> {
+
+ @Override
+ public CompletableFuture<ShuffleDescriptor> registerPartitionWithProducer(PartitionDescriptor partitionDescriptor, ProducerDescriptor producerDescriptor) {
+ return new CompletableFuture<>();
+ }
+
+ @Override
+ public void releasePartitionExternally(ShuffleDescriptor shuffleDescriptor) {
+ }
+ }
+
@Nonnull
private JobVertex createNoOpJobVertex() {
final JobVertex jobVertex = new JobVertex("Test vertex", new JobVertexID());