You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2020/01/17 14:35:52 UTC

[flink] branch release-1.10 updated: [FLINK-14163][runtime] Enforce synchronous registration of produced partitions

This is an automated email from the ASF dual-hosted git repository.

chesnay pushed a commit to branch release-1.10
in repository https://gitbox.apache.org/repos/asf/flink.git


The following commit(s) were added to refs/heads/release-1.10 by this push:
     new a16f7c7  [FLINK-14163][runtime] Enforce synchronous registration of produced partitions
a16f7c7 is described below

commit a16f7c7cf9c5e0d210591b93690784d593ff0910
Author: Yuan Mei <yu...@gmail.com>
AuthorDate: Thu Jan 16 12:12:56 2020 +0800

    [FLINK-14163][runtime] Enforce synchronous registration of produced partitions
---
 .../flink/runtime/executiongraph/Execution.java    | 24 ++++++++++
 .../flink/runtime/shuffle/ShuffleMaster.java       |  2 +
 .../runtime/executiongraph/ExecutionTest.java      | 53 ++++++++++++++++++++++
 3 files changed, 79 insertions(+)

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
index 3f9e631..3e11ac3 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
@@ -53,6 +53,7 @@ import org.apache.flink.runtime.jobmaster.LogicalSlot;
 import org.apache.flink.runtime.jobmaster.SlotRequestId;
 import org.apache.flink.runtime.messages.Acknowledge;
 import org.apache.flink.runtime.messages.TaskBackPressureResponse;
+import org.apache.flink.runtime.shuffle.NettyShuffleMaster;
 import org.apache.flink.runtime.shuffle.PartitionDescriptor;
 import org.apache.flink.runtime.shuffle.ProducerDescriptor;
 import org.apache.flink.runtime.shuffle.ShuffleDescriptor;
@@ -609,6 +610,26 @@ public class Execution implements AccessExecution, Archiveable<ArchivedExecution
 			});
 	}
 
+	/**
+	 * Register producedPartitions to {@link ShuffleMaster}
+	 *
+	 * <p>HACK: Please notice that this method simulates asynchronous registration in a synchronous way
+	 * by making sure the returned {@link CompletableFuture} from {@link ShuffleMaster#registerPartitionWithProducer}
+	 * is completed immediately.
+	 *
+	 * <p>{@link Execution#producedPartitions} are registered through an asynchronous interface
+	 * {@link ShuffleMaster#registerPartitionWithProducer} to {@link ShuffleMaster}, however they are not always
+	 * accessed through callbacks. So, it is possible that {@link Execution#producedPartitions}
+	 * have not been available yet when accessed (in {@link Execution#deploy} for example).
+	 *
+	 * <p>Since the only implementation of {@link ShuffleMaster} is {@link NettyShuffleMaster},
+	 * which indeed registers producedPartition in a synchronous way, this method enforces
+	 * synchronous registration under an asynchronous interface for now.
+	 *
+	 * <p>TODO: If asynchronous registration is needed in the future, use callbacks to access {@link Execution#producedPartitions}.
+	 *
+	 * @return completed future of partition deployment descriptors.
+	 */
 	@VisibleForTesting
 	static CompletableFuture<Map<IntermediateResultPartitionID, ResultPartitionDeploymentDescriptor>> registerProducedPartitions(
 			ExecutionVertex vertex,
@@ -630,6 +651,9 @@ public class Execution implements AccessExecution, Archiveable<ArchivedExecution
 				.getShuffleMaster()
 				.registerPartitionWithProducer(partitionDescriptor, producerDescriptor);
 
+			// temporary hack; the scheduler does not handle incomplete futures properly
+			Preconditions.checkState(shuffleDescriptorFuture.isDone(), "ShuffleDescriptor future is incomplete.");
+
 			CompletableFuture<ResultPartitionDeploymentDescriptor> partitionRegistration = shuffleDescriptorFuture
 				.thenApply(shuffleDescriptor -> new ResultPartitionDeploymentDescriptor(
 					partitionDescriptor,
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/ShuffleMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/ShuffleMaster.java
index 9f729c9..439e927 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/ShuffleMaster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/shuffle/ShuffleMaster.java
@@ -31,6 +31,8 @@ public interface ShuffleMaster<T extends ShuffleDescriptor> {
 	/**
 	 * Asynchronously register a partition and its producer with the shuffle service.
 	 *
+	 * <p>IMPORTANT: the returned future must be completed due to limitations in the default scheduler.
+	 *
 	 * <p>The returned shuffle descriptor is an internal handle
 	 * which identifies the partition internally within the shuffle service.
 	 * The descriptor should provide enough information to read from or write data to the partition.
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionTest.java
index 3be8657..42f0f48 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionTest.java
@@ -25,8 +25,10 @@ import org.apache.flink.runtime.concurrent.ComponentMainThreadExecutorServiceAda
 import org.apache.flink.runtime.execution.ExecutionState;
 import org.apache.flink.runtime.executiongraph.restart.NoRestartStrategy;
 import org.apache.flink.runtime.executiongraph.utils.SimpleAckingTaskManagerGateway;
+import org.apache.flink.runtime.jobgraph.JobGraph;
 import org.apache.flink.runtime.jobgraph.JobVertex;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
 import org.apache.flink.runtime.jobmanager.scheduler.LocationPreferenceConstraint;
 import org.apache.flink.runtime.jobmanager.scheduler.SchedulerTestUtils;
 import org.apache.flink.runtime.jobmaster.LogicalSlot;
@@ -36,6 +38,10 @@ import org.apache.flink.runtime.jobmaster.TestingLogicalSlot;
 import org.apache.flink.runtime.jobmaster.TestingLogicalSlotBuilder;
 import org.apache.flink.runtime.jobmaster.slotpool.SingleLogicalSlot;
 import org.apache.flink.runtime.jobmaster.slotpool.SlotProvider;
+import org.apache.flink.runtime.shuffle.PartitionDescriptor;
+import org.apache.flink.runtime.shuffle.ProducerDescriptor;
+import org.apache.flink.runtime.shuffle.ShuffleDescriptor;
+import org.apache.flink.runtime.shuffle.ShuffleMaster;
 import org.apache.flink.runtime.taskmanager.LocalTaskManagerLocation;
 import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
 import org.apache.flink.runtime.testtasks.NoOpInvokable;
@@ -55,6 +61,8 @@ import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
 
+import static org.apache.flink.runtime.io.network.partition.ResultPartitionType.PIPELINED;
+import static org.apache.flink.runtime.jobgraph.DistributionPattern.POINTWISE;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
@@ -540,6 +548,51 @@ public class ExecutionTest extends TestLogger {
 		});
 	}
 
+	/**
+	 * Tests that incomplete futures returned by {@link ShuffleMaster#registerPartitionWithProducer} are rejected.
+	 */
+	@Test
+	public void testIncompletePartitionRegistrationFutureIsRejected() throws Exception {
+		final ShuffleMaster<ShuffleDescriptor> shuffleMaster = new TestingShuffleMaster();
+		final JobGraph jobGraph = new JobGraph("job graph");
+		final JobVertex source = new JobVertex("source");
+		final JobVertex target = new JobVertex("target");
+
+		source.setInvokableClass(AbstractInvokable.class);
+		target.setInvokableClass(AbstractInvokable.class);
+		target.connectNewDataSetAsInput(source, POINTWISE, PIPELINED);
+		jobGraph.addVertex(source);
+		jobGraph.addVertex(target);
+		ExecutionGraph executionGraph = TestingExecutionGraphBuilder
+			.newBuilder()
+			.setJobGraph(jobGraph)
+			.setShuffleMaster(shuffleMaster)
+			.build();
+
+		final ExecutionVertex sourceVertex = executionGraph.getAllVertices().get(source.getID()).getTaskVertices()[0];
+
+		boolean incompletePartitionRegistrationRejected = false;
+		try {
+			Execution.registerProducedPartitions(sourceVertex, new LocalTaskManagerLocation(), new ExecutionAttemptID(), false);
+		} catch (IllegalStateException e) {
+			incompletePartitionRegistrationRejected = true;
+		}
+
+		assertTrue(incompletePartitionRegistrationRejected);
+	}
+
+	private static class TestingShuffleMaster implements ShuffleMaster<ShuffleDescriptor> {
+
+		@Override
+		public CompletableFuture<ShuffleDescriptor> registerPartitionWithProducer(PartitionDescriptor partitionDescriptor, ProducerDescriptor producerDescriptor) {
+			return new CompletableFuture<>();
+		}
+
+		@Override
+		public void releasePartitionExternally(ShuffleDescriptor shuffleDescriptor) {
+		}
+	}
+
 	@Nonnull
 	private JobVertex createNoOpJobVertex() {
 		final JobVertex jobVertex = new JobVertex("Test vertex", new JobVertexID());