You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2021/02/18 09:14:56 UTC
[flink] 08/12: [FLINK-21100][coordination] Adjust SharedSlot
lifecycle
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit a56ca8610125548593226da0c85426ac40c7dbe6
Author: Chesnay Schepler <ch...@apache.org>
AuthorDate: Mon Feb 8 15:09:52 2021 +0100
[FLINK-21100][coordination] Adjust SharedSlot lifecycle
The external-release callback is now only run once the last logical slot has been returned.
In practice the use-case for the callback is to bind a release call to the return of the last logical slot.
The current behavior however muddies this contract since it may also be called from #release; leading to cases where #release is called twice on a single slot.
This case got even worse when the slot was released before all logical slots have been returned; in this case all allocated logical slots are automatically being returned, triggering the callback, triggering another release call while the first one hasn't even finished yet.
---
.../declarative/allocator/SharedSlot.java | 39 +++++++++---
.../declarative/allocator/SharedSlotTest.java | 71 +++++++++++++++++++++-
2 files changed, 98 insertions(+), 12 deletions(-)
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/declarative/allocator/SharedSlot.java b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/declarative/allocator/SharedSlot.java
index 90cce74..5b41bad 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/declarative/allocator/SharedSlot.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/declarative/allocator/SharedSlot.java
@@ -34,7 +34,20 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-/** Shared slot implementation for the declarative scheduler. */
+/**
+ * Shared slot implementation for the declarative scheduler.
+ *
+ * <p>The release process of a shared slot follows one of 2 code paths:
+ *
+ * <p>1)During normal execution all allocated logical slots will be returned, with the last return
+ * triggering the {@code externalReleaseCallback} which must eventually result in a {@link
+ * #release(Throwable)} call. 2)
+ *
+ * <p>2) If the backing physical is lost (e.g., because the providing TaskManager crashed) then
+ * {@link #release(Throwable)} is called without all logical slots having been returned. The runtime
+ * relies on this also triggering the release of all logical slots. This will not trigger the {@code
+ * externalReleaseCallback}.
+ */
class SharedSlot implements SlotOwner, PhysicalSlot.Payload {
private static final Logger LOG = LoggerFactory.getLogger(SharedSlot.class);
@@ -94,7 +107,7 @@ class SharedSlot implements SlotOwner, PhysicalSlot.Payload {
public void returnLogicalSlot(LogicalSlot logicalSlot) {
LOG.debug("Returning logical slot to shared slot ({})", physicalSlotRequestId);
Preconditions.checkState(
- state == State.ALLOCATED, "The shared slot has already been released.");
+ state != State.RELEASED, "The shared slot has already been released.");
Preconditions.checkState(!logicalSlot.isAlive(), "Returned logic slot must not be alive.");
Preconditions.checkState(
@@ -103,30 +116,35 @@ class SharedSlot implements SlotOwner, PhysicalSlot.Payload {
tryReleaseExternally();
}
+ private void tryReleaseExternally() {
+ if (state == State.ALLOCATED && allocatedLogicalSlots.isEmpty()) {
+ LOG.debug("Release shared slot externally ({})", physicalSlotRequestId);
+ externalReleaseCallback.run();
+ }
+ }
+
@Override
public void release(Throwable cause) {
LOG.debug("Release shared slot ({})", physicalSlotRequestId);
Preconditions.checkState(
state == State.ALLOCATED, "The shared slot has already been released.");
+ // ensures that we won't call the external release callback if there are still logical slots
+ // to release
+ state = State.RELEASING;
+
// copy the logical slot collection to avoid ConcurrentModificationException
// if logical slot releases cause cancellation of other executions
// which will try to call returnLogicalSlot and modify allocatedLogicalSlots collection
final List<LogicalSlot> logicalSlotsToRelease =
new ArrayList<>(allocatedLogicalSlots.values());
for (LogicalSlot allocatedLogicalSlot : logicalSlotsToRelease) {
+ // this will also cause the logical slot to be returned
allocatedLogicalSlot.releaseSlot(cause);
}
allocatedLogicalSlots.clear();
- tryReleaseExternally();
- }
- private void tryReleaseExternally() {
- if (state != State.RELEASED && allocatedLogicalSlots.isEmpty()) {
- state = State.RELEASED;
- LOG.debug("Release shared slot externally ({})", physicalSlotRequestId);
- externalReleaseCallback.run();
- }
+ state = State.RELEASED;
}
@Override
@@ -136,6 +154,7 @@ class SharedSlot implements SlotOwner, PhysicalSlot.Payload {
private enum State {
ALLOCATED,
+ RELEASING,
RELEASED
}
}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/declarative/allocator/SharedSlotTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/declarative/allocator/SharedSlotTest.java
index 8989a99..bf83169 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/declarative/allocator/SharedSlotTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/declarative/allocator/SharedSlotTest.java
@@ -27,7 +27,9 @@ import org.apache.flink.util.TestLogger;
import org.junit.Test;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Consumer;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.is;
@@ -128,7 +130,7 @@ public class SharedSlotTest extends TestLogger {
}
@Test
- public void testReleaseTriggersExternalRelease() {
+ public void testReleaseDoesNotTriggersExternalRelease() {
final TestingPhysicalSlot physicalSlot = TestingPhysicalSlot.builder().build();
final AtomicBoolean externalReleaseInitiated = new AtomicBoolean(false);
final SharedSlot sharedSlot =
@@ -140,7 +142,7 @@ public class SharedSlotTest extends TestLogger {
sharedSlot.release(new Exception("test"));
- assertThat(externalReleaseInitiated.get(), is(true));
+ assertThat(externalReleaseInitiated.get(), is(false));
}
@Test
@@ -166,6 +168,48 @@ public class SharedSlotTest extends TestLogger {
sharedSlot.allocateLogicalSlot();
}
+ @Test
+ public void testCanReturnLogicalSlotDuringRelease() {
+ final TestingPhysicalSlot physicalSlot = TestingPhysicalSlot.builder().build();
+ final SharedSlot sharedSlot =
+ new SharedSlot(new SlotRequestId(), physicalSlot, false, () -> {});
+ final LogicalSlot logicalSlot1 = sharedSlot.allocateLogicalSlot();
+ final LogicalSlot logicalSlot2 = sharedSlot.allocateLogicalSlot();
+
+ // both slots try to release the other one, simulating that the failure of one execution due
+ // to the release also fails others
+ logicalSlot1.tryAssignPayload(
+ new TestLogicalSlotPayload(
+ cause -> {
+ if (logicalSlot2.isAlive()) {
+ logicalSlot2.releaseSlot(cause);
+ }
+ }));
+ logicalSlot2.tryAssignPayload(
+ new TestLogicalSlotPayload(
+ cause -> {
+ if (logicalSlot1.isAlive()) {
+ logicalSlot1.releaseSlot(cause);
+ }
+ }));
+
+ sharedSlot.release(new Exception("test"));
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void testCannotAllocateLogicalSlotDuringRelease() {
+ final TestingPhysicalSlot physicalSlot = TestingPhysicalSlot.builder().build();
+ final SharedSlot sharedSlot =
+ new SharedSlot(new SlotRequestId(), physicalSlot, false, () -> {});
+
+ final LogicalSlot logicalSlot = sharedSlot.allocateLogicalSlot();
+
+ logicalSlot.tryAssignPayload(
+ new TestLogicalSlotPayload(ignored -> sharedSlot.allocateLogicalSlot()));
+
+ sharedSlot.release(new Exception("test"));
+ }
+
private static class TestPhysicalSlotPayload implements PhysicalSlot.Payload {
@Override
@@ -176,4 +220,27 @@ public class SharedSlotTest extends TestLogger {
return false;
}
}
+
+ private static class TestLogicalSlotPayload implements LogicalSlot.Payload {
+
+ private final Consumer<Throwable> failConsumer;
+
+ public TestLogicalSlotPayload() {
+ this.failConsumer = ignored -> {};
+ }
+
+ public TestLogicalSlotPayload(Consumer<Throwable> failConsumer) {
+ this.failConsumer = failConsumer;
+ }
+
+ @Override
+ public void fail(Throwable cause) {
+ failConsumer.accept(cause);
+ }
+
+ @Override
+ public CompletableFuture<?> getTerminalStateFuture() {
+ return new CompletableFuture<>();
+ }
+ }
}