You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2021/02/18 09:14:56 UTC

[flink] 08/12: [FLINK-21100][coordination] Adjust SharedSlot lifecycle

This is an automated email from the ASF dual-hosted git repository.

chesnay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit a56ca8610125548593226da0c85426ac40c7dbe6
Author: Chesnay Schepler <ch...@apache.org>
AuthorDate: Mon Feb 8 15:09:52 2021 +0100

    [FLINK-21100][coordination] Adjust SharedSlot lifecycle
    
    The external-release callback is now only run once the last logical slot has been returned.
    
    In practice the use-case for the callback is to bind a release call to the return of the last logical slot.
    The current behavior however muddies this contract since it may also be called from #release; leading to cases where #release is called twice on a single slot.
    
    This case got even worse when the slot was released before all logical slots have been returned; in this case all allocated logical slots are automatically being returned, triggering the callback, triggering another release call while the first one hasn't even finished yet.
---
 .../declarative/allocator/SharedSlot.java          | 39 +++++++++---
 .../declarative/allocator/SharedSlotTest.java      | 71 +++++++++++++++++++++-
 2 files changed, 98 insertions(+), 12 deletions(-)

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/declarative/allocator/SharedSlot.java b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/declarative/allocator/SharedSlot.java
index 90cce74..5b41bad 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/declarative/allocator/SharedSlot.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/declarative/allocator/SharedSlot.java
@@ -34,7 +34,20 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-/** Shared slot implementation for the declarative scheduler. */
+/**
+ * Shared slot implementation for the declarative scheduler.
+ *
+ * <p>The release process of a shared slot follows one of 2 code paths:
+ *
+ * <p>1)During normal execution all allocated logical slots will be returned, with the last return
+ * triggering the {@code externalReleaseCallback} which must eventually result in a {@link
+ * #release(Throwable)} call. 2)
+ *
+ * <p>2) If the backing physical is lost (e.g., because the providing TaskManager crashed) then
+ * {@link #release(Throwable)} is called without all logical slots having been returned. The runtime
+ * relies on this also triggering the release of all logical slots. This will not trigger the {@code
+ * externalReleaseCallback}.
+ */
 class SharedSlot implements SlotOwner, PhysicalSlot.Payload {
     private static final Logger LOG = LoggerFactory.getLogger(SharedSlot.class);
 
@@ -94,7 +107,7 @@ class SharedSlot implements SlotOwner, PhysicalSlot.Payload {
     public void returnLogicalSlot(LogicalSlot logicalSlot) {
         LOG.debug("Returning logical slot to shared slot ({})", physicalSlotRequestId);
         Preconditions.checkState(
-                state == State.ALLOCATED, "The shared slot has already been released.");
+                state != State.RELEASED, "The shared slot has already been released.");
 
         Preconditions.checkState(!logicalSlot.isAlive(), "Returned logic slot must not be alive.");
         Preconditions.checkState(
@@ -103,30 +116,35 @@ class SharedSlot implements SlotOwner, PhysicalSlot.Payload {
         tryReleaseExternally();
     }
 
+    private void tryReleaseExternally() {
+        if (state == State.ALLOCATED && allocatedLogicalSlots.isEmpty()) {
+            LOG.debug("Release shared slot externally ({})", physicalSlotRequestId);
+            externalReleaseCallback.run();
+        }
+    }
+
     @Override
     public void release(Throwable cause) {
         LOG.debug("Release shared slot ({})", physicalSlotRequestId);
         Preconditions.checkState(
                 state == State.ALLOCATED, "The shared slot has already been released.");
 
+        // ensures that we won't call the external release callback if there are still logical slots
+        // to release
+        state = State.RELEASING;
+
         // copy the logical slot collection to avoid ConcurrentModificationException
         // if logical slot releases cause cancellation of other executions
         // which will try to call returnLogicalSlot and modify allocatedLogicalSlots collection
         final List<LogicalSlot> logicalSlotsToRelease =
                 new ArrayList<>(allocatedLogicalSlots.values());
         for (LogicalSlot allocatedLogicalSlot : logicalSlotsToRelease) {
+            // this will also cause the logical slot to be returned
             allocatedLogicalSlot.releaseSlot(cause);
         }
         allocatedLogicalSlots.clear();
-        tryReleaseExternally();
-    }
 
-    private void tryReleaseExternally() {
-        if (state != State.RELEASED && allocatedLogicalSlots.isEmpty()) {
-            state = State.RELEASED;
-            LOG.debug("Release shared slot externally ({})", physicalSlotRequestId);
-            externalReleaseCallback.run();
-        }
+        state = State.RELEASED;
     }
 
     @Override
@@ -136,6 +154,7 @@ class SharedSlot implements SlotOwner, PhysicalSlot.Payload {
 
     private enum State {
         ALLOCATED,
+        RELEASING,
         RELEASED
     }
 }
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/declarative/allocator/SharedSlotTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/declarative/allocator/SharedSlotTest.java
index 8989a99..bf83169 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/declarative/allocator/SharedSlotTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/declarative/allocator/SharedSlotTest.java
@@ -27,7 +27,9 @@ import org.apache.flink.util.TestLogger;
 
 import org.junit.Test;
 
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Consumer;
 
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.is;
@@ -128,7 +130,7 @@ public class SharedSlotTest extends TestLogger {
     }
 
     @Test
-    public void testReleaseTriggersExternalRelease() {
+    public void testReleaseDoesNotTriggersExternalRelease() {
         final TestingPhysicalSlot physicalSlot = TestingPhysicalSlot.builder().build();
         final AtomicBoolean externalReleaseInitiated = new AtomicBoolean(false);
         final SharedSlot sharedSlot =
@@ -140,7 +142,7 @@ public class SharedSlotTest extends TestLogger {
 
         sharedSlot.release(new Exception("test"));
 
-        assertThat(externalReleaseInitiated.get(), is(true));
+        assertThat(externalReleaseInitiated.get(), is(false));
     }
 
     @Test
@@ -166,6 +168,48 @@ public class SharedSlotTest extends TestLogger {
         sharedSlot.allocateLogicalSlot();
     }
 
+    @Test
+    public void testCanReturnLogicalSlotDuringRelease() {
+        final TestingPhysicalSlot physicalSlot = TestingPhysicalSlot.builder().build();
+        final SharedSlot sharedSlot =
+                new SharedSlot(new SlotRequestId(), physicalSlot, false, () -> {});
+        final LogicalSlot logicalSlot1 = sharedSlot.allocateLogicalSlot();
+        final LogicalSlot logicalSlot2 = sharedSlot.allocateLogicalSlot();
+
+        // both slots try to release the other one, simulating that the failure of one execution due
+        // to the release also fails others
+        logicalSlot1.tryAssignPayload(
+                new TestLogicalSlotPayload(
+                        cause -> {
+                            if (logicalSlot2.isAlive()) {
+                                logicalSlot2.releaseSlot(cause);
+                            }
+                        }));
+        logicalSlot2.tryAssignPayload(
+                new TestLogicalSlotPayload(
+                        cause -> {
+                            if (logicalSlot1.isAlive()) {
+                                logicalSlot1.releaseSlot(cause);
+                            }
+                        }));
+
+        sharedSlot.release(new Exception("test"));
+    }
+
+    @Test(expected = IllegalStateException.class)
+    public void testCannotAllocateLogicalSlotDuringRelease() {
+        final TestingPhysicalSlot physicalSlot = TestingPhysicalSlot.builder().build();
+        final SharedSlot sharedSlot =
+                new SharedSlot(new SlotRequestId(), physicalSlot, false, () -> {});
+
+        final LogicalSlot logicalSlot = sharedSlot.allocateLogicalSlot();
+
+        logicalSlot.tryAssignPayload(
+                new TestLogicalSlotPayload(ignored -> sharedSlot.allocateLogicalSlot()));
+
+        sharedSlot.release(new Exception("test"));
+    }
+
     private static class TestPhysicalSlotPayload implements PhysicalSlot.Payload {
 
         @Override
@@ -176,4 +220,27 @@ public class SharedSlotTest extends TestLogger {
             return false;
         }
     }
+
+    private static class TestLogicalSlotPayload implements LogicalSlot.Payload {
+
+        private final Consumer<Throwable> failConsumer;
+
+        public TestLogicalSlotPayload() {
+            this.failConsumer = ignored -> {};
+        }
+
+        public TestLogicalSlotPayload(Consumer<Throwable> failConsumer) {
+            this.failConsumer = failConsumer;
+        }
+
+        @Override
+        public void fail(Throwable cause) {
+            failConsumer.accept(cause);
+        }
+
+        @Override
+        public CompletableFuture<?> getTerminalStateFuture() {
+            return new CompletableFuture<>();
+        }
+    }
 }