You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ma...@apache.org on 2023/07/31 07:11:58 UTC

[flink] 01/03: [FLINK-32583][rest] Fix deadlock in RestClient

This is an automated email from the ASF dual-hosted git repository.

mapohl pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit b2920728a20c99e064c0553f2ec74c190199971f
Author: Patrick Lucas <me...@patricklucas.com>
AuthorDate: Wed Jul 12 14:56:22 2023 +0200

    [FLINK-32583][rest] Fix deadlock in RestClient
---
 .../org/apache/flink/runtime/rest/RestClient.java  |  63 ++++++++++-
 .../apache/flink/runtime/rest/RestClientTest.java  | 124 +++++++++++++++++++++
 2 files changed, 185 insertions(+), 2 deletions(-)

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestClient.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestClient.java
index ff67c6b4645..949c0badb60 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestClient.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestClient.java
@@ -59,6 +59,8 @@ import org.apache.flink.shaded.netty4.io.netty.channel.ChannelHandler;
 import org.apache.flink.shaded.netty4.io.netty.channel.ChannelHandlerContext;
 import org.apache.flink.shaded.netty4.io.netty.channel.ChannelInitializer;
 import org.apache.flink.shaded.netty4.io.netty.channel.ChannelOption;
+import org.apache.flink.shaded.netty4.io.netty.channel.DefaultSelectStrategyFactory;
+import org.apache.flink.shaded.netty4.io.netty.channel.SelectStrategyFactory;
 import org.apache.flink.shaded.netty4.io.netty.channel.SimpleChannelInboundHandler;
 import org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoopGroup;
 import org.apache.flink.shaded.netty4.io.netty.channel.socket.SocketChannel;
@@ -93,6 +95,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.StringWriter;
 import java.net.URL;
+import java.nio.channels.spi.SelectorProvider;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
@@ -104,6 +107,7 @@ import java.util.List;
 import java.util.Optional;
 import java.util.ServiceLoader;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Executor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -131,6 +135,11 @@ public class RestClient implements AutoCloseableAsync {
 
     private final String urlPrefix;
 
+    // Used to track unresolved request futures in case they need to be resolved when the client is
+    // closed
+    private final Collection<CompletableFuture<Channel>> responseChannelFutures =
+            ConcurrentHashMap.newKeySet();
+
     @VisibleForTesting List<OutboundChannelHandlerFactory> outboundChannelHandlerFactories;
 
     /**
@@ -155,6 +164,25 @@ public class RestClient implements AutoCloseableAsync {
 
     public RestClient(Configuration configuration, Executor executor, String host, int port)
             throws ConfigurationException {
+        this(configuration, executor, host, port, DefaultSelectStrategyFactory.INSTANCE);
+    }
+
+    @VisibleForTesting
+    RestClient(
+            Configuration configuration,
+            Executor executor,
+            SelectStrategyFactory selectStrategyFactory)
+            throws ConfigurationException {
+        this(configuration, executor, null, -1, selectStrategyFactory);
+    }
+
+    private RestClient(
+            Configuration configuration,
+            Executor executor,
+            String host,
+            int port,
+            SelectStrategyFactory selectStrategyFactory)
+            throws ConfigurationException {
         Preconditions.checkNotNull(configuration);
         this.executor = Preconditions.checkNotNull(executor);
         this.terminationFuture = new CompletableFuture<>();
@@ -233,8 +261,16 @@ public class RestClient implements AutoCloseableAsync {
                         }
                     }
                 };
+
+        // No NioEventLoopGroup constructor available that allows passing nThreads, threadFactory,
+        // and selectStrategyFactory without also passing a SelectorProvider, so mimicking its
+        // default value seen in other constructors
         NioEventLoopGroup group =
-                new NioEventLoopGroup(1, new ExecutorThreadFactory("flink-rest-client-netty"));
+                new NioEventLoopGroup(
+                        1,
+                        new ExecutorThreadFactory("flink-rest-client-netty"),
+                        SelectorProvider.provider(),
+                        selectStrategyFactory);
 
         bootstrap = new Bootstrap();
         bootstrap
@@ -248,6 +284,11 @@ public class RestClient implements AutoCloseableAsync {
         LOG.debug("Rest client endpoint started.");
     }
 
+    @VisibleForTesting
+    Collection<CompletableFuture<Channel>> getResponseChannelFutures() {
+        return responseChannelFutures;
+    }
+
     @Override
     public CompletableFuture<Void> closeAsync() {
         return shutdownInternally(Time.seconds(10L));
@@ -276,6 +317,8 @@ public class RestClient implements AutoCloseableAsync {
                             .shutdownGracefully(0L, timeout.toMilliseconds(), TimeUnit.MILLISECONDS)
                             .addListener(
                                     finished -> {
+                                        notifyResponseFuturesOfShutdown();
+
                                         if (finished.isSuccess()) {
                                             terminationFuture.complete(null);
                                         } else {
@@ -289,6 +332,15 @@ public class RestClient implements AutoCloseableAsync {
         return terminationFuture;
     }
 
+    private void notifyResponseFuturesOfShutdown() {
+        responseChannelFutures.forEach(
+                future ->
+                        future.completeExceptionally(
+                                new IllegalStateException(
+                                        "RestClient closed before request completed")));
+        responseChannelFutures.clear();
+    }
+
     public <
                     M extends MessageHeaders<EmptyRequestBody, P, EmptyMessageParameters>,
                     P extends ResponseBody>
@@ -514,12 +566,19 @@ public class RestClient implements AutoCloseableAsync {
 
     private <P extends ResponseBody> CompletableFuture<P> submitRequest(
             String targetAddress, int targetPort, Request httpRequest, JavaType responseType) {
-        final ChannelFuture connectFuture = bootstrap.connect(targetAddress, targetPort);
+        if (!isRunning.get()) {
+            return FutureUtils.completedExceptionally(
+                    new IllegalStateException("RestClient is already closed"));
+        }
 
         final CompletableFuture<Channel> channelFuture = new CompletableFuture<>();
+        responseChannelFutures.add(channelFuture);
 
+        final ChannelFuture connectFuture = bootstrap.connect(targetAddress, targetPort);
         connectFuture.addListener(
                 (ChannelFuture future) -> {
+                    responseChannelFutures.remove(channelFuture);
+
                     if (future.isSuccess()) {
                         channelFuture.complete(future.channel());
                     } else {
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestClientTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestClientTest.java
index cc99eeeca2c..20d49a9edc4 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestClientTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestClientTest.java
@@ -20,6 +20,7 @@ package org.apache.flink.runtime.rest;
 
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.RestOptions;
+import org.apache.flink.core.testutils.OneShotLatch;
 import org.apache.flink.runtime.rest.messages.EmptyMessageParameters;
 import org.apache.flink.runtime.rest.messages.EmptyRequestBody;
 import org.apache.flink.runtime.rest.messages.EmptyResponseBody;
@@ -33,12 +34,17 @@ import org.apache.flink.util.TestLogger;
 import org.apache.flink.util.concurrent.Executors;
 import org.apache.flink.util.function.CheckedSupplier;
 
+import org.apache.flink.shaded.netty4.io.netty.channel.Channel;
 import org.apache.flink.shaded.netty4.io.netty.channel.ConnectTimeoutException;
+import org.apache.flink.shaded.netty4.io.netty.channel.DefaultSelectStrategyFactory;
+import org.apache.flink.shaded.netty4.io.netty.channel.SelectStrategy;
+import org.apache.flink.shaded.netty4.io.netty.channel.SelectStrategyFactory;
 import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpResponseStatus;
 
 import org.junit.Assert;
 import org.junit.ClassRule;
 import org.junit.Test;
+import org.junit.function.ThrowingRunnable;
 
 import java.io.IOException;
 import java.net.ServerSocket;
@@ -51,8 +57,12 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertThrows;
 
 /** Tests for {@link RestClient}. */
 public class RestClientTest extends TestLogger {
@@ -207,6 +217,120 @@ public class RestClientTest extends TestLogger {
         }
     }
 
+    /**
+     * Tests that the futures returned by {@link RestClient} fail immediately if the client is
+     * already closed.
+     *
+     * <p>See FLINK-32583
+     */
+    @Test
+    public void testCloseClientBeforeRequest() throws Exception {
+        try (final RestClient restClient =
+                new RestClient(new Configuration(), Executors.directExecutor())) {
+            restClient.close(); // Intentionally close the client prior to the request
+
+            CompletableFuture<?> future =
+                    restClient.sendRequest(
+                            unroutableIp,
+                            80,
+                            new TestMessageHeaders(),
+                            EmptyMessageParameters.getInstance(),
+                            EmptyRequestBody.getInstance());
+
+            // Call get() on the future with a timeout of 0s so we can test that the exception
+            // thrown is not a TimeoutException, which is what would be thrown if restClient were
+            // not already closed
+            final ThrowingRunnable getFuture = () -> future.get(0, TimeUnit.SECONDS);
+
+            final Throwable cause = assertThrows(ExecutionException.class, getFuture).getCause();
+            assertThat(cause, instanceOf(IllegalStateException.class));
+            assertThat(cause.getMessage(), equalTo("RestClient is already closed"));
+        }
+    }
+
+    @Test
+    public void testCloseClientWhileProcessingRequest() throws Exception {
+        // Set up a Netty SelectStrategy with latches that allow us to step forward through Netty's
+        // request state machine, closing the client at a particular moment
+        final OneShotLatch connectTriggered = new OneShotLatch();
+        final OneShotLatch closeTriggered = new OneShotLatch();
+        final SelectStrategy fallbackSelectStrategy =
+                DefaultSelectStrategyFactory.INSTANCE.newSelectStrategy();
+        final SelectStrategyFactory selectStrategyFactory =
+                () ->
+                        (selectSupplier, hasTasks) -> {
+                            connectTriggered.trigger();
+                            closeTriggered.awaitQuietly();
+
+                            return fallbackSelectStrategy.calculateStrategy(
+                                    selectSupplier, hasTasks);
+                        };
+
+        try (final RestClient restClient =
+                new RestClient(
+                        new Configuration(), Executors.directExecutor(), selectStrategyFactory)) {
+            // Check that client's internal collection of pending response futures is empty prior to
+            // the request
+            assertThat(restClient.getResponseChannelFutures(), empty());
+
+            final CompletableFuture<?> requestFuture =
+                    restClient.sendRequest(
+                            unroutableIp,
+                            80,
+                            new TestMessageHeaders(),
+                            EmptyMessageParameters.getInstance(),
+                            EmptyRequestBody.getInstance());
+
+            // Check that client's internal collection of pending response futures now has one
+            // entry, presumably due to the call to sendRequest
+            assertThat(restClient.getResponseChannelFutures(), hasSize(1));
+
+            // Wait for Netty to start connecting, then while it's paused in the SelectStrategy,
+            // close the client before unpausing Netty
+            connectTriggered.await();
+            final CompletableFuture<Void> closeFuture = restClient.closeAsync();
+            closeTriggered.trigger();
+
+            // Close should complete successfully
+            closeFuture.get();
+
+            final Throwable cause =
+                    assertThrows(
+                                    ExecutionException.class,
+                                    () -> requestFuture.get(0, TimeUnit.SECONDS))
+                            .getCause();
+            assertThat(cause, instanceOf(IllegalStateException.class));
+            assertThat(cause.getMessage(), equalTo("executor not accepting a task"));
+        }
+    }
+
+    @Test
+    public void testResponseChannelFuturesResolvedExceptionallyOnClose() throws Exception {
+        try (final RestClient restClient =
+                new RestClient(new Configuration(), Executors.directExecutor())) {
+            CompletableFuture<Channel> responseChannelFuture = new CompletableFuture<>();
+
+            // Add the future to the client's internal collection of pending response futures
+            restClient.getResponseChannelFutures().add(responseChannelFuture);
+
+            // Close the client, which should resolve all pending response futures exceptionally and
+            // clear the collection
+            restClient.close();
+
+            // Ensure the client's internal collection of pending response futures was cleared after
+            // close
+            assertThat(restClient.getResponseChannelFutures(), empty());
+
+            final Throwable cause =
+                    assertThrows(
+                                    ExecutionException.class,
+                                    () -> responseChannelFuture.get(0, TimeUnit.SECONDS))
+                            .getCause();
+            assertThat(cause, instanceOf(IllegalStateException.class));
+            assertThat(cause.getMessage(), equalTo("RestClient closed before request completed"));
+        }
+    }
+
     private static class TestMessageHeaders
             implements RuntimeMessageHeaders<
                     EmptyRequestBody, EmptyResponseBody, EmptyMessageParameters> {