You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by za...@apache.org on 2024/09/24 02:32:50 UTC

(flink) 01/04: [FLINK-35411][Runtime] Optimize drain of async state requests

This is an automated email from the ASF dual-hosted git repository.

zakelly pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 475a751e4de463ef323f6c5328798d528183800e
Author: Zakelly <za...@gmail.com>
AuthorDate: Tue Sep 17 03:41:14 2024 +0800

    [FLINK-35411][Runtime] Optimize drain of async state requests
---
 .../asyncprocessing/AsyncExecutionController.java  |  6 ++-
 .../runtime/asyncprocessing/StateExecutor.java     |  8 ++++
 .../asyncprocessing/AbstractStateIteratorTest.java |  5 +++
 .../AsyncExecutionControllerTest.java              |  5 +++
 .../runtime/asyncprocessing/MockStateExecutor.java |  5 +++
 .../state/v2/AbstractKeyedStateTestBase.java       |  5 +++
 .../state/v2/AbstractReducingStateTest.java        |  5 +++
 .../apache/flink/state/forst/ForStDBOperation.java |  7 ++++
 .../state/forst/ForStGeneralMultiGetOperation.java | 19 ++++++++++
 .../flink/state/forst/ForStIterateOperation.java   | 19 ++++++++++
 .../flink/state/forst/ForStStateExecutor.java      | 44 +++++++++++++++++-----
 .../state/forst/ForStStateRequestClassifier.java   |  4 ++
 .../state/forst/ForStWriteBatchOperation.java      | 21 +++++++++++
 13 files changed, 143 insertions(+), 10 deletions(-)

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/AsyncExecutionController.java b/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/AsyncExecutionController.java
index eb8d6731d38..fce6f24cee7 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/AsyncExecutionController.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/AsyncExecutionController.java
@@ -351,7 +351,11 @@ public class AsyncExecutionController<K> implements StateRequestHandler {
         try {
             while (inFlightRecordNum.get() > targetNum) {
                 if (!mailboxExecutor.tryYield()) {
-                    triggerIfNeeded(true);
+                    // We force trigger the buffer if targetNum == 0 (for draining) or the state
+                    // executor is not fully loaded.
+                    if (targetNum == 0 || !stateExecutor.fullyLoaded()) {
+                        triggerIfNeeded(true);
+                    }
                     Thread.sleep(1);
                 }
             }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/StateExecutor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/StateExecutor.java
index bbc506e1b1c..924d2557416 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/StateExecutor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/StateExecutor.java
@@ -45,6 +45,14 @@ public interface StateExecutor {
      */
     StateRequestContainer createStateRequestContainer();
 
+    /**
+     * Check if this executor is fully loaded. Will be invoked to determine whether to give more
+     * requests to run or wait for a while.
+     *
+     * @return the count.
+     */
+    boolean fullyLoaded();
+
     /** Shutdown the StateExecutor, and new committed state execution requests will be rejected. */
     void shutdown();
 }
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/AbstractStateIteratorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/AbstractStateIteratorTest.java
index 6438cc1e7d0..5a4c89b3543 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/AbstractStateIteratorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/AbstractStateIteratorTest.java
@@ -201,6 +201,11 @@ public class AbstractStateIteratorTest {
             return new MockStateRequestContainer();
         }
 
+        @Override
+        public boolean fullyLoaded() {
+            return false;
+        }
+
         @Override
         public void shutdown() {}
 
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/AsyncExecutionControllerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/AsyncExecutionControllerTest.java
index 86e4b6b035c..0ee676dee48 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/AsyncExecutionControllerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/AsyncExecutionControllerTest.java
@@ -920,6 +920,11 @@ class AsyncExecutionControllerTest {
             return new MockStateRequestContainer();
         }
 
+        @Override
+        public boolean fullyLoaded() {
+            return false;
+        }
+
         @Override
         public void shutdown() {}
     }
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/MockStateExecutor.java b/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/MockStateExecutor.java
index 8939dc7210d..51cfbb3b265 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/MockStateExecutor.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/MockStateExecutor.java
@@ -41,6 +41,11 @@ public class MockStateExecutor implements StateExecutor {
         return new MockStateRequestContainer();
     }
 
+    @Override
+    public boolean fullyLoaded() {
+        return false;
+    }
+
     @Override
     public void shutdown() {}
 }
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/v2/AbstractKeyedStateTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/v2/AbstractKeyedStateTestBase.java
index 44792146fe4..4e4db4a73b3 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/v2/AbstractKeyedStateTestBase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/v2/AbstractKeyedStateTestBase.java
@@ -220,6 +220,11 @@ public class AbstractKeyedStateTestBase {
             return new TestStateRequestContainer();
         }
 
+        @Override
+        public boolean fullyLoaded() {
+            return false;
+        }
+
         @Override
         public void shutdown() {}
 
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/v2/AbstractReducingStateTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/v2/AbstractReducingStateTest.java
index 22096bdc362..ac0c6785061 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/v2/AbstractReducingStateTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/v2/AbstractReducingStateTest.java
@@ -183,6 +183,11 @@ public class AbstractReducingStateTest extends AbstractKeyedStateTestBase {
             return new MockStateRequestContainer();
         }
 
+        @Override
+        public boolean fullyLoaded() {
+            return false;
+        }
+
         @Override
         public void shutdown() {}
     }
diff --git a/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStDBOperation.java b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStDBOperation.java
index 48ed6c5b4bb..a848073e4e6 100644
--- a/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStDBOperation.java
+++ b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStDBOperation.java
@@ -36,4 +36,11 @@ public interface ForStDBOperation {
      * @return The future which indicates whether the operation is completed.
      */
     CompletableFuture<Void> process();
+
+    /**
+     * The count of sub-processes. Each sub-process is an atomic operation in one single thread.
+     *
+     * @return the count.
+     */
+    int subProcessCount();
 }
diff --git a/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStGeneralMultiGetOperation.java b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStGeneralMultiGetOperation.java
index d22364a5a91..4f4a445dbee 100644
--- a/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStGeneralMultiGetOperation.java
+++ b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStGeneralMultiGetOperation.java
@@ -38,11 +38,22 @@ public class ForStGeneralMultiGetOperation implements ForStDBOperation {
 
     private final Executor executor;
 
+    private final Runnable subProcessFinished;
+
     ForStGeneralMultiGetOperation(
             RocksDB db, List<ForStDBGetRequest<?, ?, ?, ?>> batchRequest, Executor executor) {
+        this(db, batchRequest, executor, null);
+    }
+
+    ForStGeneralMultiGetOperation(
+            RocksDB db,
+            List<ForStDBGetRequest<?, ?, ?, ?>> batchRequest,
+            Executor executor,
+            Runnable subProcessFinished) {
         this.db = db;
         this.batchRequest = batchRequest;
         this.executor = executor;
+        this.subProcessFinished = subProcessFinished;
     }
 
     @Override
@@ -76,9 +87,17 @@ public class ForStGeneralMultiGetOperation implements ForStDBOperation {
                                     && !future.isCompletedExceptionally()) {
                                 future.complete(null);
                             }
+                            if (subProcessFinished != null) {
+                                subProcessFinished.run();
+                            }
                         }
                     });
         }
         return future;
     }
+
+    @Override
+    public int subProcessCount() {
+        return batchRequest.size();
+    }
 }
diff --git a/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStIterateOperation.java b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStIterateOperation.java
index e980b3af23a..4add2e30cd2 100644
--- a/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStIterateOperation.java
+++ b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStIterateOperation.java
@@ -39,11 +39,22 @@ public class ForStIterateOperation implements ForStDBOperation {
 
     private final Executor executor;
 
+    private final Runnable subProcessFinished;
+
     ForStIterateOperation(
             RocksDB db, List<ForStDBIterRequest<?, ?, ?, ?, ?>> batchRequest, Executor executor) {
+        this(db, batchRequest, executor, null);
+    }
+
+    ForStIterateOperation(
+            RocksDB db,
+            List<ForStDBIterRequest<?, ?, ?, ?, ?>> batchRequest,
+            Executor executor,
+            Runnable subProcessFinished) {
         this.db = db;
         this.batchRequest = batchRequest;
         this.executor = executor;
+        this.subProcessFinished = subProcessFinished;
     }
 
     @Override
@@ -75,9 +86,17 @@ public class ForStIterateOperation implements ForStDBOperation {
                                     && !future.isCompletedExceptionally()) {
                                 future.complete(null);
                             }
+                            if (subProcessFinished != null) {
+                                subProcessFinished.run();
+                            }
                         }
                     });
         }
         return future;
     }
+
+    @Override
+    public int subProcessCount() {
+        return batchRequest.size();
+    }
 }
diff --git a/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStStateExecutor.java b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStStateExecutor.java
index b9ef67eb0dd..1366e688b59 100644
--- a/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStStateExecutor.java
+++ b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStStateExecutor.java
@@ -36,6 +36,7 @@ import java.util.List;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicLong;
 
 /**
  * The {@link StateExecutor} implementation which executing batch {@link StateRequest}s for
@@ -58,12 +59,17 @@ public class ForStStateExecutor implements StateExecutor {
     /** The worker thread that actually executes the write {@link StateRequest}s. */
     private final ExecutorService writeThreads;
 
+    private final int readThreadCount;
+
     private final RocksDB db;
 
     private final WriteOptions writeOptions;
 
     private Throwable executionError;
 
+    /** The ongoing sub-processes count. */
+    private final AtomicLong ongoing;
+
     public ForStStateExecutor(
             int readIoParallelism, int writeIoParallelism, RocksDB db, WriteOptions writeOptions) {
         Preconditions.checkState(readIoParallelism > 0 || writeIoParallelism > 0);
@@ -71,17 +77,17 @@ public class ForStStateExecutor implements StateExecutor {
                 Executors.newSingleThreadScheduledExecutor(
                         new ExecutorThreadFactory("ForSt-StateExecutor-Coordinator"));
         if (readIoParallelism <= 0 || writeIoParallelism <= 0) {
+            this.readThreadCount = Math.max(readIoParallelism, writeIoParallelism);
             this.readThreads =
                     Executors.newFixedThreadPool(
-                            Math.max(readIoParallelism, writeIoParallelism),
-                            new ExecutorThreadFactory("ForSt-StateExecutor-IO"));
+                            readThreadCount, new ExecutorThreadFactory("ForSt-StateExecutor-IO"));
             this.writeThreads = null;
         } else {
+            this.readThreadCount = readIoParallelism;
             this.readThreads =
                     Executors.newFixedThreadPool(
                             readIoParallelism,
                             new ExecutorThreadFactory("ForSt-StateExecutor-read-IO"));
-
             this.writeThreads =
                     Executors.newFixedThreadPool(
                             writeIoParallelism,
@@ -89,6 +95,7 @@ public class ForStStateExecutor implements StateExecutor {
         }
         this.db = db;
         this.writeOptions = writeOptions;
+        this.ongoing = new AtomicLong();
     }
 
     @Override
@@ -98,6 +105,18 @@ public class ForStStateExecutor implements StateExecutor {
         Preconditions.checkArgument(stateRequestContainer instanceof ForStStateRequestClassifier);
         ForStStateRequestClassifier stateRequestClassifier =
                 (ForStStateRequestClassifier) stateRequestContainer;
+        // Calculate ongoing sub-processes. Only count read ones.
+        // The fully loaded only consider read requests for now, since the write ones are quick.
+        final List<ForStDBGetRequest<?, ?, ?, ?>> getRequests =
+                stateRequestClassifier.pollDbGetRequests();
+        final List<ForStDBIterRequest<?, ?, ?, ?, ?>> iterRequests =
+                stateRequestClassifier.pollDbIterRequests();
+        if (!getRequests.isEmpty()) {
+            ongoing.addAndGet(1);
+        }
+        if (!iterRequests.isEmpty()) {
+            ongoing.addAndGet(1);
+        }
         CompletableFuture<Void> resultFuture = new CompletableFuture<>();
         coordinatorThread.execute(
                 () -> {
@@ -115,19 +134,21 @@ public class ForStStateExecutor implements StateExecutor {
                         futures.add(writeOperations.process());
                     }
 
-                    List<ForStDBGetRequest<?, ?, ?, ?>> getRequests =
-                            stateRequestClassifier.pollDbGetRequests();
                     if (!getRequests.isEmpty()) {
                         ForStGeneralMultiGetOperation getOperations =
-                                new ForStGeneralMultiGetOperation(db, getRequests, readThreads);
+                                new ForStGeneralMultiGetOperation(
+                                        db, getRequests, readThreads, ongoing::decrementAndGet);
+                        // sub process count should -1, since we have added 1 on top.
+                        ongoing.addAndGet(getOperations.subProcessCount() - 1);
                         futures.add(getOperations.process());
                     }
 
-                    List<ForStDBIterRequest<?, ?, ?, ?, ?>> iterRequests =
-                            stateRequestClassifier.pollDbIterRequests();
                     if (!iterRequests.isEmpty()) {
                         ForStIterateOperation iterOperations =
-                                new ForStIterateOperation(db, iterRequests, readThreads);
+                                new ForStIterateOperation(
+                                        db, iterRequests, readThreads, ongoing::decrementAndGet);
+                        // sub process count should -1, since we have added 1 on top.
+                        ongoing.addAndGet(iterOperations.subProcessCount() - 1);
                         futures.add(iterOperations.process());
                     }
 
@@ -168,6 +189,11 @@ public class ForStStateExecutor implements StateExecutor {
         return new ForStStateRequestClassifier();
     }
 
+    @Override
+    public boolean fullyLoaded() {
+        return ongoing.get() >= readThreadCount;
+    }
+
     private void checkState() {
         if (executionError != null) {
             throw new IllegalStateException(
diff --git a/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStStateRequestClassifier.java b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStStateRequestClassifier.java
index c7205f0791c..d7678bfb765 100644
--- a/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStStateRequestClassifier.java
+++ b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStStateRequestClassifier.java
@@ -135,4 +135,8 @@ public class ForStStateRequestClassifier implements StateRequestContainer {
     public List<ForStDBIterRequest<?, ?, ?, ?, ?>> pollDbIterRequests() {
         return dbIterRequests;
     }
+
+    public long size() {
+        return dbGetRequests.size() + dbPutRequests.size() + dbIterRequests.size();
+    }
 }
diff --git a/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStWriteBatchOperation.java b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStWriteBatchOperation.java
index 1ea9ad46f8e..96943ed1efa 100644
--- a/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStWriteBatchOperation.java
+++ b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStWriteBatchOperation.java
@@ -37,15 +37,27 @@ public class ForStWriteBatchOperation implements ForStDBOperation {
 
     private final Executor executor;
 
+    private final Runnable subProcessFinished;
+
     ForStWriteBatchOperation(
             RocksDB db,
             List<ForStDBPutRequest<?, ?, ?>> batchRequest,
             WriteOptions writeOptions,
             Executor executor) {
+        this(db, batchRequest, writeOptions, executor, null);
+    }
+
+    ForStWriteBatchOperation(
+            RocksDB db,
+            List<ForStDBPutRequest<?, ?, ?>> batchRequest,
+            WriteOptions writeOptions,
+            Executor executor,
+            Runnable subProcessFinished) {
         this.db = db;
         this.batchRequest = batchRequest;
         this.writeOptions = writeOptions;
         this.executor = executor;
+        this.subProcessFinished = subProcessFinished;
     }
 
     @Override
@@ -69,8 +81,17 @@ public class ForStWriteBatchOperation implements ForStDBOperation {
                         }
                         // fail the whole batch operation
                         throw new CompletionException(msg, e);
+                    } finally {
+                        if (subProcessFinished != null) {
+                            subProcessFinished.run();
+                        }
                     }
                 },
                 executor);
     }
+
+    @Override
+    public int subProcessCount() {
+        return batchRequest.size();
+    }
 }