You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by za...@apache.org on 2024/09/24 02:32:50 UTC
(flink) 01/04: [FLINK-35411][Runtime] Optimize drain of async state requests
This is an automated email from the ASF dual-hosted git repository.
zakelly pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 475a751e4de463ef323f6c5328798d528183800e
Author: Zakelly <za...@gmail.com>
AuthorDate: Tue Sep 17 03:41:14 2024 +0800
[FLINK-35411][Runtime] Optimize drain of async state requests
---
.../asyncprocessing/AsyncExecutionController.java | 6 ++-
.../runtime/asyncprocessing/StateExecutor.java | 8 ++++
.../asyncprocessing/AbstractStateIteratorTest.java | 5 +++
.../AsyncExecutionControllerTest.java | 5 +++
.../runtime/asyncprocessing/MockStateExecutor.java | 5 +++
.../state/v2/AbstractKeyedStateTestBase.java | 5 +++
.../state/v2/AbstractReducingStateTest.java | 5 +++
.../apache/flink/state/forst/ForStDBOperation.java | 7 ++++
.../state/forst/ForStGeneralMultiGetOperation.java | 19 ++++++++++
.../flink/state/forst/ForStIterateOperation.java | 19 ++++++++++
.../flink/state/forst/ForStStateExecutor.java | 44 +++++++++++++++++-----
.../state/forst/ForStStateRequestClassifier.java | 4 ++
.../state/forst/ForStWriteBatchOperation.java | 21 +++++++++++
13 files changed, 143 insertions(+), 10 deletions(-)
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/AsyncExecutionController.java b/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/AsyncExecutionController.java
index eb8d6731d38..fce6f24cee7 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/AsyncExecutionController.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/AsyncExecutionController.java
@@ -351,7 +351,11 @@ public class AsyncExecutionController<K> implements StateRequestHandler {
try {
while (inFlightRecordNum.get() > targetNum) {
if (!mailboxExecutor.tryYield()) {
- triggerIfNeeded(true);
+ // We force trigger the buffer if targetNum == 0 (for draining) or the state
+ // executor is not fully loaded.
+ if (targetNum == 0 || !stateExecutor.fullyLoaded()) {
+ triggerIfNeeded(true);
+ }
Thread.sleep(1);
}
}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/StateExecutor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/StateExecutor.java
index bbc506e1b1c..924d2557416 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/StateExecutor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/StateExecutor.java
@@ -45,6 +45,14 @@ public interface StateExecutor {
*/
StateRequestContainer createStateRequestContainer();
+ /**
+ * Check if this executor is fully loaded. Will be invoked to determine whether to give more
+ * requests to run or wait for a while.
+ *
+ * @return the count.
+ */
+ boolean fullyLoaded();
+
/** Shutdown the StateExecutor, and new committed state execution requests will be rejected. */
void shutdown();
}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/AbstractStateIteratorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/AbstractStateIteratorTest.java
index 6438cc1e7d0..5a4c89b3543 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/AbstractStateIteratorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/AbstractStateIteratorTest.java
@@ -201,6 +201,11 @@ public class AbstractStateIteratorTest {
return new MockStateRequestContainer();
}
+ @Override
+ public boolean fullyLoaded() {
+ return false;
+ }
+
@Override
public void shutdown() {}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/AsyncExecutionControllerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/AsyncExecutionControllerTest.java
index 86e4b6b035c..0ee676dee48 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/AsyncExecutionControllerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/AsyncExecutionControllerTest.java
@@ -920,6 +920,11 @@ class AsyncExecutionControllerTest {
return new MockStateRequestContainer();
}
+ @Override
+ public boolean fullyLoaded() {
+ return false;
+ }
+
@Override
public void shutdown() {}
}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/MockStateExecutor.java b/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/MockStateExecutor.java
index 8939dc7210d..51cfbb3b265 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/MockStateExecutor.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/MockStateExecutor.java
@@ -41,6 +41,11 @@ public class MockStateExecutor implements StateExecutor {
return new MockStateRequestContainer();
}
+ @Override
+ public boolean fullyLoaded() {
+ return false;
+ }
+
@Override
public void shutdown() {}
}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/v2/AbstractKeyedStateTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/v2/AbstractKeyedStateTestBase.java
index 44792146fe4..4e4db4a73b3 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/v2/AbstractKeyedStateTestBase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/v2/AbstractKeyedStateTestBase.java
@@ -220,6 +220,11 @@ public class AbstractKeyedStateTestBase {
return new TestStateRequestContainer();
}
+ @Override
+ public boolean fullyLoaded() {
+ return false;
+ }
+
@Override
public void shutdown() {}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/v2/AbstractReducingStateTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/v2/AbstractReducingStateTest.java
index 22096bdc362..ac0c6785061 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/v2/AbstractReducingStateTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/v2/AbstractReducingStateTest.java
@@ -183,6 +183,11 @@ public class AbstractReducingStateTest extends AbstractKeyedStateTestBase {
return new MockStateRequestContainer();
}
+ @Override
+ public boolean fullyLoaded() {
+ return false;
+ }
+
@Override
public void shutdown() {}
}
diff --git a/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStDBOperation.java b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStDBOperation.java
index 48ed6c5b4bb..a848073e4e6 100644
--- a/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStDBOperation.java
+++ b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStDBOperation.java
@@ -36,4 +36,11 @@ public interface ForStDBOperation {
* @return The future which indicates whether the operation is completed.
*/
CompletableFuture<Void> process();
+
+ /**
+ * The count of sub-processes. Each sub-process is an atomic operation in one single thread.
+ *
+ * @return the count.
+ */
+ int subProcessCount();
}
diff --git a/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStGeneralMultiGetOperation.java b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStGeneralMultiGetOperation.java
index d22364a5a91..4f4a445dbee 100644
--- a/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStGeneralMultiGetOperation.java
+++ b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStGeneralMultiGetOperation.java
@@ -38,11 +38,22 @@ public class ForStGeneralMultiGetOperation implements ForStDBOperation {
private final Executor executor;
+ private final Runnable subProcessFinished;
+
ForStGeneralMultiGetOperation(
RocksDB db, List<ForStDBGetRequest<?, ?, ?, ?>> batchRequest, Executor executor) {
+ this(db, batchRequest, executor, null);
+ }
+
+ ForStGeneralMultiGetOperation(
+ RocksDB db,
+ List<ForStDBGetRequest<?, ?, ?, ?>> batchRequest,
+ Executor executor,
+ Runnable subProcessFinished) {
this.db = db;
this.batchRequest = batchRequest;
this.executor = executor;
+ this.subProcessFinished = subProcessFinished;
}
@Override
@@ -76,9 +87,17 @@ public class ForStGeneralMultiGetOperation implements ForStDBOperation {
&& !future.isCompletedExceptionally()) {
future.complete(null);
}
+ if (subProcessFinished != null) {
+ subProcessFinished.run();
+ }
}
});
}
return future;
}
+
+ @Override
+ public int subProcessCount() {
+ return batchRequest.size();
+ }
}
diff --git a/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStIterateOperation.java b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStIterateOperation.java
index e980b3af23a..4add2e30cd2 100644
--- a/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStIterateOperation.java
+++ b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStIterateOperation.java
@@ -39,11 +39,22 @@ public class ForStIterateOperation implements ForStDBOperation {
private final Executor executor;
+ private final Runnable subProcessFinished;
+
ForStIterateOperation(
RocksDB db, List<ForStDBIterRequest<?, ?, ?, ?, ?>> batchRequest, Executor executor) {
+ this(db, batchRequest, executor, null);
+ }
+
+ ForStIterateOperation(
+ RocksDB db,
+ List<ForStDBIterRequest<?, ?, ?, ?, ?>> batchRequest,
+ Executor executor,
+ Runnable subProcessFinished) {
this.db = db;
this.batchRequest = batchRequest;
this.executor = executor;
+ this.subProcessFinished = subProcessFinished;
}
@Override
@@ -75,9 +86,17 @@ public class ForStIterateOperation implements ForStDBOperation {
&& !future.isCompletedExceptionally()) {
future.complete(null);
}
+ if (subProcessFinished != null) {
+ subProcessFinished.run();
+ }
}
});
}
return future;
}
+
+ @Override
+ public int subProcessCount() {
+ return batchRequest.size();
+ }
}
diff --git a/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStStateExecutor.java b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStStateExecutor.java
index b9ef67eb0dd..1366e688b59 100644
--- a/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStStateExecutor.java
+++ b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStStateExecutor.java
@@ -36,6 +36,7 @@ import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicLong;
/**
* The {@link StateExecutor} implementation which executing batch {@link StateRequest}s for
@@ -58,12 +59,17 @@ public class ForStStateExecutor implements StateExecutor {
/** The worker thread that actually executes the write {@link StateRequest}s. */
private final ExecutorService writeThreads;
+ private final int readThreadCount;
+
private final RocksDB db;
private final WriteOptions writeOptions;
private Throwable executionError;
+ /** The ongoing sub-processes count. */
+ private final AtomicLong ongoing;
+
public ForStStateExecutor(
int readIoParallelism, int writeIoParallelism, RocksDB db, WriteOptions writeOptions) {
Preconditions.checkState(readIoParallelism > 0 || writeIoParallelism > 0);
@@ -71,17 +77,17 @@ public class ForStStateExecutor implements StateExecutor {
Executors.newSingleThreadScheduledExecutor(
new ExecutorThreadFactory("ForSt-StateExecutor-Coordinator"));
if (readIoParallelism <= 0 || writeIoParallelism <= 0) {
+ this.readThreadCount = Math.max(readIoParallelism, writeIoParallelism);
this.readThreads =
Executors.newFixedThreadPool(
- Math.max(readIoParallelism, writeIoParallelism),
- new ExecutorThreadFactory("ForSt-StateExecutor-IO"));
+ readThreadCount, new ExecutorThreadFactory("ForSt-StateExecutor-IO"));
this.writeThreads = null;
} else {
+ this.readThreadCount = readIoParallelism;
this.readThreads =
Executors.newFixedThreadPool(
readIoParallelism,
new ExecutorThreadFactory("ForSt-StateExecutor-read-IO"));
-
this.writeThreads =
Executors.newFixedThreadPool(
writeIoParallelism,
@@ -89,6 +95,7 @@ public class ForStStateExecutor implements StateExecutor {
}
this.db = db;
this.writeOptions = writeOptions;
+ this.ongoing = new AtomicLong();
}
@Override
@@ -98,6 +105,18 @@ public class ForStStateExecutor implements StateExecutor {
Preconditions.checkArgument(stateRequestContainer instanceof ForStStateRequestClassifier);
ForStStateRequestClassifier stateRequestClassifier =
(ForStStateRequestClassifier) stateRequestContainer;
+ // Calculate ongoing sub-processes. Only count read ones.
+ // The fully loaded only consider read requests for now, since the write ones are quick.
+ final List<ForStDBGetRequest<?, ?, ?, ?>> getRequests =
+ stateRequestClassifier.pollDbGetRequests();
+ final List<ForStDBIterRequest<?, ?, ?, ?, ?>> iterRequests =
+ stateRequestClassifier.pollDbIterRequests();
+ if (!getRequests.isEmpty()) {
+ ongoing.addAndGet(1);
+ }
+ if (!iterRequests.isEmpty()) {
+ ongoing.addAndGet(1);
+ }
CompletableFuture<Void> resultFuture = new CompletableFuture<>();
coordinatorThread.execute(
() -> {
@@ -115,19 +134,21 @@ public class ForStStateExecutor implements StateExecutor {
futures.add(writeOperations.process());
}
- List<ForStDBGetRequest<?, ?, ?, ?>> getRequests =
- stateRequestClassifier.pollDbGetRequests();
if (!getRequests.isEmpty()) {
ForStGeneralMultiGetOperation getOperations =
- new ForStGeneralMultiGetOperation(db, getRequests, readThreads);
+ new ForStGeneralMultiGetOperation(
+ db, getRequests, readThreads, ongoing::decrementAndGet);
+ // sub process count should -1, since we have added 1 on top.
+ ongoing.addAndGet(getOperations.subProcessCount() - 1);
futures.add(getOperations.process());
}
- List<ForStDBIterRequest<?, ?, ?, ?, ?>> iterRequests =
- stateRequestClassifier.pollDbIterRequests();
if (!iterRequests.isEmpty()) {
ForStIterateOperation iterOperations =
- new ForStIterateOperation(db, iterRequests, readThreads);
+ new ForStIterateOperation(
+ db, iterRequests, readThreads, ongoing::decrementAndGet);
+ // sub process count should -1, since we have added 1 on top.
+ ongoing.addAndGet(iterOperations.subProcessCount() - 1);
futures.add(iterOperations.process());
}
@@ -168,6 +189,11 @@ public class ForStStateExecutor implements StateExecutor {
return new ForStStateRequestClassifier();
}
+ @Override
+ public boolean fullyLoaded() {
+ return ongoing.get() >= readThreadCount;
+ }
+
private void checkState() {
if (executionError != null) {
throw new IllegalStateException(
diff --git a/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStStateRequestClassifier.java b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStStateRequestClassifier.java
index c7205f0791c..d7678bfb765 100644
--- a/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStStateRequestClassifier.java
+++ b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStStateRequestClassifier.java
@@ -135,4 +135,8 @@ public class ForStStateRequestClassifier implements StateRequestContainer {
public List<ForStDBIterRequest<?, ?, ?, ?, ?>> pollDbIterRequests() {
return dbIterRequests;
}
+
+ public long size() {
+ return dbGetRequests.size() + dbPutRequests.size() + dbIterRequests.size();
+ }
}
diff --git a/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStWriteBatchOperation.java b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStWriteBatchOperation.java
index 1ea9ad46f8e..96943ed1efa 100644
--- a/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStWriteBatchOperation.java
+++ b/flink-state-backends/flink-statebackend-forst/src/main/java/org/apache/flink/state/forst/ForStWriteBatchOperation.java
@@ -37,15 +37,27 @@ public class ForStWriteBatchOperation implements ForStDBOperation {
private final Executor executor;
+ private final Runnable subProcessFinished;
+
ForStWriteBatchOperation(
RocksDB db,
List<ForStDBPutRequest<?, ?, ?>> batchRequest,
WriteOptions writeOptions,
Executor executor) {
+ this(db, batchRequest, writeOptions, executor, null);
+ }
+
+ ForStWriteBatchOperation(
+ RocksDB db,
+ List<ForStDBPutRequest<?, ?, ?>> batchRequest,
+ WriteOptions writeOptions,
+ Executor executor,
+ Runnable subProcessFinished) {
this.db = db;
this.batchRequest = batchRequest;
this.writeOptions = writeOptions;
this.executor = executor;
+ this.subProcessFinished = subProcessFinished;
}
@Override
@@ -69,8 +81,17 @@ public class ForStWriteBatchOperation implements ForStDBOperation {
}
// fail the whole batch operation
throw new CompletionException(msg, e);
+ } finally {
+ if (subProcessFinished != null) {
+ subProcessFinished.run();
+ }
}
},
executor);
}
+
+ @Override
+ public int subProcessCount() {
+ return batchRequest.size();
+ }
}