You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by za...@apache.org on 2024/09/24 02:32:51 UTC
(flink) 02/04: [FLINK-35411][Runtime] Introduce periodic timeout check for state requests buffer
This is an automated email from the ASF dual-hosted git repository.
zakelly pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit a2cd64c7ea57db6137a86caa7e6f9f99c7f77743
Author: Zakelly <za...@gmail.com>
AuthorDate: Thu Sep 19 20:27:29 2024 +0800
[FLINK-35411][Runtime] Introduce periodic timeout check for state requests buffer
---
.../asyncprocessing/AsyncExecutionController.java | 20 ++-
.../asyncprocessing/StateRequestBuffer.java | 70 ++++++----
.../AbstractAsyncStateStreamOperator.java | 6 +
.../AbstractAsyncStateStreamOperatorV2.java | 6 +
.../AsyncExecutionControllerTest.java | 146 +++++----------------
5 files changed, 108 insertions(+), 140 deletions(-)
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/AsyncExecutionController.java b/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/AsyncExecutionController.java
index fce6f24cee7..f69b4d3b72c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/AsyncExecutionController.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/AsyncExecutionController.java
@@ -35,6 +35,8 @@ import org.slf4j.LoggerFactory;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
+import java.io.Closeable;
+import java.io.IOException;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicInteger;
@@ -51,10 +53,12 @@ import java.util.concurrent.atomic.AtomicInteger;
*
* @param <K> the type of the key
*/
-public class AsyncExecutionController<K> implements StateRequestHandler {
+public class AsyncExecutionController<K> implements StateRequestHandler, Closeable {
private static final Logger LOG = LoggerFactory.getLogger(AsyncExecutionController.class);
+ private static final long DEFAULT_BUFFER_TIMEOUT_CHECK_INTERVAL = 100;
+
/**
* The batch size. When the number of state requests in the active buffer exceeds the batch
* size, a batched state execution would be triggered.
@@ -68,6 +72,14 @@ public class AsyncExecutionController<K> implements StateRequestHandler {
*/
private final long bufferTimeout;
+ /**
+ * There might be huge overhead when inserting a timer for each buffer. A periodic check is a
+ * good trade-off to save much GC and CPU for this. This var defines the interval for periodic
+ * check of timeout. As a result, the real trigger time of timeout buffer might be [timeout,
+ * timeout+interval]. We don't make it configurable for now.
+ */
+ private final long bufferTimeoutCheckInterval = DEFAULT_BUFFER_TIMEOUT_CHECK_INTERVAL;
+
/** The max allowed number of in-flight records. */
private final int maxInFlightRecordNum;
@@ -142,6 +154,7 @@ public class AsyncExecutionController<K> implements StateRequestHandler {
this.stateRequestsBuffer =
new StateRequestBuffer<>(
bufferTimeout,
+ bufferTimeoutCheckInterval,
(scheduledSeq) ->
mailboxExecutor.execute(
() -> {
@@ -387,6 +400,11 @@ public class AsyncExecutionController<K> implements StateRequestHandler {
return inFlightRecordNum.get();
}
+ @Override
+ public void close() throws IOException {
+ stateRequestsBuffer.close();
+ }
+
/** A listener listens the key context switch. */
public interface SwitchContextListener<K> {
void switchContext(RecordContext<K> context);
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/StateRequestBuffer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/StateRequestBuffer.java
index 04379d2d34b..1433e83c7bf 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/StateRequestBuffer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/StateRequestBuffer.java
@@ -18,11 +18,14 @@
package org.apache.flink.runtime.asyncprocessing;
+import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.util.concurrent.ExecutorThreadFactory;
import javax.annotation.Nullable;
import javax.annotation.concurrent.NotThreadSafe;
+import java.io.Closeable;
+import java.io.IOException;
import java.util.Deque;
import java.util.HashMap;
import java.util.LinkedList;
@@ -43,7 +46,7 @@ import java.util.function.Supplier;
* @param <K> the type of the key
*/
@NotThreadSafe
-public class StateRequestBuffer<K> {
+public class StateRequestBuffer<K> implements Closeable {
/** All StateRequestBuffer in the same task manager share one ScheduledExecutorService. */
private static final ScheduledThreadPoolExecutor DELAYER =
@@ -75,44 +78,68 @@ public class StateRequestBuffer<K> {
/** The timeout of {@link #activeQueue} triggering in milliseconds. */
final long bufferTimeout;
+ /** The interval of periodic buffer timeout check. */
+ final long bufferTimeoutCheckInterval;
+
/** The handler to trigger when timeout. */
final Consumer<Long> timeoutHandler;
/** The executor service that schedules and calls the triggers of this task. */
- ScheduledExecutorService scheduledExecutor;
+ final ScheduledExecutorService scheduledExecutor;
/**
* The current scheduled future, when the next scheduling occurs, the previous one that has not
* yet been executed will be canceled.
*/
- ScheduledFuture<Void> currentScheduledFuture;
+ ScheduledFuture<?> currentScheduledFuture;
/**
* The current scheduled trigger sequence number, a timeout trigger is scheduled only if {@code
* scheduledSeq} is less than {@code currentSeq}.
*/
- AtomicLong scheduledSeq;
+ volatile Tuple2<Long, Long> seqAndTimeout = null;
/**
* The current trigger sequence number, used to distinguish different triggers. Every time a
* trigger occurs, {@code currentSeq} increases by 1.
*/
- AtomicLong currentSeq;
+ final AtomicLong currentSeq;
- public StateRequestBuffer(long bufferTimeout, Consumer<Long> timeoutHandler) {
+ public StateRequestBuffer(
+ long bufferTimeout, long bufferTimeoutCheckInterval, Consumer<Long> timeoutHandler) {
this.activeQueue = new LinkedList<>();
this.blockingQueue = new HashMap<>();
this.blockingQueueSize = 0;
this.bufferTimeout = bufferTimeout;
+ this.bufferTimeoutCheckInterval = bufferTimeoutCheckInterval;
this.timeoutHandler = timeoutHandler;
- this.scheduledSeq = new AtomicLong(-1);
this.currentSeq = new AtomicLong(0);
if (bufferTimeout > 0) {
this.scheduledExecutor = DELAYER;
+ initPeriodicTimeoutCheck();
+ } else {
+ this.scheduledExecutor = null;
}
}
+ private void initPeriodicTimeoutCheck() {
+ currentScheduledFuture =
+ scheduledExecutor.scheduleAtFixedRate(
+ () -> {
+ final Tuple2<Long, Long> theSeqAndTimeout = seqAndTimeout;
+ if (theSeqAndTimeout != null
+ && theSeqAndTimeout.f0 == currentSeq.get()
+ && theSeqAndTimeout.f1 <= System.currentTimeMillis()) {
+ timeoutHandler.accept(theSeqAndTimeout.f0);
+ }
+ },
+ bufferTimeout,
+ bufferTimeoutCheckInterval,
+ TimeUnit.MILLISECONDS);
+ }
+
void advanceSeq() {
+ seqAndTimeout = null;
currentSeq.incrementAndGet();
}
@@ -125,24 +152,9 @@ public class StateRequestBuffer<K> {
request.getFuture().complete(null);
} else {
activeQueue.add(request);
- if (bufferTimeout > 0 && currentSeq.get() > scheduledSeq.get()) {
- if (currentScheduledFuture != null
- && !currentScheduledFuture.isDone()
- && !currentScheduledFuture.isCancelled()) {
- currentScheduledFuture.cancel(false);
- }
- final long thisScheduledSeq = currentSeq.get();
- scheduledSeq.set(thisScheduledSeq);
- currentScheduledFuture =
- (ScheduledFuture<Void>)
- scheduledExecutor.schedule(
- () -> {
- if (thisScheduledSeq == currentSeq.get()) {
- timeoutHandler.accept(thisScheduledSeq);
- }
- },
- bufferTimeout,
- TimeUnit.MILLISECONDS);
+ if (bufferTimeout > 0 && seqAndTimeout == null) {
+ seqAndTimeout =
+ Tuple2.of(currentSeq.get(), System.currentTimeMillis() + bufferTimeout);
}
}
}
@@ -213,4 +225,12 @@ public class StateRequestBuffer<K> {
}
return Optional.of(stateRequestContainer);
}
+
+ @Override
+ public synchronized void close() throws IOException {
+ if (currentScheduledFuture != null) {
+ currentScheduledFuture.cancel(true);
+ currentScheduledFuture = null;
+ }
+ }
}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/operators/AbstractAsyncStateStreamOperator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/operators/AbstractAsyncStateStreamOperator.java
index c74befba770..afe380158a2 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/operators/AbstractAsyncStateStreamOperator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/operators/AbstractAsyncStateStreamOperator.java
@@ -316,4 +316,10 @@ public abstract class AbstractAsyncStateStreamOperator<OUT> extends AbstractStre
RecordContext getCurrentProcessingContext() {
return currentProcessingContext;
}
+
+ @Override
+ public void close() throws Exception {
+ super.close();
+ asyncExecutionController.close();
+ }
}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/operators/AbstractAsyncStateStreamOperatorV2.java b/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/operators/AbstractAsyncStateStreamOperatorV2.java
index e686efab5ae..f3322a46301 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/operators/AbstractAsyncStateStreamOperatorV2.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/operators/AbstractAsyncStateStreamOperatorV2.java
@@ -253,4 +253,10 @@ public abstract class AbstractAsyncStateStreamOperatorV2<OUT> extends AbstractSt
public RecordContext getCurrentProcessingContext() {
return currentProcessingContext;
}
+
+ @Override
+ public void close() throws Exception {
+ super.close();
+ asyncExecutionController.close();
+ }
}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/AsyncExecutionControllerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/AsyncExecutionControllerTest.java
index 0ee676dee48..8d7853eb7ef 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/AsyncExecutionControllerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/AsyncExecutionControllerTest.java
@@ -28,7 +28,6 @@ import org.apache.flink.core.state.StateFutureImpl.AsyncFrameworkExceptionHandle
import org.apache.flink.core.state.StateFutureUtils;
import org.apache.flink.runtime.asyncprocessing.EpochManager.Epoch;
import org.apache.flink.runtime.asyncprocessing.EpochManager.ParallelMode;
-import org.apache.flink.runtime.concurrent.ManuallyTriggeredScheduledExecutorService;
import org.apache.flink.runtime.mailbox.SyncMailboxExecutor;
import org.apache.flink.runtime.state.AsyncKeyedStateBackend;
import org.apache.flink.runtime.state.StateBackend;
@@ -46,7 +45,6 @@ import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.util.HashMap;
import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
import java.util.function.Supplier;
@@ -527,7 +525,7 @@ class AsyncExecutionControllerTest {
}
@Test
- void testBufferTimeout() throws IOException {
+ void testBufferTimeout() throws Exception {
int batchSize = 5;
int timeout = 1000;
CloseableRegistry resourceRegistry = new CloseableRegistry();
@@ -538,11 +536,11 @@ class AsyncExecutionControllerTest {
new SyncMailboxExecutor(),
new TestAsyncFrameworkExceptionHandler(),
resourceRegistry);
- ManuallyTriggeredScheduledExecutorService scheduledExecutor =
- new ManuallyTriggeredScheduledExecutorService();
- aec.stateRequestsBuffer.scheduledExecutor = scheduledExecutor;
+
Runnable userCode = () -> valueState.asyncValue();
+ assertThat(aec.stateRequestsBuffer.currentSeq.get()).isEqualTo(0L);
+ assertThat(aec.stateRequestsBuffer.seqAndTimeout).isNull();
// ------------ basic timeout -------------------
for (int i = 0; i < batchSize - 1; i++) {
String record = String.format("key%d-r%d", i, i);
@@ -551,127 +549,47 @@ class AsyncExecutionControllerTest {
aec.setCurrentContext(recordContext);
userCode.run();
}
- assertThat(aec.stateRequestsBuffer.currentSeq.get()).isEqualTo(0);
- assertThat(aec.stateRequestsBuffer.scheduledSeq.get()).isEqualTo(0);
+ assertThat(aec.stateRequestsBuffer.currentSeq.get()).isEqualTo(0L);
+ assertThat(aec.stateRequestsBuffer.seqAndTimeout.f0).isEqualTo(0L);
assertThat(aec.stateRequestsBuffer.currentScheduledFuture.isDone()).isFalse();
assertThat(aec.inFlightRecordNum.get()).isEqualTo(batchSize - 1);
assertThat(aec.stateRequestsBuffer.activeQueueSize()).isEqualTo(batchSize - 1);
assertThat(aec.stateRequestsBuffer.blockingQueueSize()).isEqualTo(0);
// buffer timeout, trigger
- scheduledExecutor.triggerNonPeriodicScheduledTasks();
+ Thread.sleep(2000);
assertThat(aec.stateRequestsBuffer.activeQueueSize()).isEqualTo(0);
- assertThat(aec.stateRequestsBuffer.currentScheduledFuture.isDone()).isTrue();
- assertThat(aec.inFlightRecordNum.get()).isEqualTo(0);
- assertThat(aec.stateRequestsBuffer.currentSeq.get()).isEqualTo(1);
- assertThat(aec.stateRequestsBuffer.scheduledSeq.get()).isEqualTo(0);
-
- // ----------------- oldest state request timeout ------------------
- // r5 and r6 should be triggered due to r5 exceeding timeout
- String record5 = "key5-r5";
- String key5 = "key5";
- RecordContext<String> recordContext5 = aec.buildContext(record5, key5);
- aec.setCurrentContext(recordContext5);
- // execute user code
- userCode.run();
- assertThat(aec.inFlightRecordNum.get()).isEqualTo(1);
- assertThat(aec.stateRequestsBuffer.activeQueueSize()).isEqualTo(1);
assertThat(aec.stateRequestsBuffer.currentScheduledFuture.isDone()).isFalse();
- ScheduledFuture<Void> scheduledFuture = aec.stateRequestsBuffer.currentScheduledFuture;
- String record6 = "key6-r6";
- String key6 = "key6";
- RecordContext<String> recordContext6 = aec.buildContext(record6, key6);
- aec.setCurrentContext(recordContext6);
- // execute user code
- userCode.run();
- assertThat(aec.inFlightRecordNum.get()).isEqualTo(2);
- assertThat(aec.stateRequestsBuffer.activeQueueSize()).isEqualTo(2);
-
- assertThat(scheduledExecutor.getActiveNonPeriodicScheduledTask().size()).isEqualTo(1);
- assertThat(scheduledExecutor.getAllNonPeriodicScheduledTask().size()).isEqualTo(1);
- assertThat(aec.stateRequestsBuffer.currentSeq.get()).isEqualTo(1);
- assertThat(aec.stateRequestsBuffer.scheduledSeq.get()).isEqualTo(1);
- scheduledExecutor.triggerNonPeriodicScheduledTasks();
-
assertThat(aec.inFlightRecordNum.get()).isEqualTo(0);
- assertThat(aec.stateRequestsBuffer.activeQueueSize()).isEqualTo(0);
- assertThat(scheduledFuture).isEqualTo(aec.stateRequestsBuffer.currentScheduledFuture);
- assertThat(scheduledFuture.isDone()).isTrue();
- assertThat(aec.stateRequestsBuffer.currentSeq.get()).isEqualTo(2);
- assertThat(aec.stateRequestsBuffer.scheduledSeq.get()).isEqualTo(1);
-
- resourceRegistry.close();
- }
+ assertThat(aec.stateRequestsBuffer.currentSeq.get()).isEqualTo(1L);
+ assertThat(aec.stateRequestsBuffer.seqAndTimeout).isNull();
- @Test
- void testBufferTimeoutSkip() throws IOException {
- int batchSize = 3;
- int timeout = 1000;
- CloseableRegistry resourceRegistry = new CloseableRegistry();
- setup(
- batchSize,
- timeout,
- 1000,
- new SyncMailboxExecutor(),
- new TestAsyncFrameworkExceptionHandler(),
- resourceRegistry);
- ManuallyTriggeredScheduledExecutorService scheduledExecutor =
- new ManuallyTriggeredScheduledExecutorService();
- aec.stateRequestsBuffer.scheduledExecutor = scheduledExecutor;
- Runnable userCode = () -> valueState.asyncValue();
-
- assertThat(aec.stateRequestsBuffer.currentSeq.get()).isEqualTo(0);
- assertThat(aec.stateRequestsBuffer.scheduledSeq.get()).isEqualTo(-1);
- // register r1 timeout
- RecordContext<String> recordContext = aec.buildContext("record1", "key1");
- aec.setCurrentContext(recordContext);
- userCode.run();
- assertThat(aec.stateRequestsBuffer.activeQueueSize()).isEqualTo(1);
- assertThat(aec.inFlightRecordNum.get()).isEqualTo(1);
- assertThat(aec.stateRequestsBuffer.currentSeq.get()).isEqualTo(0);
- assertThat(aec.stateRequestsBuffer.scheduledSeq.get()).isEqualTo(0);
-
- // before r1 timeout execute, the active buffer size reach batch size.
- RecordContext<String> recordContext2 = aec.buildContext("record2", "key2");
- aec.setCurrentContext(recordContext2);
- userCode.run();
- assertThat(aec.stateRequestsBuffer.currentSeq.get()).isEqualTo(0);
- assertThat(aec.stateRequestsBuffer.scheduledSeq.get()).isEqualTo(0);
- RecordContext<String> recordContext3 = aec.buildContext("record3", "key3");
- aec.setCurrentContext(recordContext3);
- userCode.run();
- assertThat(aec.stateRequestsBuffer.activeQueueSize()).isEqualTo(0);
- assertThat(aec.stateRequestsBuffer.currentSeq.get()).isEqualTo(1);
- assertThat(aec.stateRequestsBuffer.scheduledSeq.get()).isEqualTo(0);
-
- // r1 timeout executes, but r1 is already triggered in [r1,r2,r3], so r1 timeout should skip
- assertThat(scheduledExecutor.getActiveNonPeriodicScheduledTask().size()).isEqualTo(1);
- assertThat(scheduledExecutor.getAllNonPeriodicScheduledTask().size()).isEqualTo(1);
- scheduledExecutor.triggerNonPeriodicScheduledTask();
- assertThat(aec.stateRequestsBuffer.currentScheduledFuture.isDone()).isTrue();
- assertThat(aec.stateRequestsBuffer.currentSeq.get()).isEqualTo(1);
- assertThat(aec.stateRequestsBuffer.scheduledSeq.get()).isEqualTo(0);
-
- RecordContext<String> recordContext4 = aec.buildContext("record4", "key4");
- aec.setCurrentContext(recordContext4);
- userCode.run();
-
- // register r4 timeout, set new currentScheduledFuture
- assertThat(scheduledExecutor.getActiveNonPeriodicScheduledTask().size()).isEqualTo(1);
- assertThat(aec.stateRequestsBuffer.activeQueueSize()).isEqualTo(1);
- assertThat(aec.inFlightRecordNum.get()).isEqualTo(1);
+ // ---------- buffer full before timeout ------------------
+ for (int i = 0; i < batchSize - 1; i++) {
+ String record = String.format("key%d-r%d", i, i);
+ String key = String.format("key%d", batchSize + i);
+ RecordContext<String> recordContext = aec.buildContext(record, key);
+ aec.setCurrentContext(recordContext);
+ userCode.run();
+ }
+ assertThat(aec.stateRequestsBuffer.currentSeq.get()).isEqualTo(1L);
+ assertThat(aec.stateRequestsBuffer.seqAndTimeout.f0).isEqualTo(1L);
assertThat(aec.stateRequestsBuffer.currentScheduledFuture.isDone()).isFalse();
- assertThat(aec.stateRequestsBuffer.currentScheduledFuture.isCancelled()).isFalse();
- assertThat(aec.stateRequestsBuffer.currentSeq.get()).isEqualTo(1);
- assertThat(aec.stateRequestsBuffer.scheduledSeq.get()).isEqualTo(1);
-
- // r4 timeout
- scheduledExecutor.triggerNonPeriodicScheduledTask();
+ assertThat(aec.inFlightRecordNum.get()).isEqualTo(batchSize - 1);
+ assertThat(aec.stateRequestsBuffer.activeQueueSize()).isEqualTo(batchSize - 1);
+ assertThat(aec.stateRequestsBuffer.blockingQueueSize()).isEqualTo(0);
+ for (int i = batchSize - 1; i < batchSize; i++) {
+ String record = String.format("key%d-r%d", i, i);
+ String key = String.format("key%d", batchSize + i);
+ RecordContext<String> recordContext = aec.buildContext(record, key);
+ aec.setCurrentContext(recordContext);
+ userCode.run();
+ }
assertThat(aec.stateRequestsBuffer.activeQueueSize()).isEqualTo(0);
+ assertThat(aec.stateRequestsBuffer.currentScheduledFuture.isDone()).isFalse();
assertThat(aec.inFlightRecordNum.get()).isEqualTo(0);
- assertThat(aec.stateRequestsBuffer.currentSeq.get()).isEqualTo(2);
- assertThat(aec.stateRequestsBuffer.scheduledSeq.get()).isEqualTo(1);
- assertThat(aec.stateRequestsBuffer.currentScheduledFuture.isDone()).isTrue();
+ assertThat(aec.stateRequestsBuffer.currentSeq.get()).isEqualTo(2L);
+ assertThat(aec.stateRequestsBuffer.seqAndTimeout).isNull();
resourceRegistry.close();
}