You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by za...@apache.org on 2024/09/13 02:33:05 UTC
(flink) branch master updated: [FLINK-35027][Runtime/Checkpoint] Drain async state request before snapshot barrier (#25317)
This is an automated email from the ASF dual-hosted git repository.
zakelly pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new e94d433dd0b [FLINK-35027][Runtime/Checkpoint] Drain async state request before snapshot barrier (#25317)
e94d433dd0b is described below
commit e94d433dd0b242e68eb2eb2a4b2546c2872e0776
Author: Zakelly <za...@gmail.com>
AuthorDate: Fri Sep 13 10:32:58 2024 +0800
[FLINK-35027][Runtime/Checkpoint] Drain async state request before snapshot barrier (#25317)
---
.../operators/AbstractAsyncStateStreamOperator.java | 14 +-------------
.../operators/AbstractAsyncStateStreamOperatorV2.java | 14 +-------------
.../operators/AbstractAsyncStateStreamOperatorTest.java | 14 +-------------
.../operators/AbstractAsyncStateStreamOperatorV2Test.java | 12 +-----------
4 files changed, 4 insertions(+), 50 deletions(-)
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/operators/AbstractAsyncStateStreamOperator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/operators/AbstractAsyncStateStreamOperator.java
index f3432b43d23..6ea16456c5b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/operators/AbstractAsyncStateStreamOperator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/operators/AbstractAsyncStateStreamOperator.java
@@ -27,17 +27,14 @@ import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.runtime.asyncprocessing.AsyncExecutionController;
import org.apache.flink.runtime.asyncprocessing.AsyncStateException;
import org.apache.flink.runtime.asyncprocessing.RecordContext;
-import org.apache.flink.runtime.checkpoint.CheckpointOptions;
import org.apache.flink.runtime.execution.Environment;
import org.apache.flink.runtime.state.AsyncKeyedStateBackend;
-import org.apache.flink.runtime.state.CheckpointStreamFactory;
import org.apache.flink.runtime.state.KeyedStateBackend;
import org.apache.flink.runtime.state.v2.StateDescriptor;
import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
import org.apache.flink.streaming.api.operators.Input;
import org.apache.flink.streaming.api.operators.InternalTimeServiceManager;
import org.apache.flink.streaming.api.operators.InternalTimerService;
-import org.apache.flink.streaming.api.operators.OperatorSnapshotFutures;
import org.apache.flink.streaming.api.operators.StreamTaskStateInitializer;
import org.apache.flink.streaming.api.operators.Triggerable;
import org.apache.flink.streaming.api.operators.TwoInputStreamOperator;
@@ -212,19 +209,10 @@ public abstract class AbstractAsyncStateStreamOperator<OUT> extends AbstractStre
}
@Override
- public final OperatorSnapshotFutures snapshotState(
- long checkpointId,
- long timestamp,
- CheckpointOptions checkpointOptions,
- CheckpointStreamFactory factory)
- throws Exception {
+ public void prepareSnapshotPreBarrier(long checkpointId) throws Exception {
if (isAsyncStateProcessingEnabled()) {
asyncExecutionController.drainInflightRecords(0);
}
- // {@link #snapshotState(StateSnapshotContext)} will be called in
- // stateHandler#snapshotState, so the {@link #snapshotState(StateSnapshotContext)} is not
- // needed to override.
- return super.snapshotState(checkpointId, timestamp, checkpointOptions, factory);
}
/**
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/operators/AbstractAsyncStateStreamOperatorV2.java b/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/operators/AbstractAsyncStateStreamOperatorV2.java
index c54ecf81f56..fbf5acd30d7 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/operators/AbstractAsyncStateStreamOperatorV2.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/asyncprocessing/operators/AbstractAsyncStateStreamOperatorV2.java
@@ -26,16 +26,13 @@ import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.runtime.asyncprocessing.AsyncExecutionController;
import org.apache.flink.runtime.asyncprocessing.AsyncStateException;
import org.apache.flink.runtime.asyncprocessing.RecordContext;
-import org.apache.flink.runtime.checkpoint.CheckpointOptions;
import org.apache.flink.runtime.execution.Environment;
import org.apache.flink.runtime.state.AsyncKeyedStateBackend;
-import org.apache.flink.runtime.state.CheckpointStreamFactory;
import org.apache.flink.runtime.state.KeyedStateBackend;
import org.apache.flink.runtime.state.v2.StateDescriptor;
import org.apache.flink.streaming.api.operators.AbstractStreamOperatorV2;
import org.apache.flink.streaming.api.operators.InternalTimeServiceManager;
import org.apache.flink.streaming.api.operators.InternalTimerService;
-import org.apache.flink.streaming.api.operators.OperatorSnapshotFutures;
import org.apache.flink.streaming.api.operators.StreamOperatorParameters;
import org.apache.flink.streaming.api.operators.StreamTaskStateInitializer;
import org.apache.flink.streaming.api.operators.Triggerable;
@@ -187,19 +184,10 @@ public abstract class AbstractAsyncStateStreamOperatorV2<OUT> extends AbstractSt
}
@Override
- public final OperatorSnapshotFutures snapshotState(
- long checkpointId,
- long timestamp,
- CheckpointOptions checkpointOptions,
- CheckpointStreamFactory factory)
- throws Exception {
+ public void prepareSnapshotPreBarrier(long checkpointId) throws Exception {
if (isAsyncStateProcessingEnabled()) {
asyncExecutionController.drainInflightRecords(0);
}
- // {@link #snapshotState(StateSnapshotContext)} will be called in
- // stateHandler#snapshotState, so the {@link #snapshotState(StateSnapshotContext)} is not
- // needed to override.
- return super.snapshotState(checkpointId, timestamp, checkpointOptions, factory);
}
@SuppressWarnings("unchecked")
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/operators/AbstractAsyncStateStreamOperatorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/operators/AbstractAsyncStateStreamOperatorTest.java
index 58113e687ef..d0376eec1d0 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/operators/AbstractAsyncStateStreamOperatorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/operators/AbstractAsyncStateStreamOperatorTest.java
@@ -18,19 +18,15 @@
package org.apache.flink.runtime.asyncprocessing.operators;
-import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.runtime.asyncprocessing.AsyncExecutionController;
import org.apache.flink.runtime.asyncprocessing.StateRequestType;
-import org.apache.flink.runtime.checkpoint.CheckpointOptions;
-import org.apache.flink.runtime.checkpoint.CheckpointType;
import org.apache.flink.runtime.state.CheckpointStorageLocationReference;
import org.apache.flink.runtime.state.VoidNamespace;
import org.apache.flink.runtime.state.VoidNamespaceSerializer;
import org.apache.flink.runtime.state.hashmap.HashMapStateBackend;
-import org.apache.flink.runtime.state.storage.JobManagerCheckpointStorage;
import org.apache.flink.streaming.api.operators.InternalTimer;
import org.apache.flink.streaming.api.operators.InternalTimerServiceAsyncImpl;
import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
@@ -218,15 +214,7 @@ public class AbstractAsyncStateStreamOperatorTest {
((AbstractAsyncStateStreamOperator<String>) testHarness.getOperator())
.postProcessElement();
assertThat(asyncExecutionController.getInFlightRecordNum()).isEqualTo(1);
- testHarness
- .getOperator()
- .snapshotState(
- 1,
- 1,
- new CheckpointOptions(CheckpointType.CHECKPOINT, locationReference),
- new JobManagerCheckpointStorage()
- .createCheckpointStorage(new JobID())
- .resolveCheckpointStorageLocation(1, locationReference));
+ testHarness.getOperator().prepareSnapshotPreBarrier(1);
assertThat(asyncExecutionController.getInFlightRecordNum()).isEqualTo(0);
}
}
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/AbstractAsyncStateStreamOperatorV2Test.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/AbstractAsyncStateStreamOperatorV2Test.java
index 6615675907c..40c0715fa33 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/AbstractAsyncStateStreamOperatorV2Test.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/AbstractAsyncStateStreamOperatorV2Test.java
@@ -18,7 +18,6 @@
package org.apache.flink.streaming.api.operators;
-import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.functions.KeySelector;
@@ -27,13 +26,10 @@ import org.apache.flink.runtime.asyncprocessing.AsyncExecutionController;
import org.apache.flink.runtime.asyncprocessing.StateRequestType;
import org.apache.flink.runtime.asyncprocessing.operators.AbstractAsyncStateStreamOperatorTest.TestKeySelector;
import org.apache.flink.runtime.asyncprocessing.operators.AbstractAsyncStateStreamOperatorV2;
-import org.apache.flink.runtime.checkpoint.CheckpointOptions;
-import org.apache.flink.runtime.checkpoint.CheckpointType;
import org.apache.flink.runtime.state.CheckpointStorageLocationReference;
import org.apache.flink.runtime.state.VoidNamespace;
import org.apache.flink.runtime.state.VoidNamespaceSerializer;
import org.apache.flink.runtime.state.hashmap.HashMapStateBackend;
-import org.apache.flink.runtime.state.storage.JobManagerCheckpointStorage;
import org.apache.flink.streaming.runtime.io.RecordProcessorUtils;
import org.apache.flink.streaming.runtime.operators.asyncprocessing.ElementOrder;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
@@ -220,13 +216,7 @@ class AbstractAsyncStateStreamOperatorV2Test {
asyncExecutionController.handleRequest(null, StateRequestType.VALUE_GET, null);
testOperator.postProcessElement();
assertThat(asyncExecutionController.getInFlightRecordNum()).isEqualTo(1);
- testOperator.snapshotState(
- 1,
- 1,
- new CheckpointOptions(CheckpointType.CHECKPOINT, locationReference),
- new JobManagerCheckpointStorage()
- .createCheckpointStorage(new JobID())
- .resolveCheckpointStorageLocation(1, locationReference));
+ testOperator.prepareSnapshotPreBarrier(1);
assertThat(asyncExecutionController.getInFlightRecordNum()).isEqualTo(0);
}
}