You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by za...@apache.org on 2025/07/28 03:14:17 UTC
(flink) branch master updated: [FLINK-37294][state] Support state migration between disabling and enabling ttl in HeapKeyedStateBackend (#26651)
This is an automated email from the ASF dual-hosted git repository.
zakelly pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 076b8bd66c6 [FLINK-37294][state] Support state migration between disabling and enabling ttl in HeapKeyedStateBackend (#26651)
076b8bd66c6 is described below
commit 076b8bd66c6dae125ccf57ec10c638c4d2d3e0b0
Author: hejufang <he...@bytedance.com>
AuthorDate: Mon Jul 28 11:14:11 2025 +0800
[FLINK-37294][state] Support state migration between disabling and enabling ttl in HeapKeyedStateBackend (#26651)
Co-authored-by: Xiangyu Feng <xi...@gmail.com>
---
.../runtime/state/heap/AbstractHeapState.java | 15 ++++
.../runtime/state/heap/HeapKeyedStateBackend.java | 67 +++++++++++++---
.../flink/runtime/state/heap/HeapListState.java | 17 ++++
.../flink/runtime/state/heap/HeapMapState.java | 25 ++++++
.../flink/runtime/state/heap/StateTable.java | 26 +++++-
.../state/StateBackendMigrationTestBase.java | 83 ++++++++++++++-----
.../flink/runtime/state/ttl/TtlStateTestBase.java | 5 +-
.../ChangelogStateBackendMigrationTest.java | 14 ----
.../EmbeddedRocksDBStateBackendMigrationTest.java | 92 ----------------------
.../rocksdb/RocksDBStateBackendMigrationTest.java | 92 ----------------------
.../state/rocksdb/ttl/RocksDBTtlStateTestBase.java | 21 -----
11 files changed, 200 insertions(+), 257 deletions(-)
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/AbstractHeapState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/AbstractHeapState.java
index 0de3ba1a777..4db1f4ca196 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/AbstractHeapState.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/AbstractHeapState.java
@@ -24,6 +24,9 @@ import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.queryablestate.client.state.serialization.KvStateSerializer;
import org.apache.flink.runtime.state.internal.InternalKvState;
+import org.apache.flink.runtime.state.ttl.TtlAwareSerializer;
+import org.apache.flink.runtime.state.ttl.TtlTimeProvider;
+import org.apache.flink.runtime.state.ttl.TtlValue;
import org.apache.flink.util.Preconditions;
/**
@@ -112,6 +115,18 @@ public abstract class AbstractHeapState<K, N, SV> implements InternalKvState<K,
return KvStateSerializer.serializeValue(result, safeValueSerializer);
}
+ @SuppressWarnings("unchecked")
+ public SV migrateTtlValue(
+ SV stateValue,
+ TtlAwareSerializer<SV, ?> currentTtlAwareSerializer,
+ TtlTimeProvider ttlTimeProvider) {
+ if (currentTtlAwareSerializer.isTtlEnabled()) {
+ return (SV) new TtlValue<>(stateValue, ttlTimeProvider.currentTimestamp());
+ }
+
+ return (SV) ((TtlValue<?>) stateValue).getUserValue();
+ }
+
/** This should only be used for testing. */
@VisibleForTesting
public StateTable<K, N, SV> getStateTable() {
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java
index 20aea092694..cf8e56dfba7 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java
@@ -50,6 +50,7 @@ import org.apache.flink.runtime.state.StateEntry;
import org.apache.flink.runtime.state.StateSnapshotRestore;
import org.apache.flink.runtime.state.StateSnapshotTransformer.StateSnapshotTransformFactory;
import org.apache.flink.runtime.state.StateSnapshotTransformers;
+import org.apache.flink.runtime.state.StateTransformationFunction;
import org.apache.flink.runtime.state.StreamCompressionDecorator;
import org.apache.flink.runtime.state.metrics.LatencyTrackingStateConfig;
import org.apache.flink.runtime.state.metrics.SizeTrackingStateConfig;
@@ -65,6 +66,7 @@ import javax.annotation.Nonnull;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Spliterators;
@@ -212,7 +214,7 @@ public class HeapKeyedStateBackend<K> extends AbstractKeyedStateBackend<K> {
StateDescriptor<?, V> stateDesc,
@Nonnull StateSnapshotTransformFactory<V> snapshotTransformFactory,
boolean allowFutureMetadataUpdates)
- throws StateMigrationException {
+ throws Exception {
@SuppressWarnings("unchecked")
StateTable<K, N, V> stateTable =
@@ -259,17 +261,12 @@ public class HeapKeyedStateBackend<K> extends AbstractKeyedStateBackend<K> {
+ ") must not be incompatible with the old state serializer ("
+ previousStateSerializer
+ ").");
- }
-
- // HeapKeyedStateBackend doesn't support ttl state migration currently.
- if (TtlAwareSerializer.needTtlStateMigration(
- previousStateSerializer, newStateSerializer)) {
- throw new StateMigrationException(
- "For heap backends, the new state serializer ("
- + newStateSerializer
- + ") must not need ttl state migration with the old state serializer ("
- + previousStateSerializer
- + ").");
+ } else if (stateCompatibility.isCompatibleAfterMigration()
+ && TtlAwareSerializer.needTtlStateMigration(
+ previousStateSerializer, newStateSerializer)) {
+ // State migration without ttl change will be performed automatically during
+ // checkpoint, so we only preform state ttl migration here.
+ migrateTtlAwareStateValues(stateDesc, previousStateSerializer, newStateSerializer);
}
restoredKvMetaInfo =
@@ -299,6 +296,52 @@ public class HeapKeyedStateBackend<K> extends AbstractKeyedStateBackend<K> {
return stateTable;
}
+ @SuppressWarnings("unchecked")
+ private <V, N> void migrateTtlAwareStateValues(
+ StateDescriptor<?, V> stateDesc,
+ TypeSerializer<V> previousSerializer,
+ TypeSerializer<V> currentSerializer)
+ throws Exception {
+ final StateTable<K, N, V> stateTable =
+ (StateTable<K, N, V>) registeredKVStates.get(stateDesc.getName());
+ final Iterator<StateEntry<K, N, V>> iterator = stateTable.iterator();
+
+ LOG.info(
+ "Performing state migration for state {} because the state serializer's ttl"
+ + " config has been changed from {} to {}.",
+ stateDesc,
+ TtlAwareSerializer.isSerializerTtlEnabled(previousSerializer),
+ TtlAwareSerializer.isSerializerTtlEnabled(currentSerializer));
+
+ // we need to get an actual state instance because migration is different
+ // for different state types. For example, ListState needs to deal with
+ // individual elements
+ StateCreateFactory stateCreateFactory = STATE_CREATE_FACTORIES.get(stateDesc.getType());
+ if (stateCreateFactory == null) {
+ throw new FlinkRuntimeException(stateNotSupportedMessage(stateDesc));
+ }
+ State state =
+ stateCreateFactory.createState(stateDesc, stateTable, stateTable.keySerializer);
+ if (!(state instanceof AbstractHeapState)) {
+ throw new FlinkRuntimeException(
+ "State should be an AbstractRocksDBState but is " + state);
+ }
+ AbstractHeapState<K, N, V> heapState = (AbstractHeapState<K, N, V>) state;
+ TtlAwareSerializer<V, ?> currentTtlAwareSerializer =
+ (TtlAwareSerializer<V, ?>)
+ TtlAwareSerializer.wrapTtlAwareSerializer(currentSerializer);
+
+ stateTable.transformAll(
+ null,
+ new StateTransformationFunction<V, V>() {
+ @Override
+ public V apply(V previousState, V value) throws Exception {
+ return heapState.migrateTtlValue(
+ previousState, currentTtlAwareSerializer, ttlTimeProvider);
+ }
+ });
+ }
+
@SuppressWarnings("unchecked")
@Override
public <N> Stream<K> getKeys(String state, N namespace) {
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapListState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapListState.java
index 919700f539d..6f719ae234e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapListState.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapListState.java
@@ -27,6 +27,9 @@ import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
import org.apache.flink.queryablestate.client.state.serialization.KvStateSerializer;
import org.apache.flink.runtime.state.internal.InternalListState;
+import org.apache.flink.runtime.state.ttl.TtlAwareSerializer;
+import org.apache.flink.runtime.state.ttl.TtlTimeProvider;
+import org.apache.flink.runtime.state.ttl.TtlValue;
import org.apache.flink.util.Preconditions;
import java.io.ByteArrayOutputStream;
@@ -142,6 +145,20 @@ class HeapListState<K, N, V> extends AbstractHeapMergingState<K, N, V, List<V>,
return baos.toByteArray();
}
+ @SuppressWarnings("unchecked")
+ public List<V> migrateTtlValue(
+ List<V> stateValue,
+ TtlAwareSerializer<List<V>, ?> currentTtlAwareSerializer,
+ TtlTimeProvider ttlTimeProvider) {
+ if (currentTtlAwareSerializer.isTtlEnabled()) {
+ stateValue.replaceAll(v -> (V) new TtlValue<>(v, ttlTimeProvider.currentTimestamp()));
+ } else {
+ stateValue.replaceAll(v -> (V) ((TtlValue<?>) v).getUserValue());
+ }
+
+ return stateValue;
+ }
+
// ------------------------------------------------------------------------
// state merging
// ------------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapMapState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapMapState.java
index 0e7eda514d0..ef171f52b3a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapMapState.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapMapState.java
@@ -26,6 +26,9 @@ import org.apache.flink.api.common.typeutils.base.MapSerializer;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.queryablestate.client.state.serialization.KvStateSerializer;
import org.apache.flink.runtime.state.internal.InternalMapState;
+import org.apache.flink.runtime.state.ttl.TtlAwareSerializer;
+import org.apache.flink.runtime.state.ttl.TtlTimeProvider;
+import org.apache.flink.runtime.state.ttl.TtlValue;
import org.apache.flink.util.Preconditions;
import java.util.Collections;
@@ -200,6 +203,28 @@ class HeapMapState<K, N, UK, UV> extends AbstractHeapState<K, N, Map<UK, UV>>
result.entrySet(), dupUserKeySerializer, dupUserValueSerializer);
}
+ @SuppressWarnings("unchecked")
+ public Map<UK, UV> migrateTtlValue(
+ Map<UK, UV> stateValue,
+ TtlAwareSerializer<Map<UK, UV>, ?> currentTtlAwareSerializer,
+ TtlTimeProvider ttlTimeProvider) {
+
+ if (currentTtlAwareSerializer.isTtlEnabled()) {
+ for (Map.Entry<UK, UV> entry : stateValue.entrySet()) {
+ UV value =
+ (UV) new TtlValue<>(entry.getValue(), ttlTimeProvider.currentTimestamp());
+ stateValue.put(entry.getKey(), value);
+ }
+ } else {
+ for (Map.Entry<UK, UV> entry : stateValue.entrySet()) {
+ UV value = (UV) ((TtlValue<?>) entry.getValue()).getUserValue();
+ stateValue.put(entry.getKey(), value);
+ }
+ }
+
+ return stateValue;
+ }
+
@SuppressWarnings("unchecked")
static <UK, UV, K, N, SV, S extends State, IS extends S> IS create(
StateDescriptor<S, SV> stateDesc,
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/StateTable.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/StateTable.java
index af48a7f4e08..d79706ac872 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/StateTable.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/StateTable.java
@@ -39,8 +39,10 @@ import javax.annotation.Nonnull;
import java.util.Arrays;
import java.util.Collection;
import java.util.Iterator;
+import java.util.List;
import java.util.Objects;
import java.util.Spliterators;
+import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
@@ -212,6 +214,29 @@ public abstract class StateTable<K, N, S>
getMapForKeyGroup(keyGroup).transform(key, namespace, value, transformation);
}
+ /**
+ * Applies the given {@link StateTransformationFunction} to all state (1st input argument),
+ * using the given value as second input argument. The result of {@link
+ * StateTransformationFunction#apply(Object, Object)} is then stored as the new state. This
+ * function is basically an optimization for get-update-put pattern.
+ *
+ * @param value the value to use in transforming the state. Can be null.
+ * @throws Exception if some exception happens in the transformation function.
+ */
+ public <T> void transformAll(T value, StateTransformationFunction<S, T> transformation)
+ throws Exception {
+ for (StateMap<K, N, S> stateMap : keyGroupedStateMaps) {
+ List<StateEntry<K, N, S>> entries =
+ StreamSupport.stream(
+ Spliterators.spliteratorUnknownSize(stateMap.iterator(), 0),
+ false)
+ .collect(Collectors.toList());
+ for (StateEntry<K, N, S> entry : entries) {
+ stateMap.transform(entry.getKey(), entry.getNamespace(), value, transformation);
+ }
+ }
+ }
+
// For queryable state ------------------------------------------------------------------------
/**
@@ -297,7 +322,6 @@ public abstract class StateTable<K, N, S>
return keyGroupRange.getStartKeyGroup();
}
- @VisibleForTesting
public StateMap<K, N, S> getMapForKeyGroup(int keyGroupIndex) {
final int pos = indexToOffset(keyGroupIndex);
if (pos >= 0 && pos < keyGroupedStateMaps.length) {
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendMigrationTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendMigrationTestBase.java
index 415d21dbf30..5ec34c9941c 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendMigrationTestBase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendMigrationTestBase.java
@@ -1203,44 +1203,85 @@ public abstract class StateBackendMigrationTestBase<B extends StateBackend> {
}
@TestTemplate
- protected void testStateMigrationAfterChangingTTLFromEnablingToDisabling() throws Exception {
+ protected void testStateMigrationAfterChangingTTLFromDisablingToEnabling() throws Exception {
final String stateName = "test-ttl";
ValueStateDescriptor<TestType> initialAccessDescriptor =
new ValueStateDescriptor<>(stateName, new TestType.V1TestTypeSerializer());
- initialAccessDescriptor.enableTimeToLive(
+ ValueStateDescriptor<TestType> newAccessDescriptorAfterRestore =
+ new ValueStateDescriptor<>(
+ stateName,
+ // restore with a V2 serializer that has a different schema
+ new TestType.V2TestTypeSerializer());
+ newAccessDescriptorAfterRestore.enableTimeToLive(
StateTtlConfig.newBuilder(Duration.ofDays(1)).build());
- ValueStateDescriptor<TestType> newAccessDescriptorAfterRestore =
- new ValueStateDescriptor<>(stateName, new TestType.V2TestTypeSerializer());
+ ListStateDescriptor<TestType> initialAccessListDescriptor =
+ new ListStateDescriptor<>(stateName, new TestType.V1TestTypeSerializer());
+ ListStateDescriptor<TestType> newAccessListDescriptorAfterRestore =
+ new ListStateDescriptor<>(
+ stateName,
+ // restore with a V2 serializer that has a different schema
+ new TestType.V2TestTypeSerializer());
+ newAccessListDescriptorAfterRestore.enableTimeToLive(
+ StateTtlConfig.newBuilder(Duration.ofDays(1)).build());
- assertThatThrownBy(
- () ->
- testKeyedValueStateUpgrade(
- initialAccessDescriptor, newAccessDescriptorAfterRestore))
- .satisfiesAnyOf(
- e -> assertThat(e).isInstanceOf(StateMigrationException.class),
- e -> assertThat(e).hasCauseInstanceOf(StateMigrationException.class));
+ MapStateDescriptor<Integer, TestType> initialAccessMapDescriptor =
+ new MapStateDescriptor<>(
+ stateName, IntSerializer.INSTANCE, new TestType.V1TestTypeSerializer());
+ MapStateDescriptor<Integer, TestType> newAccessMapDescriptorAfterRestore =
+ new MapStateDescriptor<>(
+ stateName,
+ IntSerializer.INSTANCE,
+ // restore with a V2 serializer that has a different schema
+ new TestType.V2TestTypeSerializer());
+ newAccessMapDescriptorAfterRestore.enableTimeToLive(
+ StateTtlConfig.newBuilder(Duration.ofDays(1)).build());
+
+ testKeyedValueStateUpgrade(initialAccessDescriptor, newAccessDescriptorAfterRestore);
+ testKeyedListStateUpgrade(initialAccessListDescriptor, newAccessListDescriptorAfterRestore);
+ testKeyedMapStateUpgrade(initialAccessMapDescriptor, newAccessMapDescriptorAfterRestore);
}
@TestTemplate
- protected void testStateMigrationAfterChangingTTLFromDisablingToEnabling() throws Exception {
+ protected void testStateMigrationAfterChangingTTLFromEnablingToDisabling() throws Exception {
final String stateName = "test-ttl";
ValueStateDescriptor<TestType> initialAccessDescriptor =
new ValueStateDescriptor<>(stateName, new TestType.V1TestTypeSerializer());
+ initialAccessDescriptor.enableTimeToLive(
+ StateTtlConfig.newBuilder(Duration.ofDays(1)).build());
ValueStateDescriptor<TestType> newAccessDescriptorAfterRestore =
- new ValueStateDescriptor<>(stateName, new TestType.V2TestTypeSerializer());
- newAccessDescriptorAfterRestore.enableTimeToLive(
+ new ValueStateDescriptor<>(
+ stateName,
+ // restore with a V2 serializer that has a different schema
+ new TestType.V2TestTypeSerializer());
+
+ ListStateDescriptor<TestType> initialAccessListDescriptor =
+ new ListStateDescriptor<>(stateName, new TestType.V1TestTypeSerializer());
+ initialAccessListDescriptor.enableTimeToLive(
StateTtlConfig.newBuilder(Duration.ofDays(1)).build());
+ ListStateDescriptor<TestType> newAccessListDescriptorAfterRestore =
+ new ListStateDescriptor<>(
+ stateName,
+ // restore with a V2 serializer that has a different schema
+ new TestType.V2TestTypeSerializer());
- assertThatThrownBy(
- () ->
- testKeyedValueStateUpgrade(
- initialAccessDescriptor, newAccessDescriptorAfterRestore))
- .satisfiesAnyOf(
- e -> assertThat(e).isInstanceOf(StateMigrationException.class),
- e -> assertThat(e).hasCauseInstanceOf(StateMigrationException.class));
+ MapStateDescriptor<Integer, TestType> initialAccessMapDescriptor =
+ new MapStateDescriptor<>(
+ stateName, IntSerializer.INSTANCE, new TestType.V1TestTypeSerializer());
+ initialAccessMapDescriptor.enableTimeToLive(
+ StateTtlConfig.newBuilder(Duration.ofDays(1)).build());
+ MapStateDescriptor<Integer, TestType> newAccessMapDescriptorAfterRestore =
+ new MapStateDescriptor<>(
+ stateName,
+ IntSerializer.INSTANCE,
+ // restore with a V2 serializer that has a different schema
+ new TestType.V2TestTypeSerializer());
+
+ testKeyedValueStateUpgrade(initialAccessDescriptor, newAccessDescriptorAfterRestore);
+ testKeyedListStateUpgrade(initialAccessListDescriptor, newAccessListDescriptorAfterRestore);
+ testKeyedMapStateUpgrade(initialAccessMapDescriptor, newAccessMapDescriptorAfterRestore);
}
// -------------------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/ttl/TtlStateTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/ttl/TtlStateTestBase.java
index b652932d092..ca06f4b5c6a 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/ttl/TtlStateTestBase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/ttl/TtlStateTestBase.java
@@ -31,7 +31,6 @@ import org.apache.flink.testutils.junit.extensions.parameterized.Parameter;
import org.apache.flink.testutils.junit.extensions.parameterized.ParameterizedTestExtension;
import org.apache.flink.testutils.junit.extensions.parameterized.Parameters;
import org.apache.flink.util.Preconditions;
-import org.apache.flink.util.StateMigrationException;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
@@ -46,7 +45,6 @@ import java.util.function.Consumer;
import static org.apache.flink.runtime.state.ttl.StateBackendTestContext.NUMBER_OF_KEY_GROUPS;
import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.assertj.core.api.Assumptions.assumeThat;
/** State TTL base test suite. */
@@ -508,8 +506,7 @@ public abstract class TtlStateTestBase {
sbetc.createAndRestoreKeyedStateBackend(snapshot);
sbetc.setCurrentKey("defaultKey");
- assertThatThrownBy(() -> sbetc.createState(ctx().createStateDescriptor(), ""))
- .isInstanceOf(StateMigrationException.class);
+ sbetc.createState(ctx().createStateDescriptor(), "");
}
@TestTemplate
diff --git a/flink-state-backends/flink-statebackend-changelog/src/test/java/org/apache/flink/state/changelog/ChangelogStateBackendMigrationTest.java b/flink-state-backends/flink-statebackend-changelog/src/test/java/org/apache/flink/state/changelog/ChangelogStateBackendMigrationTest.java
index 858570bb67d..fb2b69698c0 100644
--- a/flink-state-backends/flink-statebackend-changelog/src/test/java/org/apache/flink/state/changelog/ChangelogStateBackendMigrationTest.java
+++ b/flink-state-backends/flink-statebackend-changelog/src/test/java/org/apache/flink/state/changelog/ChangelogStateBackendMigrationTest.java
@@ -64,18 +64,4 @@ public class ChangelogStateBackendMigrationTest
// TODO support checking key serializer
return false;
}
-
- @Override
- protected void testStateMigrationAfterChangingTTLFromDisablingToEnabling() throws Exception {
- if (!(this.delegatedStateBackendSupplier.get() instanceof EmbeddedRocksDBStateBackend)) {
- super.testStateMigrationAfterChangingTTLFromDisablingToEnabling();
- }
- }
-
- @Override
- protected void testStateMigrationAfterChangingTTLFromEnablingToDisabling() throws Exception {
- if (!(this.delegatedStateBackendSupplier.get() instanceof EmbeddedRocksDBStateBackend)) {
- super.testStateMigrationAfterChangingTTLFromEnablingToDisabling();
- }
- }
}
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/state/rocksdb/EmbeddedRocksDBStateBackendMigrationTest.java b/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/state/rocksdb/EmbeddedRocksDBStateBackendMigrationTest.java
index 8f94c7868b9..835d9ed0d59 100644
--- a/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/state/rocksdb/EmbeddedRocksDBStateBackendMigrationTest.java
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/state/rocksdb/EmbeddedRocksDBStateBackendMigrationTest.java
@@ -18,29 +18,21 @@
package org.apache.flink.state.rocksdb;
-import org.apache.flink.api.common.state.ListStateDescriptor;
-import org.apache.flink.api.common.state.MapStateDescriptor;
-import org.apache.flink.api.common.state.StateTtlConfig;
-import org.apache.flink.api.common.state.ValueStateDescriptor;
-import org.apache.flink.api.common.typeutils.base.IntSerializer;
import org.apache.flink.runtime.state.CheckpointStorage;
import org.apache.flink.runtime.state.StateBackendMigrationTestBase;
import org.apache.flink.runtime.state.storage.FileSystemCheckpointStorage;
import org.apache.flink.runtime.state.storage.JobManagerCheckpointStorage;
-import org.apache.flink.runtime.testutils.statemigration.TestType;
import org.apache.flink.testutils.junit.extensions.parameterized.Parameter;
import org.apache.flink.testutils.junit.extensions.parameterized.ParameterizedTestExtension;
import org.apache.flink.testutils.junit.extensions.parameterized.Parameters;
import org.apache.flink.testutils.junit.utils.TempDirUtils;
import org.apache.flink.util.function.SupplierWithException;
-import org.junit.jupiter.api.TestTemplate;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.io.TempDir;
import java.io.IOException;
import java.nio.file.Path;
-import java.time.Duration;
import java.util.Arrays;
import java.util.List;
@@ -87,88 +79,4 @@ public class EmbeddedRocksDBStateBackendMigrationTest
protected CheckpointStorage getCheckpointStorage() throws Exception {
return storageSupplier.get();
}
-
- /// Todo: Move to StateBackendMigrationTestBase when all backends support ttl state migration
- @TestTemplate
- protected void testStateMigrationAfterChangingTTLFromDisablingToEnabling() throws Exception {
- final String stateName = "test-ttl";
-
- ValueStateDescriptor<TestType> initialAccessDescriptor =
- new ValueStateDescriptor<>(stateName, new TestType.V1TestTypeSerializer());
- ValueStateDescriptor<TestType> newAccessDescriptorAfterRestore =
- new ValueStateDescriptor<>(
- stateName,
- // restore with a V2 serializer that has a different schema
- new TestType.V2TestTypeSerializer());
- newAccessDescriptorAfterRestore.enableTimeToLive(
- StateTtlConfig.newBuilder(Duration.ofDays(1)).build());
-
- ListStateDescriptor<TestType> initialAccessListDescriptor =
- new ListStateDescriptor<>(stateName, new TestType.V1TestTypeSerializer());
- ListStateDescriptor<TestType> newAccessListDescriptorAfterRestore =
- new ListStateDescriptor<>(
- stateName,
- // restore with a V2 serializer that has a different schema
- new TestType.V2TestTypeSerializer());
- newAccessListDescriptorAfterRestore.enableTimeToLive(
- StateTtlConfig.newBuilder(Duration.ofDays(1)).build());
-
- MapStateDescriptor<Integer, TestType> initialAccessMapDescriptor =
- new MapStateDescriptor<>(
- stateName, IntSerializer.INSTANCE, new TestType.V1TestTypeSerializer());
- MapStateDescriptor<Integer, TestType> newAccessMapDescriptorAfterRestore =
- new MapStateDescriptor<>(
- stateName,
- IntSerializer.INSTANCE,
- // restore with a V2 serializer that has a different schema
- new TestType.V2TestTypeSerializer());
- newAccessMapDescriptorAfterRestore.enableTimeToLive(
- StateTtlConfig.newBuilder(Duration.ofDays(1)).build());
-
- testKeyedValueStateUpgrade(initialAccessDescriptor, newAccessDescriptorAfterRestore);
- testKeyedListStateUpgrade(initialAccessListDescriptor, newAccessListDescriptorAfterRestore);
- testKeyedMapStateUpgrade(initialAccessMapDescriptor, newAccessMapDescriptorAfterRestore);
- }
-
- /// Todo: Move to StateBackendMigrationTestBase when all backends support ttl state migration
- @TestTemplate
- protected void testStateMigrationAfterChangingTTLFromEnablingToDisabling() throws Exception {
- final String stateName = "test-ttl";
-
- ValueStateDescriptor<TestType> initialAccessDescriptor =
- new ValueStateDescriptor<>(stateName, new TestType.V1TestTypeSerializer());
- initialAccessDescriptor.enableTimeToLive(
- StateTtlConfig.newBuilder(Duration.ofDays(1)).build());
- ValueStateDescriptor<TestType> newAccessDescriptorAfterRestore =
- new ValueStateDescriptor<>(
- stateName,
- // restore with a V2 serializer that has a different schema
- new TestType.V2TestTypeSerializer());
-
- ListStateDescriptor<TestType> initialAccessListDescriptor =
- new ListStateDescriptor<>(stateName, new TestType.V1TestTypeSerializer());
- initialAccessListDescriptor.enableTimeToLive(
- StateTtlConfig.newBuilder(Duration.ofDays(1)).build());
- ListStateDescriptor<TestType> newAccessListDescriptorAfterRestore =
- new ListStateDescriptor<>(
- stateName,
- // restore with a V2 serializer that has a different schema
- new TestType.V2TestTypeSerializer());
-
- MapStateDescriptor<Integer, TestType> initialAccessMapDescriptor =
- new MapStateDescriptor<>(
- stateName, IntSerializer.INSTANCE, new TestType.V1TestTypeSerializer());
- initialAccessMapDescriptor.enableTimeToLive(
- StateTtlConfig.newBuilder(Duration.ofDays(1)).build());
- MapStateDescriptor<Integer, TestType> newAccessMapDescriptorAfterRestore =
- new MapStateDescriptor<>(
- stateName,
- IntSerializer.INSTANCE,
- // restore with a V2 serializer that has a different schema
- new TestType.V2TestTypeSerializer());
-
- testKeyedValueStateUpgrade(initialAccessDescriptor, newAccessDescriptorAfterRestore);
- testKeyedListStateUpgrade(initialAccessListDescriptor, newAccessListDescriptorAfterRestore);
- testKeyedMapStateUpgrade(initialAccessMapDescriptor, newAccessMapDescriptorAfterRestore);
- }
}
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/state/rocksdb/RocksDBStateBackendMigrationTest.java b/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/state/rocksdb/RocksDBStateBackendMigrationTest.java
index 964b45f4e0d..6baa2c7e4a7 100644
--- a/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/state/rocksdb/RocksDBStateBackendMigrationTest.java
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/state/rocksdb/RocksDBStateBackendMigrationTest.java
@@ -18,27 +18,19 @@
package org.apache.flink.state.rocksdb;
-import org.apache.flink.api.common.state.ListStateDescriptor;
-import org.apache.flink.api.common.state.MapStateDescriptor;
-import org.apache.flink.api.common.state.StateTtlConfig;
-import org.apache.flink.api.common.state.ValueStateDescriptor;
-import org.apache.flink.api.common.typeutils.base.IntSerializer;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.state.CheckpointStorage;
import org.apache.flink.runtime.state.StateBackendMigrationTestBase;
import org.apache.flink.runtime.state.storage.FileSystemCheckpointStorage;
-import org.apache.flink.runtime.testutils.statemigration.TestType;
import org.apache.flink.testutils.junit.extensions.parameterized.Parameter;
import org.apache.flink.testutils.junit.extensions.parameterized.ParameterizedTestExtension;
import org.apache.flink.testutils.junit.extensions.parameterized.Parameters;
import org.apache.flink.testutils.junit.utils.TempDirUtils;
import org.apache.flink.util.TernaryBoolean;
-import org.junit.jupiter.api.TestTemplate;
import org.junit.jupiter.api.extension.ExtendWith;
import java.io.IOException;
-import java.time.Duration;
import java.util.Arrays;
import java.util.Collection;
@@ -78,88 +70,4 @@ public class RocksDBStateBackendMigrationTest
String checkpointPath = TempDirUtils.newFolder(tempFolder).toURI().toString();
return new FileSystemCheckpointStorage(checkpointPath);
}
-
- /// Todo: Move to StateBackendMigrationTestBase when all backends support ttl state migration
- @TestTemplate
- protected void testStateMigrationAfterChangingTTLFromDisablingToEnabling() throws Exception {
- final String stateName = "test-ttl";
-
- ValueStateDescriptor<TestType> initialAccessDescriptor =
- new ValueStateDescriptor<>(stateName, new TestType.V1TestTypeSerializer());
- ValueStateDescriptor<TestType> newAccessDescriptorAfterRestore =
- new ValueStateDescriptor<>(
- stateName,
- // restore with a V2 serializer that has a different schema
- new TestType.V2TestTypeSerializer());
- newAccessDescriptorAfterRestore.enableTimeToLive(
- StateTtlConfig.newBuilder(Duration.ofDays(1)).build());
-
- ListStateDescriptor<TestType> initialAccessListDescriptor =
- new ListStateDescriptor<>(stateName, new TestType.V1TestTypeSerializer());
- ListStateDescriptor<TestType> newAccessListDescriptorAfterRestore =
- new ListStateDescriptor<>(
- stateName,
- // restore with a V2 serializer that has a different schema
- new TestType.V2TestTypeSerializer());
- newAccessListDescriptorAfterRestore.enableTimeToLive(
- StateTtlConfig.newBuilder(Duration.ofDays(1)).build());
-
- MapStateDescriptor<Integer, TestType> initialAccessMapDescriptor =
- new MapStateDescriptor<>(
- stateName, IntSerializer.INSTANCE, new TestType.V1TestTypeSerializer());
- MapStateDescriptor<Integer, TestType> newAccessMapDescriptorAfterRestore =
- new MapStateDescriptor<>(
- stateName,
- IntSerializer.INSTANCE,
- // restore with a V2 serializer that has a different schema
- new TestType.V2TestTypeSerializer());
- newAccessMapDescriptorAfterRestore.enableTimeToLive(
- StateTtlConfig.newBuilder(Duration.ofDays(1)).build());
-
- testKeyedValueStateUpgrade(initialAccessDescriptor, newAccessDescriptorAfterRestore);
- testKeyedListStateUpgrade(initialAccessListDescriptor, newAccessListDescriptorAfterRestore);
- testKeyedMapStateUpgrade(initialAccessMapDescriptor, newAccessMapDescriptorAfterRestore);
- }
-
- /// Todo: Move to StateBackendMigrationTestBase when all backends support ttl state migration
- @TestTemplate
- protected void testStateMigrationAfterChangingTTLFromEnablingToDisabling() throws Exception {
- final String stateName = "test-ttl";
-
- ValueStateDescriptor<TestType> initialAccessDescriptor =
- new ValueStateDescriptor<>(stateName, new TestType.V1TestTypeSerializer());
- initialAccessDescriptor.enableTimeToLive(
- StateTtlConfig.newBuilder(Duration.ofDays(1)).build());
- ValueStateDescriptor<TestType> newAccessDescriptorAfterRestore =
- new ValueStateDescriptor<>(
- stateName,
- // restore with a V2 serializer that has a different schema
- new TestType.V2TestTypeSerializer());
-
- ListStateDescriptor<TestType> initialAccessListDescriptor =
- new ListStateDescriptor<>(stateName, new TestType.V1TestTypeSerializer());
- initialAccessListDescriptor.enableTimeToLive(
- StateTtlConfig.newBuilder(Duration.ofDays(1)).build());
- ListStateDescriptor<TestType> newAccessListDescriptorAfterRestore =
- new ListStateDescriptor<>(
- stateName,
- // restore with a V2 serializer that has a different schema
- new TestType.V2TestTypeSerializer());
-
- MapStateDescriptor<Integer, TestType> initialAccessMapDescriptor =
- new MapStateDescriptor<>(
- stateName, IntSerializer.INSTANCE, new TestType.V1TestTypeSerializer());
- initialAccessMapDescriptor.enableTimeToLive(
- StateTtlConfig.newBuilder(Duration.ofDays(1)).build());
- MapStateDescriptor<Integer, TestType> newAccessMapDescriptorAfterRestore =
- new MapStateDescriptor<>(
- stateName,
- IntSerializer.INSTANCE,
- // restore with a V2 serializer that has a different schema
- new TestType.V2TestTypeSerializer());
-
- testKeyedValueStateUpgrade(initialAccessDescriptor, newAccessDescriptorAfterRestore);
- testKeyedListStateUpgrade(initialAccessListDescriptor, newAccessListDescriptorAfterRestore);
- testKeyedMapStateUpgrade(initialAccessMapDescriptor, newAccessMapDescriptorAfterRestore);
- }
}
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/state/rocksdb/ttl/RocksDBTtlStateTestBase.java b/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/state/rocksdb/ttl/RocksDBTtlStateTestBase.java
index 17d05201eaa..de5d0316c2a 100644
--- a/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/state/rocksdb/ttl/RocksDBTtlStateTestBase.java
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/state/rocksdb/ttl/RocksDBTtlStateTestBase.java
@@ -22,10 +22,8 @@ import org.apache.flink.api.common.state.StateDescriptor;
import org.apache.flink.api.common.state.StateTtlConfig;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.state.CheckpointStorage;
-import org.apache.flink.runtime.state.KeyedStateHandle;
import org.apache.flink.runtime.state.StateBackend;
import org.apache.flink.runtime.state.storage.FileSystemCheckpointStorage;
-import org.apache.flink.runtime.state.ttl.MockTtlStateTest;
import org.apache.flink.runtime.state.ttl.StateBackendTestContext;
import org.apache.flink.runtime.state.ttl.TtlStateTestBase;
import org.apache.flink.runtime.state.ttl.TtlTimeProvider;
@@ -43,7 +41,6 @@ import java.io.IOException;
import java.nio.file.Path;
import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.api.Assumptions.assumeThat;
/** Base test suite for rocksdb state TTL. */
public abstract class RocksDBTtlStateTestBase extends TtlStateTestBase {
@@ -92,24 +89,6 @@ public abstract class RocksDBTtlStateTestBase extends TtlStateTestBase {
testCompactFilter(false, false);
}
- /// Todo: Move to TtlStateTestBase when all backends support ttl state migration
- @TestTemplate
- @Override
- protected void testRestoreTtlAndRegisterNonTtlStateCompatFailure() throws Exception {
- assumeThat(this).isNotInstanceOf(MockTtlStateTest.class);
-
- initTest();
-
- timeProvider.time = 0;
- ctx().update(ctx().updateEmpty);
-
- KeyedStateHandle snapshot = sbetc.takeSnapshot();
- sbetc.createAndRestoreKeyedStateBackend(snapshot);
-
- sbetc.setCurrentKey("defaultKey");
- sbetc.createState(ctx().createStateDescriptor(), "");
- }
-
@TestTemplate
public void testCompactFilterWithSnapshot() throws Exception {
testCompactFilter(true, false);