You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by le...@apache.org on 2024/05/16 02:12:21 UTC
(flink) branch master updated: [FLINK-32092][tests] Integrate snapshot file-merging with existing IT cases (#24789)
This is an automated email from the ASF dual-hosted git repository.
leiyanfei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new b87ead743dc [FLINK-32092][tests] Integrate snapshot file-merging with existing IT cases (#24789)
b87ead743dc is described below
commit b87ead743dca161cdae8a1fef761954d206b81fb
Author: Yanfei Lei <fr...@gmail.com>
AuthorDate: Thu May 16 10:12:12 2024 +0800
[FLINK-32092][tests] Integrate snapshot file-merging with existing IT cases (#24789)
---
.../flink/runtime/state/OperatorStateRestoreOperation.java | 4 +++-
.../runtime/state/filemerging/SegmentFileStateHandle.java | 3 ++-
.../org/apache/flink/streaming/runtime/tasks/StreamTask.java | 10 ++++++----
.../org/apache/flink/streaming/util/TestStreamEnvironment.java | 7 ++++++-
.../flink/test/checkpointing/ChangelogRecoveryITCaseBase.java | 6 +++++-
.../flink/test/checkpointing/RestoreUpgradedJobITCase.java | 7 ++++++-
.../apache/flink/test/state/ChangelogCompatibilityITCase.java | 4 ++++
.../flink/test/state/ChangelogRecoveryCachingITCase.java | 3 +++
.../org/apache/flink/test/state/ChangelogRescalingITCase.java | 3 +++
9 files changed, 38 insertions(+), 9 deletions(-)
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorStateRestoreOperation.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorStateRestoreOperation.java
index 33ec3d92ab8..e7ba144d5b7 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorStateRestoreOperation.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorStateRestoreOperation.java
@@ -24,6 +24,7 @@ import org.apache.flink.core.fs.CloseableRegistry;
import org.apache.flink.core.fs.FSDataInputStream;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataInputViewStreamWrapper;
+import org.apache.flink.runtime.state.filemerging.EmptyFileMergingOperatorStreamStateHandle;
import org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot;
import org.apache.flink.util.Preconditions;
@@ -66,7 +67,8 @@ public class OperatorStateRestoreOperation implements RestoreOperation<Void> {
for (OperatorStateHandle stateHandle : stateHandles) {
- if (stateHandle == null) {
+ if (stateHandle == null
+ || stateHandle instanceof EmptyFileMergingOperatorStreamStateHandle) {
continue;
}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/filemerging/SegmentFileStateHandle.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/filemerging/SegmentFileStateHandle.java
index 11d68e7b394..a4b31cc310e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/filemerging/SegmentFileStateHandle.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/filemerging/SegmentFileStateHandle.java
@@ -106,7 +106,8 @@ public class SegmentFileStateHandle implements StreamStateHandle {
@Override
public PhysicalStateHandleID getStreamStateHandleID() {
- return new PhysicalStateHandleID(filePath.toUri().toString());
+ return new PhysicalStateHandleID(
+ String.format("%s-%d-%d", filePath.toUri(), startPos, stateSize));
}
public long getStartPos() {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
index e0ffe9b9267..433edf63f25 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
@@ -562,10 +562,12 @@ public abstract class StreamTask<OUT, OP extends StreamOperator<OUT>>
return checkpointStorageAccess;
}
try {
- CheckpointStorageWorkerView mergingCheckpointStorageAccess =
- checkpointStorageAccess.toFileMergingStorage(
- fileMergingSnapshotManager, environment);
- return (CheckpointStorageAccess) mergingCheckpointStorageAccess;
+ CheckpointStorageAccess mergingCheckpointStorageAccess =
+ (CheckpointStorageAccess)
+ checkpointStorageAccess.toFileMergingStorage(
+ fileMergingSnapshotManager, environment);
+ mergingCheckpointStorageAccess.initializeBaseLocationsForCheckpoint();
+ return mergingCheckpointStorageAccess;
} catch (IOException e) {
LOG.warn(
"Initiating FsMergingCheckpointStorageAccess failed "
diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/streaming/util/TestStreamEnvironment.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/streaming/util/TestStreamEnvironment.java
index cd64384eaaf..0f319c8b428 100644
--- a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/streaming/util/TestStreamEnvironment.java
+++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/streaming/util/TestStreamEnvironment.java
@@ -122,6 +122,9 @@ public class TestStreamEnvironment extends StreamExecutionEnvironment {
Duration.ofSeconds(2));
randomize(conf, CheckpointingOptions.CLEANER_PARALLEL_MODE, true, false);
randomize(conf, ExecutionOptions.SNAPSHOT_COMPRESSION, true, false);
+ if (!conf.contains(CheckpointingOptions.FILE_MERGING_ENABLED)) {
+ randomize(conf, CheckpointingOptions.FILE_MERGING_ENABLED, true);
+ }
}
randomize(
@@ -134,7 +137,9 @@ public class TestStreamEnvironment extends StreamExecutionEnvironment {
false);
// randomize ITTests for enabling state change log
- if (!conf.contains(StateChangelogOptions.ENABLE_STATE_CHANGE_LOG)) {
+ // TODO: remove the file merging check after FLINK-32085
+ if (!conf.contains(StateChangelogOptions.ENABLE_STATE_CHANGE_LOG)
+ && !conf.get(CheckpointingOptions.FILE_MERGING_ENABLED)) {
if (STATE_CHANGE_LOG_CONFIG.equalsIgnoreCase(STATE_CHANGE_LOG_CONFIG_ON)) {
conf.set(StateChangelogOptions.ENABLE_STATE_CHANGE_LOG, true);
} else if (STATE_CHANGE_LOG_CONFIG.equalsIgnoreCase(STATE_CHANGE_LOG_CONFIG_RAND)) {
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ChangelogRecoveryITCaseBase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ChangelogRecoveryITCaseBase.java
index b5bc898954a..170cb83ec2c 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ChangelogRecoveryITCaseBase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ChangelogRecoveryITCaseBase.java
@@ -93,6 +93,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import java.util.stream.Stream;
+import static org.apache.flink.configuration.CheckpointingOptions.FILE_MERGING_ENABLED;
import static org.apache.flink.runtime.testutils.CommonTestUtils.getLatestCompletedCheckpointPath;
import static org.apache.flink.shaded.guava31.com.google.common.collect.Iterables.get;
import static org.apache.flink.test.util.TestUtils.loadCheckpointMetadata;
@@ -154,7 +155,10 @@ public abstract class ChangelogRecoveryITCaseBase extends TestLogger {
int restartAttempts,
long materializationInterval,
int materializationMaxFailure) {
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ Configuration conf = new Configuration();
+ conf.set(
+ FILE_MERGING_ENABLED, false); // TODO: remove file merging setting after FLINK-32085
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(conf);
env.enableCheckpointing(checkpointInterval).enableChangelogStateBackend(true);
env.getCheckpointConfig().enableUnalignedCheckpoints(false);
env.setStateBackend(stateBackend)
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/RestoreUpgradedJobITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/RestoreUpgradedJobITCase.java
index 8899047ad9b..38daeb661ed 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/RestoreUpgradedJobITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/RestoreUpgradedJobITCase.java
@@ -22,6 +22,7 @@ import org.apache.flink.api.common.functions.RichMapFunction;
import org.apache.flink.api.common.state.ListState;
import org.apache.flink.api.common.state.ListStateDescriptor;
import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.configuration.CheckpointingOptions;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.StateRecoveryOptions;
import org.apache.flink.core.execution.JobClient;
@@ -166,7 +167,10 @@ public class RestoreUpgradedJobITCase extends TestLogger {
@NotNull
private String runOriginalJob() throws Exception {
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ Configuration conf = new Configuration();
+ // TODO: remove this after FLINK-32081
+ conf.set(CheckpointingOptions.FILE_MERGING_ENABLED, false);
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(conf);
env.getCheckpointConfig()
.setExternalizedCheckpointCleanup(
CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
@@ -215,6 +219,7 @@ public class RestoreUpgradedJobITCase extends TestLogger {
StreamExecutionEnvironment env;
Configuration conf = new Configuration();
conf.set(StateRecoveryOptions.SAVEPOINT_PATH, snapshotPath);
+ conf.set(CheckpointingOptions.FILE_MERGING_ENABLED, false);
env = StreamExecutionEnvironment.getExecutionEnvironment(conf);
env.setParallelism(PARALLELISM);
env.addSource(new StringSource(allDataEmittedLatch))
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/ChangelogCompatibilityITCase.java b/flink-tests/src/test/java/org/apache/flink/test/state/ChangelogCompatibilityITCase.java
index 8b74d203148..85eb4a5863d 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/state/ChangelogCompatibilityITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/ChangelogCompatibilityITCase.java
@@ -45,6 +45,7 @@ import java.util.NoSuchElementException;
import java.util.Optional;
import static org.apache.flink.configuration.CheckpointingOptions.CHECKPOINTS_DIRECTORY;
+import static org.apache.flink.configuration.CheckpointingOptions.FILE_MERGING_ENABLED;
import static org.apache.flink.configuration.CheckpointingOptions.SAVEPOINT_DIRECTORY;
import static org.apache.flink.runtime.jobgraph.SavepointRestoreSettings.forPath;
import static org.apache.flink.runtime.testutils.CommonTestUtils.waitForAllTaskRunning;
@@ -135,6 +136,9 @@ public class ChangelogCompatibilityITCase {
}
private StreamExecutionEnvironment initEnvironment() {
+ Configuration conf = new Configuration();
+ conf.set(
+ FILE_MERGING_ENABLED, false); // TODO: remove file merging setting after FLINK-32085
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.enableChangelogStateBackend(testCase.startWithChangelog);
if (testCase.restoreSource == RestoreSource.CHECKPOINT) {
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/ChangelogRecoveryCachingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/state/ChangelogRecoveryCachingITCase.java
index 543512643ac..a4770f525df 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/state/ChangelogRecoveryCachingITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/ChangelogRecoveryCachingITCase.java
@@ -62,6 +62,7 @@ import static org.apache.flink.changelog.fs.FsStateChangelogOptions.CACHE_IDLE_T
import static org.apache.flink.changelog.fs.FsStateChangelogOptions.PREEMPTIVE_PERSIST_THRESHOLD;
import static org.apache.flink.configuration.CheckpointingOptions.CHECKPOINTS_DIRECTORY;
import static org.apache.flink.configuration.CheckpointingOptions.CHECKPOINT_STORAGE;
+import static org.apache.flink.configuration.CheckpointingOptions.FILE_MERGING_ENABLED;
import static org.apache.flink.configuration.CoreOptions.DEFAULT_PARALLELISM;
import static org.apache.flink.configuration.RestartStrategyOptions.RESTART_STRATEGY;
import static org.apache.flink.configuration.StateBackendOptions.STATE_BACKEND;
@@ -187,6 +188,8 @@ public class ChangelogRecoveryCachingITCase extends TestLogger {
1); // prevent file is opened multiple times
conf.set(BUFFER_DEBLOAT_ENABLED, false); // prevent randomization
conf.set(RESTART_STRATEGY, "none"); // not expecting any failures
+ conf.set(
+ FILE_MERGING_ENABLED, false); // TODO: remove file merging setting after FLINK-32085
return conf;
}
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/ChangelogRescalingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/state/ChangelogRescalingITCase.java
index b1bcdb7d18c..7a5a2670969 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/state/ChangelogRescalingITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/ChangelogRescalingITCase.java
@@ -70,6 +70,7 @@ import java.util.function.Consumer;
import static org.apache.flink.changelog.fs.FsStateChangelogOptions.PREEMPTIVE_PERSIST_THRESHOLD;
import static org.apache.flink.configuration.CheckpointingOptions.CHECKPOINTS_DIRECTORY;
import static org.apache.flink.configuration.CheckpointingOptions.CHECKPOINT_STORAGE;
+import static org.apache.flink.configuration.CheckpointingOptions.FILE_MERGING_ENABLED;
import static org.apache.flink.configuration.CheckpointingOptions.FS_SMALL_FILE_THRESHOLD;
import static org.apache.flink.configuration.CoreOptions.DEFAULT_PARALLELISM;
import static org.apache.flink.configuration.PipelineOptions.OBJECT_REUSE;
@@ -232,6 +233,8 @@ public class ChangelogRescalingITCase extends TestLogger {
conf.set(ALIGNED_CHECKPOINT_TIMEOUT, Duration.ZERO); // prevent randomization
conf.set(BUFFER_DEBLOAT_ENABLED, false); // prevent randomization
conf.set(RESTART_STRATEGY, "none"); // not expecting any failures
+ conf.set(
+ FILE_MERGING_ENABLED, false); // TODO: remove file merging setting after FLINK-32085
return conf;
}