You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by zh...@apache.org on 2024/09/19 03:54:03 UTC
(flink) branch master updated: [FLINK-36250][streaming-java] Remove CheckpointStorage-related configuration getters/setters that return/set complex Java objects.
This is an automated email from the ASF dual-hosted git repository.
zhuzh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new df412add389 [FLINK-36250][streaming-java] Remove CheckpointStorage-related configuration getters/setters that return/set complex Java objects.
df412add389 is described below
commit df412add3895e5fa88a54c26b67f188940345d0c
Author: JunRuiLee <jr...@gmail.com>
AuthorDate: Sat Sep 14 11:50:02 2024 +0800
[FLINK-36250][streaming-java] Remove CheckpointStorage-related configuration getters/setters that return/set complex Java objects.
This closes #25332.
---
.../client/program/StreamContextEnvironment.java | 23 ---
.../program/StreamContextEnvironmentTest.java | 56 -------
.../file/sink/FileSinkCompactionSwitchITCase.java | 4 +-
.../tests/DataStreamAllroundTestJobFactory.java | 5 +-
.../StickyAllocationAndLocalRecoveryTestJob.java | 6 +-
.../reference/pyflink.datastream/checkpoint.rst | 3 -
.../pyflink/datastream/checkpoint_config.py | 61 -------
.../datastream/tests/test_check_point_config.py | 11 --
.../api/environment/CheckpointConfig.java | 177 +--------------------
.../streaming/api/graph/StreamGraphGenerator.java | 5 -
.../CheckpointConfigFromConfigurationTest.java | 31 +---
.../streaming/util/CheckpointStorageUtils.java | 102 ++++++++++++
.../operators/lifecycle/BoundedSourceITCase.java | 6 +-
.../lifecycle/PartiallyFinishedSourcesITCase.java | 9 +-
.../lifecycle/StopWithSavepointITCase.java | 6 +-
.../ChangelogLocalRecoveryITCase.java | 3 +-
.../checkpointing/ChangelogRecoveryITCaseBase.java | 3 +-
.../ChangelogRecoverySwitchEnvTestBase.java | 8 +-
.../CheckpointFailureManagerITCase.java | 17 +-
.../test/checkpointing/ManualCheckpointITCase.java | 25 ++-
.../RescaleCheckpointManuallyITCase.java | 4 +-
.../checkpointing/RestoreUpgradedJobITCase.java | 5 +-
.../flink/test/checkpointing/SavepointITCase.java | 4 +-
.../UnalignedCheckpointFailureHandlingITCase.java | 38 +++--
24 files changed, 203 insertions(+), 409 deletions(-)
diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/StreamContextEnvironment.java b/flink-clients/src/main/java/org/apache/flink/client/program/StreamContextEnvironment.java
index f1bd8be1f0e..11c8ba1fdf7 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/program/StreamContextEnvironment.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/program/StreamContextEnvironment.java
@@ -23,7 +23,6 @@ import org.apache.flink.api.common.JobExecutionResult;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.client.ClientUtils;
import org.apache.flink.client.cli.ClientOptions;
-import org.apache.flink.configuration.CheckpointingOptions;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.DeploymentOptions;
import org.apache.flink.core.execution.DetachedJobExecutionResult;
@@ -31,7 +30,6 @@ import org.apache.flink.core.execution.JobClient;
import org.apache.flink.core.execution.JobListener;
import org.apache.flink.core.execution.PipelineExecutorServiceLoader;
import org.apache.flink.runtime.dispatcher.ConfigurationNotAllowedMessage;
-import org.apache.flink.streaming.api.environment.CheckpointConfig;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironmentFactory;
import org.apache.flink.streaming.api.graph.StreamGraph;
@@ -49,7 +47,6 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
-import java.util.Objects;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@@ -271,7 +268,6 @@ public class StreamContextEnvironment extends StreamExecutionEnvironment {
removeProgramConfigWildcards(clusterConfigMap);
checkMainConfiguration(clusterConfigMap, errors);
- checkCheckpointStorage(clusterConfigMap, errors);
return errors;
}
@@ -300,25 +296,6 @@ public class StreamContextEnvironment extends StreamExecutionEnvironment {
k, v)));
}
- private void checkCheckpointStorage(Configuration clusterConfigMap, List<String> errors) {
- CheckpointConfig expectedCheckpointConfig = new CheckpointConfig();
- expectedCheckpointConfig.configure(clusterConfigMap);
-
- /**
- * Unfortunately, {@link CheckpointConfig#setCheckpointStorage} is not backed by a {@link
- * Configuration}, but it also has to be validated. For this validation we are implementing
- * a one off manual check.
- */
- if (!programConfigWildcards.contains(CheckpointingOptions.CHECKPOINTS_DIRECTORY.key())
- && !Objects.equals(
- checkpointCfg.getCheckpointStorage(),
- expectedCheckpointConfig.getCheckpointStorage())) {
- errors.add(
- ConfigurationNotAllowedMessage.ofConfigurationObjectSetterUsed(
- checkpointCfg.getClass().getSimpleName(), "setCheckpointStorage"));
- }
- }
-
private void checkConfigurationObject(
Configuration expectedConfiguration,
Configuration actualConfiguration,
diff --git a/flink-clients/src/test/java/org/apache/flink/client/program/StreamContextEnvironmentTest.java b/flink-clients/src/test/java/org/apache/flink/client/program/StreamContextEnvironmentTest.java
index dcbc536e1aa..d637e13cd1e 100644
--- a/flink-clients/src/test/java/org/apache/flink/client/program/StreamContextEnvironmentTest.java
+++ b/flink-clients/src/test/java/org/apache/flink/client/program/StreamContextEnvironmentTest.java
@@ -28,8 +28,6 @@ import org.apache.flink.configuration.StateRecoveryOptions;
import org.apache.flink.core.execution.CheckpointingMode;
import org.apache.flink.core.execution.PipelineExecutorFactory;
import org.apache.flink.core.execution.PipelineExecutorServiceLoader;
-import org.apache.flink.runtime.state.storage.JobManagerCheckpointStorage;
-import org.apache.flink.streaming.api.environment.CheckpointConfig;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.sink.v2.DiscardingSink;
import org.apache.flink.util.function.ThrowingConsumer;
@@ -86,37 +84,6 @@ class StreamContextEnvironmentTest {
PipelineOptions.MAX_PARALLELISM.key());
}
- @ParameterizedTest
- @MethodSource("provideExecutors")
- void testDisallowCheckpointStorage(
- ThrowingConsumer<StreamExecutionEnvironment, Exception> executor) {
- final Configuration clusterConfig = new Configuration();
- clusterConfig.set(DeploymentOptions.PROGRAM_CONFIG_ENABLED, false);
- clusterConfig.set(DeploymentOptions.TARGET, "local");
- clusterConfig.set(CheckpointingOptions.CHECKPOINTS_DIRECTORY, "file:///flink/checkpoints");
-
- final StreamContextEnvironment environment =
- constructStreamContextEnvironment(clusterConfig, Collections.emptyList());
-
- String disallowedPath = "file:///flink/disallowed/modification";
- // Change the CheckpointConfig
- environment.getCheckpointConfig().setCheckpointStorage(disallowedPath);
-
- environment.fromData(Collections.singleton(1)).sinkTo(new DiscardingSink<>());
- assertThatThrownBy(() -> executor.accept(environment))
- .isInstanceOf(MutatedConfigurationException.class)
- .hasMessageContainingAll(
- CheckpointConfig.class.getSimpleName(), "setCheckpointStorage");
-
- environment.getCheckpointConfig().setCheckpointStorage(new JobManagerCheckpointStorage());
-
- environment.fromData(Collections.singleton(1)).sinkTo(new DiscardingSink<>());
- assertThatThrownBy(() -> executor.accept(environment))
- .isInstanceOf(MutatedConfigurationException.class)
- .hasMessageContainingAll(
- CheckpointConfig.class.getSimpleName(), "setCheckpointStorage");
- }
-
@ParameterizedTest
@MethodSource("provideExecutors")
void testDisallowCheckpointStorageByConfiguration(
@@ -147,29 +114,6 @@ class StreamContextEnvironmentTest {
CheckpointingOptions.CHECKPOINTS_DIRECTORY.key());
}
- @ParameterizedTest
- @MethodSource("provideExecutors")
- void testAllowCheckpointStorage(
- ThrowingConsumer<StreamExecutionEnvironment, Exception> executor) {
- final Configuration clusterConfig = new Configuration();
- clusterConfig.set(DeploymentOptions.PROGRAM_CONFIG_ENABLED, false);
- clusterConfig.set(DeploymentOptions.TARGET, "local");
- clusterConfig.set(CheckpointingOptions.CHECKPOINTS_DIRECTORY, "file:///flink/checkpoints");
-
- final StreamContextEnvironment environment =
- constructStreamContextEnvironment(
- clusterConfig,
- Arrays.asList(CheckpointingOptions.CHECKPOINTS_DIRECTORY.key()));
-
- String allowedPath = "file:///flink/allowed/modification";
- // Change the CheckpointConfig
- environment.getCheckpointConfig().setCheckpointStorage(allowedPath);
-
- environment.fromData(Collections.singleton(1)).sinkTo(new DiscardingSink<>());
- assertThatThrownBy(() -> executor.accept(environment))
- .isInstanceOf(ExecutorReachedException.class);
- }
-
@ParameterizedTest
@MethodSource("provideExecutors")
void testNotModifiedCheckpointStorage(
diff --git a/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/FileSinkCompactionSwitchITCase.java b/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/FileSinkCompactionSwitchITCase.java
index 0f06fcf4f3b..a17204fc9aa 100644
--- a/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/FileSinkCompactionSwitchITCase.java
+++ b/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/FileSinkCompactionSwitchITCase.java
@@ -46,12 +46,12 @@ import org.apache.flink.runtime.state.CheckpointListener;
import org.apache.flink.runtime.state.FunctionInitializationContext;
import org.apache.flink.runtime.state.FunctionSnapshotContext;
import org.apache.flink.runtime.state.hashmap.HashMapStateBackend;
-import org.apache.flink.runtime.state.storage.FileSystemCheckpointStorage;
import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration;
import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
import org.apache.flink.streaming.api.graph.StreamGraph;
+import org.apache.flink.streaming.util.CheckpointStorageUtils;
import org.apache.flink.streaming.util.RestartStrategyUtils;
import org.apache.flink.test.junit5.MiniClusterExtension;
import org.apache.flink.testutils.junit.SharedObjectsExtension;
@@ -223,7 +223,7 @@ public class FileSinkCompactionSwitchITCase {
env.enableCheckpointing(100, CheckpointingMode.EXACTLY_ONCE);
RestartStrategyUtils.configureNoRestartStrategy(env);
- env.getCheckpointConfig().setCheckpointStorage(new FileSystemCheckpointStorage(cpPath));
+ CheckpointStorageUtils.configureFileSystemCheckpointStorage(env, cpPath);
env.setStateBackend(new HashMapStateBackend());
env.addSource(new CountingTestSource(latchId, NUM_RECORDS, isFinite, sendCountMap))
diff --git a/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/DataStreamAllroundTestJobFactory.java b/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/DataStreamAllroundTestJobFactory.java
index 7a6164aea39..d1dffdd8c17 100644
--- a/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/DataStreamAllroundTestJobFactory.java
+++ b/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/DataStreamAllroundTestJobFactory.java
@@ -29,6 +29,7 @@ import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
import org.apache.flink.api.java.utils.ParameterTool;
+import org.apache.flink.configuration.CheckpointingOptions;
import org.apache.flink.configuration.ConfigOption;
import org.apache.flink.configuration.ConfigOptions;
import org.apache.flink.configuration.Configuration;
@@ -292,7 +293,9 @@ public class DataStreamAllroundTestJobFactory {
env.enableCheckpointing(checkpointInterval, checkpointingMode);
final String checkpointDir = pt.getRequired(STATE_BACKEND_CHECKPOINT_DIR.key());
- env.getCheckpointConfig().setCheckpointStorage(checkpointDir);
+ Configuration configuration = new Configuration();
+ configuration.set(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir);
+ env.configure(configuration);
boolean enableExternalizedCheckpoints =
pt.getBoolean(
diff --git a/flink-end-to-end-tests/flink-local-recovery-and-allocation-test/src/main/java/org/apache/flink/streaming/tests/StickyAllocationAndLocalRecoveryTestJob.java b/flink-end-to-end-tests/flink-local-recovery-and-allocation-test/src/main/java/org/apache/flink/streaming/tests/StickyAllocationAndLocalRecoveryTestJob.java
index 14c21f09122..378f8f6b6e7 100644
--- a/flink-end-to-end-tests/flink-local-recovery-and-allocation-test/src/main/java/org/apache/flink/streaming/tests/StickyAllocationAndLocalRecoveryTestJob.java
+++ b/flink-end-to-end-tests/flink-local-recovery-and-allocation-test/src/main/java/org/apache/flink/streaming/tests/StickyAllocationAndLocalRecoveryTestJob.java
@@ -27,6 +27,7 @@ import org.apache.flink.api.common.state.ValueState;
import org.apache.flink.api.common.state.ValueStateDescriptor;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.utils.ParameterTool;
+import org.apache.flink.configuration.CheckpointingOptions;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.ExternalizedCheckpointRetention;
import org.apache.flink.configuration.RestartStrategyOptions;
@@ -98,6 +99,8 @@ public class StickyAllocationAndLocalRecoveryTestJob {
configuration.set(
RestartStrategyOptions.RESTART_STRATEGY_FIXED_DELAY_DELAY,
Duration.ofMillis(pt.getInt("restartDelay", 0)));
+ String checkpointDir = pt.getRequired("checkpointDir");
+ configuration.set(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir);
env.configure(configuration);
if (pt.getBoolean("externalizedCheckpoints", false)) {
@@ -106,9 +109,6 @@ public class StickyAllocationAndLocalRecoveryTestJob {
ExternalizedCheckpointRetention.RETAIN_ON_CANCELLATION);
}
- String checkpointDir = pt.getRequired("checkpointDir");
- env.getCheckpointConfig().setCheckpointStorage(checkpointDir);
-
boolean killJvmOnFail = pt.getBoolean("killJvmOnFail", false);
String stateBackend = pt.get("stateBackend", "hashmap");
diff --git a/flink-python/docs/reference/pyflink.datastream/checkpoint.rst b/flink-python/docs/reference/pyflink.datastream/checkpoint.rst
index 10a2783fc7d..d3321b774c7 100644
--- a/flink-python/docs/reference/pyflink.datastream/checkpoint.rst
+++ b/flink-python/docs/reference/pyflink.datastream/checkpoint.rst
@@ -78,9 +78,6 @@ The default limit of concurrently happening checkpoints: one.
CheckpointConfig.get_alignment_timeout
CheckpointConfig.set_force_unaligned_checkpoints
CheckpointConfig.is_force_unaligned_checkpoints
- CheckpointConfig.set_checkpoint_storage
- CheckpointConfig.set_checkpoint_storage_dir
- CheckpointConfig.get_checkpoint_storage
ExternalizedCheckpointCleanup
ExternalizedCheckpointRetention
diff --git a/flink-python/pyflink/datastream/checkpoint_config.py b/flink-python/pyflink/datastream/checkpoint_config.py
index 5034e9d85bd..d5b2e7fa0f5 100644
--- a/flink-python/pyflink/datastream/checkpoint_config.py
+++ b/flink-python/pyflink/datastream/checkpoint_config.py
@@ -15,13 +15,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
################################################################################
-import warnings
from enum import Enum
from typing import Optional
from pyflink.common import Duration
from pyflink.datastream.externalized_checkpoint_retention import ExternalizedCheckpointRetention
-from pyflink.datastream.checkpoint_storage import CheckpointStorage, _from_j_checkpoint_storage
from pyflink.datastream.checkpointing_mode import CheckpointingMode
from pyflink.java_gateway import get_gateway
@@ -481,65 +479,6 @@ class CheckpointConfig(object):
"""
return self._j_checkpoint_config.isForceUnalignedCheckpoints()
- def set_checkpoint_storage(self, storage: CheckpointStorage) -> 'CheckpointConfig':
- """
- Checkpoint storage defines how stat backends checkpoint their state for fault
- tolerance in streaming applications. Various implementations store their checkpoints
- in different fashions and have different requirements and availability guarantees.
-
- For example, `JobManagerCheckpointStorage` stores checkpoints in the memory of the
- JobManager. It is lightweight and without additional dependencies but is not highly
- available and only supports small state sizes. This checkpoint storage policy is convenient
- for local testing and development.
-
- The `FileSystemCheckpointStorage` stores checkpoints in a filesystem. For systems like
- HDFS, NFS Drivs, S3, and GCS, this storage policy supports large state size, in the
- magnitude of many terabytes while providing a highly available foundation for stateful
- applications. This checkpoint storage policy is recommended for most production deployments.
-
- .. note:: Deprecated since version 1.19: This method is deprecated and will be removed in
- future FLINK major version. Use `stream_execution_environment.configure` method
- instead to set the checkpoint storage.
- """
- warnings.warn("Deprecated since version 1.19: This method is deprecated and will be removed"
- " in future FLINK major version. Use `stream_execution_environment.configure`"
- " method instead to set the checkpoint storage.", DeprecationWarning)
- self._j_checkpoint_config.setCheckpointStorage(storage._j_checkpoint_storage)
- return self
-
- def set_checkpoint_storage_dir(self, checkpoint_path: str) -> 'CheckpointConfig':
- """
- Configures the application to write out checkpoint snapshots to the configured directory.
- See `FileSystemCheckpointStorage` for more details on checkpointing to a file system.
-
- .. note:: Deprecated since version 1.19: This method is deprecated and will be removed in
- future FLINK major version. Use `stream_execution_environment.configure` method
- instead to set the checkpoint storage.
- """
- warnings.warn("Deprecated since version 1.19: This method is deprecated and will be removed"
- " in future FLINK major version. Use `stream_execution_environment.configure`"
- " method instead to set the checkpoint storage.", DeprecationWarning)
- self._j_checkpoint_config.setCheckpointStorage(checkpoint_path)
- return self
-
- def get_checkpoint_storage(self) -> Optional[CheckpointStorage]:
- """
- The checkpoint storage that has been configured for the Job, or None if
- none has been set.
-
- .. note:: Deprecated since version 1.19: This method is deprecated and will be removed in
- future FLINK major version. It is recommended to find which checkpoint storage is
- used by checkpoint storage ConfigOption.
- """
- warnings.warn("Deprecated since version 1.19: This method is deprecated and will be removed"
- " in future FLINK major version. It is recommended to find which checkpoint"
- " storage is used by checkpoint storage ConfigOption.", DeprecationWarning)
- j_storage = self._j_checkpoint_config.getCheckpointStorage()
- if j_storage is None:
- return None
- else:
- return _from_j_checkpoint_storage(j_storage)
-
class ExternalizedCheckpointCleanup(Enum):
"""
diff --git a/flink-python/pyflink/datastream/tests/test_check_point_config.py b/flink-python/pyflink/datastream/tests/test_check_point_config.py
index 115e89ccee6..9c798515164 100644
--- a/flink-python/pyflink/datastream/tests/test_check_point_config.py
+++ b/flink-python/pyflink/datastream/tests/test_check_point_config.py
@@ -182,14 +182,3 @@ class CheckpointConfigTests(PyFlinkTestCase):
self.checkpoint_config.set_alignment_timeout(Duration.of_minutes(1))
self.assertEqual(self.checkpoint_config.get_alignment_timeout(), Duration.of_minutes(1))
-
- def test_get_set_checkpoint_storage(self):
-
- self.assertIsNone(self.checkpoint_config.get_checkpoint_storage(),
- "Default checkpoint storage should be None")
-
- self.checkpoint_config.set_checkpoint_storage_dir("file://var/checkpoints/")
-
- self.assertEqual(self.checkpoint_config.get_checkpoint_storage().get_checkpoint_path(),
- "file://var/checkpoints",
- "Wrong checkpoints directory")
diff --git a/flink-runtime/src/main/java/org/apache/flink/streaming/api/environment/CheckpointConfig.java b/flink-runtime/src/main/java/org/apache/flink/streaming/api/environment/CheckpointConfig.java
index 8308dcaee2c..d3c0d1530f1 100644
--- a/flink-runtime/src/main/java/org/apache/flink/streaming/api/environment/CheckpointConfig.java
+++ b/flink-runtime/src/main/java/org/apache/flink/streaming/api/environment/CheckpointConfig.java
@@ -31,19 +31,12 @@ import org.apache.flink.configuration.ReadableConfig;
import org.apache.flink.configuration.StateRecoveryOptions;
import org.apache.flink.configuration.description.InlineElement;
import org.apache.flink.core.execution.CheckpointingMode;
-import org.apache.flink.core.fs.Path;
import org.apache.flink.runtime.jobgraph.tasks.CheckpointCoordinatorConfiguration;
-import org.apache.flink.runtime.state.CheckpointStorage;
-import org.apache.flink.runtime.state.StateBackend;
-import org.apache.flink.runtime.state.storage.FileSystemCheckpointStorage;
import org.apache.flink.util.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.annotation.Nullable;
-
-import java.net.URI;
import java.time.Duration;
import static org.apache.flink.configuration.description.TextElement.text;
@@ -129,14 +122,6 @@ public class CheckpointConfig implements java.io.Serializable {
*/
private final Configuration configuration;
- /**
- * The checkpoint storage for this application. This field is marked as transient because it may
- * contain user-code.
- *
- * @deprecated this should be moved somehow to {@link #configuration}.
- */
- @Deprecated private transient CheckpointStorage storage;
-
/**
* Creates a deep copy of the provided {@link CheckpointConfig}.
*
@@ -146,7 +131,6 @@ public class CheckpointConfig implements java.io.Serializable {
checkNotNull(checkpointConfig);
this.configuration = new Configuration(checkpointConfig.configuration);
- this.storage = checkpointConfig.getCheckpointStorage();
}
public CheckpointConfig() {
@@ -801,154 +785,6 @@ public class CheckpointConfig implements java.io.Serializable {
return configuration.get(CheckpointingOptions.EXTERNALIZED_CHECKPOINT_RETENTION);
}
- /**
- * CheckpointStorage defines how {@link StateBackend}'s checkpoint their state for fault
- * tolerance in streaming applications. Various implementations store their checkpoints in
- * different fashions and have different requirements and availability guarantees.
- *
- * <p>For example, {@link org.apache.flink.runtime.state.storage.JobManagerCheckpointStorage
- * JobManagerCheckpointStorage} stores checkpoints in the memory of the JobManager. It is
- * lightweight and without additional dependencies but is not highly available and only supports
- * small state sizes. This checkpoint storage policy is convenient for local testing and
- * development.
- *
- * <p>{@link org.apache.flink.runtime.state.storage.FileSystemCheckpointStorage
- * FileSystemCheckpointStorage} stores checkpoints in a filesystem. For systems like HDFS, NFS
- * Drives, S3, and GCS, this storage policy supports large state size, in the magnitude of many
- * terabytes while providing a highly available foundation for stateful applications. This
- * checkpoint storage policy is recommended for most production deployments.
- *
- * @deprecated The method is marked as deprecated because starting from Flink 1.19, the usage of
- * all complex Java objects related to configuration, including their getter and setter
- * methods, should be replaced by ConfigOption. In a future major version of Flink, this
- * method will be removed entirely. It is recommended to switch to using the ConfigOptions
- * provided for configuring checkpoint storage like the following code snippet:
- * <pre>{@code
- * Configuration config = new Configuration();
- * config.set(CheckpointingOptions.CHECKPOINT_STORAGE, "filesystem");
- * config.set(CheckpointingOptions.CHECKPOINTS_DIRECTORY, "file:///flink/checkpoints");
- * StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(config);
- * }</pre>
- * For more details on using ConfigOption for checkpoint storage configuration, please refer
- * to the Flink documentation: <a
- * href="https://nightlies.apache.org/flink/flink-docs-stable/docs/ops/state/checkpoints">Checkpoints</a>
- * @param storage The checkpoint storage policy.
- */
- @Deprecated
- @PublicEvolving
- public void setCheckpointStorage(CheckpointStorage storage) {
- Preconditions.checkNotNull(storage, "Checkpoint storage must not be null");
- this.storage = storage;
- }
-
- /**
- * Configures the application to write out checkpoint snapshots to the configured directory. See
- * {@link FileSystemCheckpointStorage} for more details on checkpointing to a file system.
- *
- * @deprecated The method is marked as deprecated because starting from Flink 1.19, the usage of
- * all complex Java objects related to configuration, including their getter and setter
- * methods, should be replaced by ConfigOption. In a future major version of Flink, this
- * method will be removed entirely. It is recommended to switch to using the ConfigOptions
- * provided for configuring checkpoint storage like the following code snippet:
- * <pre>{@code
- * Configuration config = new Configuration();
- * config.set(CheckpointingOptions.CHECKPOINT_STORAGE, "filesystem");
- * config.set(CheckpointingOptions.CHECKPOINTS_DIRECTORY, "file:///flink/checkpoints");
- * StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(config);
- * }</pre>
- * For more details on using ConfigOption for checkpoint storage configuration, please refer
- * to the Flink documentation: <a
- * href="https://nightlies.apache.org/flink/flink-docs-stable/docs/ops/state/checkpoints">Checkpoints</a>
- * @param checkpointDirectory The path to write checkpoint metadata to.
- * @see #setCheckpointStorage(CheckpointStorage)
- */
- @Deprecated
- @PublicEvolving
- public void setCheckpointStorage(String checkpointDirectory) {
- Preconditions.checkNotNull(checkpointDirectory, "Checkpoint directory must not be null");
- this.configuration.set(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDirectory);
- this.storage = new FileSystemCheckpointStorage(checkpointDirectory);
- }
-
- /**
- * Configures the application to write out checkpoint snapshots to the configured directory. See
- * {@link FileSystemCheckpointStorage} for more details on checkpointing to a file system.
- *
- * @deprecated The method is marked as deprecated because starting from Flink 1.19, the usage of
- * all complex Java objects related to configuration, including their getter and setter
- * methods, should be replaced by ConfigOption. In a future major version of Flink, this
- * method will be removed entirely. It is recommended to switch to using the ConfigOptions
- * provided for configuring checkpoint storage like the following code snippet:
- * <pre>{@code
- * Configuration config = new Configuration();
- * config.set(CheckpointingOptions.CHECKPOINT_STORAGE, "filesystem");
- * config.set(CheckpointingOptions.CHECKPOINTS_DIRECTORY, "file:///flink/checkpoints");
- * StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(config);
- * }</pre>
- * For more details on using ConfigOption for checkpoint storage configuration, please refer
- * to the Flink documentation: <a
- * href="https://nightlies.apache.org/flink/flink-docs-stable/docs/ops/state/checkpoints">Checkpoints</a>
- * @param checkpointDirectory The path to write checkpoint metadata to.
- * @see #setCheckpointStorage(CheckpointStorage)
- */
- @Deprecated
- @PublicEvolving
- public void setCheckpointStorage(URI checkpointDirectory) {
- Preconditions.checkNotNull(checkpointDirectory, "Checkpoint directory must not be null");
- this.configuration.set(
- CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDirectory.toString());
- this.storage = new FileSystemCheckpointStorage(checkpointDirectory);
- }
-
- /**
- * Configures the application to write out checkpoint snapshots to the configured directory. See
- * {@link FileSystemCheckpointStorage} for more details on checkpointing to a file system.
- *
- * @deprecated The method is marked as deprecated because starting from Flink 1.19, the usage of
- * all complex Java objects related to configuration, including their getter and setter
- * methods, should be replaced by ConfigOption. In a future major version of Flink, this
- * method will be removed entirely. It is recommended to switch to using the ConfigOptions
- * provided for configuring checkpoint storage like the following code snippet:
- * <pre>{@code
- * Configuration config = new Configuration();
- * config.set(CheckpointingOptions.CHECKPOINT_STORAGE, "filesystem");
- * config.set(CheckpointingOptions.CHECKPOINTS_DIRECTORY, "file:///flink/checkpoints");
- * StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(config);
- * }</pre>
- * For more details on using ConfigOption for checkpoint storage configuration, please refer
- * to the Flink documentation: <a
- * href="https://nightlies.apache.org/flink/flink-docs-stable/docs/ops/state/checkpoints">Checkpoints</a>
- * @param checkpointDirectory The path to write checkpoint metadata to.
- * @see #setCheckpointStorage(String)
- */
- @Deprecated
- @PublicEvolving
- public void setCheckpointStorage(Path checkpointDirectory) {
- Preconditions.checkNotNull(checkpointDirectory, "Checkpoint directory must not be null");
- this.configuration.set(
- CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDirectory.toString());
- this.storage = new FileSystemCheckpointStorage(checkpointDirectory);
- }
-
- /**
- * @deprecated The method is marked as deprecated because starting from Flink 1.19, the usage of
- * all complex Java objects related to configuration, including their getter and setter
- * methods, should be replaced by ConfigOption. In a future major version of Flink, this
- * method will be removed entirely. It is recommended to find which checkpoint storage is
- * used by checkpoint storage ConfigOption. For more details on using ConfigOption for
- * checkpoint storage configuration, please refer to the Flink documentation: <a
- * href="https://nightlies.apache.org/flink/flink-docs-stable/docs/ops/state/checkpoints">Checkpoints</a>
- * @return The {@link CheckpointStorage} that has been configured for the job. Or {@code null}
- * if none has been set.
- * @see #setCheckpointStorage(CheckpointStorage)
- */
- @Deprecated
- @Nullable
- @PublicEvolving
- public CheckpointStorage getCheckpointStorage() {
- return this.storage;
- }
-
/**
* Setup the checkpoint id for which the in-flight data will be ignored for all operators in
* case of the recovery from this checkpoint.
@@ -1092,16 +928,17 @@ public class CheckpointConfig implements java.io.Serializable {
.ifPresent(this::setForceUnalignedCheckpoints);
configuration
.getOptional(CheckpointingOptions.CHECKPOINTS_DIRECTORY)
- .ifPresent(this::setCheckpointStorage);
- // reset checkpoint storage for backward compatibility
- configuration
- .getOptional(CheckpointingOptions.CHECKPOINT_STORAGE)
- .ifPresent(ignored -> this.storage = null);
+ .ifPresent(this::setCheckpointDirectory);
+ }
+
+ private void setCheckpointDirectory(String checkpointDirectory) {
+ Preconditions.checkNotNull(checkpointDirectory, "Checkpoint directory must not be null");
+ this.configuration.set(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDirectory);
}
/**
* @return A copy of internal {@link #configuration}. Note it is missing all options that are
- * stored as plain java fields in {@link CheckpointConfig}, for example {@link #storage}.
+ * stored as plain java fields in {@link CheckpointConfig}.
*/
@Internal
public Configuration toConfiguration() {
diff --git a/flink-runtime/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java b/flink-runtime/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java
index 0639197d507..d71cd6451ed 100644
--- a/flink-runtime/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java
@@ -39,7 +39,6 @@ import org.apache.flink.configuration.StateChangelogOptions;
import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
import org.apache.flink.runtime.jobgraph.JobType;
import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings;
-import org.apache.flink.runtime.state.CheckpointStorage;
import org.apache.flink.runtime.state.KeyGroupRangeAssignment;
import org.apache.flink.runtime.state.StateBackend;
import org.apache.flink.streaming.api.TimeCharacteristic;
@@ -161,8 +160,6 @@ public class StreamGraphGenerator {
private StateBackend stateBackend;
- private CheckpointStorage checkpointStorage;
-
private TimeCharacteristic timeCharacteristic = DEFAULT_TIME_CHARACTERISTIC;
private SavepointRestoreSettings savepointRestoreSettings;
@@ -232,7 +229,6 @@ public class StreamGraphGenerator {
this.executionConfig = checkNotNull(executionConfig);
this.checkpointConfig = new CheckpointConfig(checkpointConfig);
this.configuration = checkNotNull(configuration);
- this.checkpointStorage = this.checkpointConfig.getCheckpointStorage();
this.savepointRestoreSettings = SavepointRestoreSettings.fromConfiguration(configuration);
}
@@ -362,7 +358,6 @@ public class StreamGraphGenerator {
graph.setJobName(deriveJobName(DEFAULT_STREAMING_JOB_NAME));
graph.setStateBackend(stateBackend);
- graph.setCheckpointStorage(checkpointStorage);
graph.setGlobalStreamExchangeMode(deriveGlobalStreamExchangeModeStreaming());
}
diff --git a/flink-runtime/src/test/java/org/apache/flink/streaming/api/environment/CheckpointConfigFromConfigurationTest.java b/flink-runtime/src/test/java/org/apache/flink/streaming/api/environment/CheckpointConfigFromConfigurationTest.java
index d97ca4b9c30..fd73158fc68 100644
--- a/flink-runtime/src/test/java/org/apache/flink/streaming/api/environment/CheckpointConfigFromConfigurationTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/streaming/api/environment/CheckpointConfigFromConfigurationTest.java
@@ -18,15 +18,11 @@
package org.apache.flink.streaming.api.environment;
-import org.apache.flink.configuration.CheckpointingOptions;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.ExternalizedCheckpointRetention;
import org.apache.flink.configuration.ReadableConfig;
import org.apache.flink.core.execution.CheckpointingMode;
-import org.apache.flink.runtime.state.CheckpointStorage;
-import org.apache.flink.runtime.state.storage.FileSystemCheckpointStorage;
-import org.assertj.core.api.InstanceOfAssertFactories;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
@@ -158,32 +154,7 @@ public class CheckpointConfigFromConfigurationTest {
.viaSetter(CheckpointConfig::enableUnalignedCheckpointsInterruptibleTimers)
.getterVia(
CheckpointConfig::isUnalignedCheckpointsInterruptibleTimersEnabled)
- .nonDefaultValue(true),
- TestSpec.testValue(
- (CheckpointStorage)
- new FileSystemCheckpointStorage(
- "file:///path/to/checkpoint/dir"))
- .whenSetFromFile(
- CheckpointingOptions.CHECKPOINTS_DIRECTORY.key(),
- "file:///path/to/checkpoint/dir")
- .viaSetter(CheckpointConfig::setCheckpointStorage)
- .getterVia(CheckpointConfig::getCheckpointStorage)
- .nonDefaultValue(
- new FileSystemCheckpointStorage("file:///path/to/checkpoint/dir"))
- .customMatcher(
- (actualValue, expectedValue) ->
- assertThat(actualValue)
- .hasSameClassAs(expectedValue)
- .asInstanceOf(
- InstanceOfAssertFactories.type(
- FileSystemCheckpointStorage.class))
- .extracting(
- FileSystemCheckpointStorage
- ::getCheckpointPath)
- .isEqualTo(
- ((FileSystemCheckpointStorage)
- expectedValue)
- .getCheckpointPath())));
+ .nonDefaultValue(true));
}
@ParameterizedTest
diff --git a/flink-runtime/src/test/java/org/apache/flink/streaming/util/CheckpointStorageUtils.java b/flink-runtime/src/test/java/org/apache/flink/streaming/util/CheckpointStorageUtils.java
new file mode 100644
index 00000000000..4b9930c0584
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/streaming/util/CheckpointStorageUtils.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.util;
+
+import org.apache.flink.configuration.CheckpointingOptions;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.util.Preconditions;
+
+import java.net.URI;
+import java.nio.file.Path;
+
+/** A utility class for configuring checkpoint storage. */
+public class CheckpointStorageUtils {
+
+ /**
+ * Configures the checkpoint storage to use the JobManager for storing checkpoints.
+ *
+ * @param env The StreamExecutionEnvironment to configure.
+ */
+ public static void configureJobManagerCheckpointStorage(StreamExecutionEnvironment env) {
+ env.configure(
+ new Configuration().set(CheckpointingOptions.CHECKPOINT_STORAGE, "jobmanager"));
+ }
+
+ /**
+ * Configures the checkpoint storage with a given directory as a string.
+ *
+ * @param env The StreamExecutionEnvironment to configure.
+ * @param checkpointDirectory The directory where checkpoints will be stored, must not be null.
+ * @throws NullPointerException if checkpointDirectory is null.
+ */
+ public static void configureFileSystemCheckpointStorage(
+ StreamExecutionEnvironment env, String checkpointDirectory) {
+ Preconditions.checkNotNull(checkpointDirectory, "Checkpoint directory must not be null");
+ env.configure(
+ new Configuration()
+ .set(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDirectory));
+ }
+
+ /**
+ * Configures the checkpoint storage with a given directory as a URI.
+ *
+ * @param env The StreamExecutionEnvironment to configure.
+ * @param checkpointDirectory The URI of the directory where checkpoints will be stored, must
+ * not be null.
+ * @throws NullPointerException if checkpointDirectory is null.
+ */
+ public static void configureFileSystemCheckpointStorage(
+ StreamExecutionEnvironment env, URI checkpointDirectory) {
+ Preconditions.checkNotNull(checkpointDirectory, "Checkpoint directory must not be null");
+ configureFileSystemCheckpointStorage(env, checkpointDirectory.toString());
+ }
+
+ /**
+ * Sets the checkpoint storage with a given directory as a Path.
+ *
+ * @param env The StreamExecutionEnvironment to configure.
+ * @param checkpointDirectory The Path of the directory where checkpoints will be stored, must
+ * not be null.
+ * @throws NullPointerException if checkpointDirectory is null.
+ */
+ public static void configureFileSystemCheckpointStorage(
+ StreamExecutionEnvironment env, Path checkpointDirectory) {
+ Preconditions.checkNotNull(checkpointDirectory, "Checkpoint directory must not be null");
+ configureFileSystemCheckpointStorage(env, checkpointDirectory.toString());
+ }
+
+ /**
+ * Configures the checkpoint storage using a specified storage factory.
+ *
+ * @param env The StreamExecutionEnvironment to configure.
+ * @param checkpointStorageFactory The fully qualified class name of the checkpoint storage
+ * factory, must not be null or empty.
+ * @throws IllegalArgumentException if checkpointStorageFactory is null or empty.
+ */
+ public static void configureCheckpointStorageWithFactory(
+ StreamExecutionEnvironment env, String checkpointStorageFactory) {
+ Preconditions.checkNotNull(
+ checkpointStorageFactory != null, "Checkpoint storage factory must not be null.");
+
+ env.configure(
+ new Configuration()
+ .set(CheckpointingOptions.CHECKPOINT_STORAGE, checkpointStorageFactory));
+ }
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/BoundedSourceITCase.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/BoundedSourceITCase.java
index 1e462be38eb..893de27828e 100644
--- a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/BoundedSourceITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/BoundedSourceITCase.java
@@ -24,6 +24,7 @@ import org.apache.flink.runtime.operators.lifecycle.graph.TestJobBuilders.Testin
import org.apache.flink.runtime.operators.lifecycle.validation.DrainingValidator;
import org.apache.flink.runtime.operators.lifecycle.validation.FinishingValidator;
import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration;
+import org.apache.flink.streaming.util.CheckpointStorageUtils;
import org.apache.flink.test.util.MiniClusterWithClientResource;
import org.apache.flink.testutils.junit.SharedObjects;
import org.apache.flink.util.TestLogger;
@@ -101,9 +102,8 @@ public class BoundedSourceITCase extends TestLogger {
sharedObjects,
cfg -> {},
env ->
- env.getCheckpointConfig()
- .setCheckpointStorage(
- TEMPORARY_FOLDER.newFolder().toURI()));
+ CheckpointStorageUtils.configureFileSystemCheckpointStorage(
+ env, TEMPORARY_FOLDER.newFolder().toURI()));
TestJobExecutor.execute(testJob, miniClusterResource)
.waitForEvent(CheckpointCompletedEvent.class)
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/PartiallyFinishedSourcesITCase.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/PartiallyFinishedSourcesITCase.java
index 5d8edf4925f..87fbb5fc6a8 100644
--- a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/PartiallyFinishedSourcesITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/PartiallyFinishedSourcesITCase.java
@@ -27,6 +27,7 @@ import org.apache.flink.runtime.operators.lifecycle.graph.TestJobBuilders.Testin
import org.apache.flink.runtime.operators.lifecycle.validation.DrainingValidator;
import org.apache.flink.runtime.operators.lifecycle.validation.FinishingValidator;
import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration;
+import org.apache.flink.streaming.util.CheckpointStorageUtils;
import org.apache.flink.streaming.util.RestartStrategyUtils;
import org.apache.flink.test.util.MiniClusterWithClientResource;
import org.apache.flink.testutils.junit.SharedObjects;
@@ -175,10 +176,10 @@ public class PartiallyFinishedSourcesITCase extends TestLogger {
.setTolerableCheckpointFailureNumber(Integer.MAX_VALUE);
// explicitly set to one to ease avoiding race conditions
env.getCheckpointConfig().setMaxConcurrentCheckpoints(1);
- env.getCheckpointConfig()
- // with unaligned checkpoints state size can grow beyond the default
- // limits of in-memory storage
- .setCheckpointStorage(TEMPORARY_FOLDER.newFolder().toURI());
+ // with unaligned checkpoints state size can grow beyond the default
+ // limits of in-memory storage
+ CheckpointStorageUtils.configureFileSystemCheckpointStorage(
+ env, TEMPORARY_FOLDER.newFolder().toURI());
});
}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/StopWithSavepointITCase.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/StopWithSavepointITCase.java
index 9729dded806..0c31229ff90 100644
--- a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/StopWithSavepointITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/StopWithSavepointITCase.java
@@ -30,6 +30,7 @@ import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
import org.apache.flink.streaming.api.operators.BoundedMultiInput;
import org.apache.flink.streaming.api.operators.StreamOperator;
import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.streaming.util.CheckpointStorageUtils;
import org.apache.flink.test.util.AbstractTestBaseJUnit4;
import org.apache.flink.testutils.junit.SharedObjects;
@@ -115,9 +116,8 @@ public class StopWithSavepointITCase extends AbstractTestBaseJUnit4 {
sharedObjects,
cfg -> {},
env ->
- env.getCheckpointConfig()
- .setCheckpointStorage(
- TEMPORARY_FOLDER.newFolder().toURI()));
+ CheckpointStorageUtils.configureFileSystemCheckpointStorage(
+ env, TEMPORARY_FOLDER.newFolder().toURI()));
TestJobExecutor.execute(testJob, MINI_CLUSTER_RESOURCE)
.waitForEvent(WatermarkReceivedEvent.class)
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ChangelogLocalRecoveryITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ChangelogLocalRecoveryITCase.java
index 58ba035272b..30cd52687b6 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ChangelogLocalRecoveryITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ChangelogLocalRecoveryITCase.java
@@ -30,6 +30,7 @@ import org.apache.flink.runtime.state.StateBackend;
import org.apache.flink.runtime.state.hashmap.HashMapStateBackend;
import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.util.CheckpointStorageUtils;
import org.apache.flink.streaming.util.RestartStrategyUtils;
import org.apache.flink.test.checkpointing.ChangelogRecoveryITCaseBase.CollectionSink;
import org.apache.flink.test.checkpointing.ChangelogRecoveryITCaseBase.CountFunction;
@@ -167,7 +168,7 @@ public class ChangelogLocalRecoveryITCase extends TestLogger {
RestartStrategyUtils.configureFixedDelayRestartStrategy(env, 3, 10L);
env.configure(new Configuration().set(LOCAL_RECOVERY, true));
- env.getCheckpointConfig().setCheckpointStorage(checkpointFile.toURI());
+ CheckpointStorageUtils.configureFileSystemCheckpointStorage(env, checkpointFile.toURI());
env.enableChangelogStateBackend(changelogEnabled);
env.configure(
new Configuration()
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ChangelogRecoveryITCaseBase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ChangelogRecoveryITCaseBase.java
index 020b6448f59..5e7622753d3 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ChangelogRecoveryITCaseBase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ChangelogRecoveryITCaseBase.java
@@ -57,6 +57,7 @@ import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction;
import org.apache.flink.streaming.api.windowing.assigners.TumblingProcessingTimeWindows;
import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
+import org.apache.flink.streaming.util.CheckpointStorageUtils;
import org.apache.flink.streaming.util.RestartStrategyUtils;
import org.apache.flink.test.util.MiniClusterWithClientResource;
import org.apache.flink.testutils.junit.SharedObjects;
@@ -195,7 +196,7 @@ public abstract class ChangelogRecoveryITCaseBase extends TestLogger {
restartAttempts,
materializationInterval,
materializationMaxFailure);
- env.getCheckpointConfig().setCheckpointStorage(checkpointFile.toURI());
+ CheckpointStorageUtils.configureFileSystemCheckpointStorage(env, checkpointFile.toURI());
return env;
}
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ChangelogRecoverySwitchEnvTestBase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ChangelogRecoverySwitchEnvTestBase.java
index 63b72b4f7c0..7a0d4a6cc02 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ChangelogRecoverySwitchEnvTestBase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ChangelogRecoverySwitchEnvTestBase.java
@@ -24,6 +24,7 @@ import org.apache.flink.runtime.minicluster.MiniCluster;
import org.apache.flink.runtime.state.AbstractStateBackend;
import org.apache.flink.runtime.state.StateHandleID;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.util.CheckpointStorageUtils;
import org.apache.flink.testutils.junit.SharedReference;
import org.apache.flink.util.ExceptionUtils;
import org.apache.flink.util.Preconditions;
@@ -48,7 +49,9 @@ public abstract class ChangelogRecoverySwitchEnvTestBase extends ChangelogRecove
SharedReference<MiniCluster> miniCluster = sharedObjects.add(cluster.getMiniCluster());
SharedReference<Set<StateHandleID>> currentMaterializationId =
sharedObjects.add(ConcurrentHashMap.newKeySet());
- firstEnv.getCheckpointConfig().setCheckpointStorage(firstCheckpointFolder.toURI());
+
+ CheckpointStorageUtils.configureFileSystemCheckpointStorage(
+ firstEnv, firstCheckpointFolder.toURI());
JobGraph firstJobGraph =
firstNormalJobGraph(firstEnv, miniCluster, currentMaterializationId);
@@ -61,7 +64,8 @@ public abstract class ChangelogRecoverySwitchEnvTestBase extends ChangelogRecove
}
File secondCheckpointFolder = TEMPORARY_FOLDER.newFolder();
- secondEnv.getCheckpointConfig().setCheckpointStorage(secondCheckpointFolder.toURI());
+ CheckpointStorageUtils.configureFileSystemCheckpointStorage(
+ secondEnv, secondCheckpointFolder.toURI());
JobGraph jobGraph = nextNormalJobGraph(secondEnv, miniCluster, currentMaterializationId);
File checkpointFile = getMostRecentCompletedCheckpoint(firstCheckpointFolder);
jobGraph.setSavepointRestoreSettings(
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/CheckpointFailureManagerITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/CheckpointFailureManagerITCase.java
index ba7242686c0..7ef2b5ab95e 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/CheckpointFailureManagerITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/CheckpointFailureManagerITCase.java
@@ -21,6 +21,7 @@ package org.apache.flink.test.checkpointing;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.state.ListState;
import org.apache.flink.api.common.state.ListStateDescriptor;
+import org.apache.flink.configuration.IllegalConfigurationException;
import org.apache.flink.configuration.ReadableConfig;
import org.apache.flink.core.fs.CloseableRegistry;
import org.apache.flink.runtime.checkpoint.CheckpointFailureManager;
@@ -30,6 +31,7 @@ import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.state.CheckpointMetadataOutputStream;
import org.apache.flink.runtime.state.CheckpointStorage;
import org.apache.flink.runtime.state.CheckpointStorageAccess;
+import org.apache.flink.runtime.state.CheckpointStorageFactory;
import org.apache.flink.runtime.state.CheckpointStorageLocation;
import org.apache.flink.runtime.state.CheckpointStreamFactory;
import org.apache.flink.runtime.state.CompletedCheckpointStorageLocation;
@@ -53,6 +55,7 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.sink.v2.DiscardingSink;
import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
import org.apache.flink.streaming.api.graph.StreamingJobGraphGenerator;
+import org.apache.flink.streaming.util.CheckpointStorageUtils;
import org.apache.flink.streaming.util.RestartStrategyUtils;
import org.apache.flink.test.util.MiniClusterWithClientResource;
import org.apache.flink.test.util.TestUtils;
@@ -87,7 +90,9 @@ public class CheckpointFailureManagerITCase extends TestLogger {
public void testFinalizationFailureCounted() throws Exception {
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.enableCheckpointing(10);
- env.getCheckpointConfig().setCheckpointStorage(new FailingFinalizationCheckpointStorage());
+ CheckpointStorageUtils.configureCheckpointStorageWithFactory(
+ env,
+ "org.apache.flink.test.checkpointing.CheckpointFailureManagerITCase$FailingFinalizationCheckpointStorageFactory");
env.getCheckpointConfig().setTolerableCheckpointFailureNumber(0);
RestartStrategyUtils.configureNoRestartStrategy(env);
env.fromSequence(Long.MIN_VALUE, Long.MAX_VALUE).sinkTo(new DiscardingSink<>());
@@ -232,6 +237,16 @@ public class CheckpointFailureManagerITCase extends TestLogger {
}
}
+ public static class FailingFinalizationCheckpointStorageFactory
+ implements CheckpointStorageFactory<FailingFinalizationCheckpointStorage> {
+ @Override
+ public FailingFinalizationCheckpointStorage createFromConfig(
+ ReadableConfig config, ClassLoader classLoader)
+ throws IllegalConfigurationException {
+ return new FailingFinalizationCheckpointStorage();
+ }
+ }
+
private static class FailingFinalizationCheckpointStorage implements CheckpointStorage {
private static final long serialVersionUID = 8134582566514272546L;
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ManualCheckpointITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ManualCheckpointITCase.java
index 6b8d3ae34a6..185e85262c0 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ManualCheckpointITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ManualCheckpointITCase.java
@@ -29,12 +29,10 @@ import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
import org.apache.flink.api.connector.source.mocks.MockSource;
import org.apache.flink.core.execution.JobClient;
import org.apache.flink.runtime.minicluster.MiniCluster;
-import org.apache.flink.runtime.state.CheckpointStorage;
-import org.apache.flink.runtime.state.storage.FileSystemCheckpointStorage;
-import org.apache.flink.runtime.state.storage.JobManagerCheckpointStorage;
import org.apache.flink.runtime.testutils.CommonTestUtils;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.sink.v2.DiscardingSink;
+import org.apache.flink.streaming.util.CheckpointStorageUtils;
import org.apache.flink.test.util.AbstractTestBaseJUnit4;
import org.apache.flink.util.Collector;
@@ -48,23 +46,24 @@ import java.io.IOException;
import java.util.Collections;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
-import java.util.function.Function;
+import java.util.function.BiConsumer;
import java.util.function.Predicate;
/** Tests for manually triggering checkpoints. */
@RunWith(Parameterized.class)
public class ManualCheckpointITCase extends AbstractTestBaseJUnit4 {
- @Parameterized.Parameter public StorageSupplier storageSupplier;
+ @Parameterized.Parameter public StorageConfigurer storageConfigurer;
@Rule public TemporaryFolder temporaryFolder = new TemporaryFolder();
- private interface StorageSupplier extends Function<String, CheckpointStorage> {}
+ private interface StorageConfigurer extends BiConsumer<String, StreamExecutionEnvironment> {}
@Parameterized.Parameters
- public static StorageSupplier[] parameters() throws IOException {
- return new StorageSupplier[] {
- path -> new JobManagerCheckpointStorage(), FileSystemCheckpointStorage::new
+ public static StorageConfigurer[] parameters() throws IOException {
+ return new StorageConfigurer[] {
+ (s, env) -> CheckpointStorageUtils.configureJobManagerCheckpointStorage(env),
+ (s, env) -> CheckpointStorageUtils.configureFileSystemCheckpointStorage(env, s)
};
}
@@ -73,9 +72,7 @@ public class ManualCheckpointITCase extends AbstractTestBaseJUnit4 {
int parallelism = MINI_CLUSTER_RESOURCE.getNumberSlots();
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(parallelism);
- env.getCheckpointConfig()
- .setCheckpointStorage(
- storageSupplier.apply(temporaryFolder.newFolder().toURI().toString()));
+ storageConfigurer.accept(temporaryFolder.newFolder().toURI().toString(), env);
env.fromSource(
MockSource.continuous(parallelism).build(),
@@ -105,9 +102,7 @@ public class ManualCheckpointITCase extends AbstractTestBaseJUnit4 {
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(parallelism);
env.enableCheckpointing(checkpointingInterval);
- env.getCheckpointConfig()
- .setCheckpointStorage(
- storageSupplier.apply(temporaryFolder.newFolder().toURI().toString()));
+ storageConfigurer.accept(temporaryFolder.newFolder().toURI().toString(), env);
env.fromSource(
MockSource.continuous(parallelism).build(),
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/RescaleCheckpointManuallyITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/RescaleCheckpointManuallyITCase.java
index 2d79ff14264..8ec7f842df8 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/RescaleCheckpointManuallyITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/RescaleCheckpointManuallyITCase.java
@@ -39,6 +39,7 @@ import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.sink.SinkFunction;
import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
+import org.apache.flink.streaming.util.CheckpointStorageUtils;
import org.apache.flink.streaming.util.RestartStrategyUtils;
import org.apache.flink.test.util.MiniClusterWithClientResource;
import org.apache.flink.testutils.junit.SharedObjects;
@@ -238,7 +239,8 @@ public class RescaleCheckpointManuallyITCase extends TestLogger {
env.getConfig().setMaxParallelism(maxParallelism);
}
env.enableCheckpointing(checkpointingInterval);
- env.getCheckpointConfig().setCheckpointStorage(temporaryFolder.newFolder().toURI());
+ CheckpointStorageUtils.configureFileSystemCheckpointStorage(
+ env, temporaryFolder.newFolder().toURI());
env.getCheckpointConfig()
.setExternalizedCheckpointRetention(
ExternalizedCheckpointRetention.RETAIN_ON_CANCELLATION);
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/RestoreUpgradedJobITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/RestoreUpgradedJobITCase.java
index 881dd0a16ea..cd7ddda88ec 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/RestoreUpgradedJobITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/RestoreUpgradedJobITCase.java
@@ -36,6 +36,7 @@ import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.sink.SinkFunction;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
+import org.apache.flink.streaming.util.CheckpointStorageUtils;
import org.apache.flink.test.util.MiniClusterWithClientResource;
import org.apache.flink.testutils.junit.SharedObjects;
import org.apache.flink.testutils.junit.SharedReference;
@@ -176,8 +177,8 @@ public class RestoreUpgradedJobITCase extends TestLogger {
.setExternalizedCheckpointRetention(
ExternalizedCheckpointRetention.RETAIN_ON_CANCELLATION);
env.getCheckpointConfig().enableUnalignedCheckpoints(false);
- env.getCheckpointConfig()
- .setCheckpointStorage("file://" + temporaryFolder.getRoot().getAbsolutePath());
+ CheckpointStorageUtils.configureFileSystemCheckpointStorage(
+ env, "file://" + temporaryFolder.getRoot().getAbsolutePath());
env.setParallelism(PARALLELISM);
// Checkpointing is enabled with a large interval, and no checkpoints will be triggered.
env.enableCheckpointing(Integer.MAX_VALUE);
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/SavepointITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/SavepointITCase.java
index 4caf8cb5e6f..9c016d3c44a 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/SavepointITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/SavepointITCase.java
@@ -87,6 +87,7 @@ import org.apache.flink.streaming.api.operators.BoundedOneInput;
import org.apache.flink.streaming.api.operators.ChainingStrategy;
import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.streaming.util.CheckpointStorageUtils;
import org.apache.flink.streaming.util.RestartStrategyUtils;
import org.apache.flink.test.util.MiniClusterWithClientResource;
import org.apache.flink.testutils.EntropyInjectingTestFileSystem;
@@ -470,7 +471,8 @@ public class SavepointITCase extends TestLogger {
env.getCheckpointConfig()
.setExternalizedCheckpointRetention(
ExternalizedCheckpointRetention.RETAIN_ON_CANCELLATION);
- env.getCheckpointConfig().setCheckpointStorage(folder.newFolder().toURI());
+ CheckpointStorageUtils.configureFileSystemCheckpointStorage(
+ env, folder.newFolder().toURI());
env.setParallelism(parallelism);
final SharedReference<CountDownLatch> counter =
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/UnalignedCheckpointFailureHandlingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/UnalignedCheckpointFailureHandlingITCase.java
index 53ae1313da5..fa637dbbfaa 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/UnalignedCheckpointFailureHandlingITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/UnalignedCheckpointFailureHandlingITCase.java
@@ -20,6 +20,8 @@ package org.apache.flink.test.checkpointing;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.connector.source.lib.NumberSequenceSource;
+import org.apache.flink.configuration.IllegalConfigurationException;
+import org.apache.flink.configuration.ReadableConfig;
import org.apache.flink.core.execution.CheckpointingMode;
import org.apache.flink.core.execution.JobClient;
import org.apache.flink.core.fs.FileSystem;
@@ -28,6 +30,7 @@ import org.apache.flink.runtime.state.CheckpointStateOutputStream;
import org.apache.flink.runtime.state.CheckpointStateToolset;
import org.apache.flink.runtime.state.CheckpointStorage;
import org.apache.flink.runtime.state.CheckpointStorageAccess;
+import org.apache.flink.runtime.state.CheckpointStorageFactory;
import org.apache.flink.runtime.state.CheckpointStorageLocation;
import org.apache.flink.runtime.state.CheckpointStorageLocationReference;
import org.apache.flink.runtime.state.CheckpointStreamFactory;
@@ -41,6 +44,7 @@ import org.apache.flink.runtime.state.ttl.mock.MockStateBackend;
import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.sink.v2.DiscardingSink;
+import org.apache.flink.streaming.util.CheckpointStorageUtils;
import org.apache.flink.test.util.MiniClusterWithClientResource;
import org.apache.flink.testutils.junit.SharedObjects;
import org.apache.flink.testutils.junit.SharedReference;
@@ -93,11 +97,12 @@ public class UnalignedCheckpointFailureHandlingITCase {
@Test
public void testCheckpointSuccessAfterFailure() throws Exception {
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- TestCheckpointStorage storage =
- new TestCheckpointStorage(
- new JobManagerCheckpointStorage(), sharedObjects, temporaryFolder);
+ TestCheckpointStorageFactory.failOnCloseRef = sharedObjects.add(new AtomicBoolean(true));
+ TestCheckpointStorageFactory.tempFolderRef = sharedObjects.add(temporaryFolder);
- configure(env, storage);
+ configure(
+ env,
+ "org.apache.flink.test.checkpointing.UnalignedCheckpointFailureHandlingITCase$TestCheckpointStorageFactory");
buildGraph(env);
JobClient jobClient = env.executeAsync();
@@ -112,11 +117,11 @@ public class UnalignedCheckpointFailureHandlingITCase {
miniCluster.triggerCheckpoint(jobID).get();
}
- private void configure(StreamExecutionEnvironment env, TestCheckpointStorage storage) {
+ private void configure(StreamExecutionEnvironment env, String storageFactory) {
// enable checkpointing but only via API
env.enableCheckpointing(Long.MAX_VALUE, CheckpointingMode.EXACTLY_ONCE);
- env.getCheckpointConfig().setCheckpointStorage(storage);
+ CheckpointStorageUtils.configureCheckpointStorageWithFactory(env, storageFactory);
// use non-snapshotting backend to test channel state persistence integration with
// checkpoint storage
@@ -184,6 +189,19 @@ public class UnalignedCheckpointFailureHandlingITCase {
.isPresent();
}
+ public static class TestCheckpointStorageFactory implements CheckpointStorageFactory {
+
+ private static SharedReference<AtomicBoolean> failOnCloseRef;
+ private static SharedReference<TemporaryFolder> tempFolderRef;
+
+ @Override
+ public CheckpointStorage createFromConfig(ReadableConfig config, ClassLoader classLoader)
+ throws IllegalConfigurationException {
+ return new TestCheckpointStorage(
+ new JobManagerCheckpointStorage(), failOnCloseRef, tempFolderRef);
+ }
+ }
+
private static class TestCheckpointStorage implements CheckpointStorage {
private final CheckpointStorage delegate;
private final SharedReference<AtomicBoolean> failOnCloseRef;
@@ -191,11 +209,11 @@ public class UnalignedCheckpointFailureHandlingITCase {
private TestCheckpointStorage(
CheckpointStorage delegate,
- SharedObjects sharedObjects,
- TemporaryFolder tempFolder) {
+ SharedReference<AtomicBoolean> failOnCloseRef,
+ SharedReference<TemporaryFolder> tempFolderRef) {
this.delegate = delegate;
- this.failOnCloseRef = sharedObjects.add(new AtomicBoolean(true));
- this.tempFolderRef = sharedObjects.add(tempFolder);
+ this.failOnCloseRef = failOnCloseRef;
+ this.tempFolderRef = tempFolderRef;
}
@Override