You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ga...@apache.org on 2025/02/24 19:09:26 UTC
(flink) branch master updated: [FLINK-37363][state] Make state.backend.type config optional for state SQL connector
This is an automated email from the ASF dual-hosted git repository.
gaborgsomogyi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 4deb3a1c796 [FLINK-37363][state] Make state.backend.type config optional for state SQL connector
4deb3a1c796 is described below
commit 4deb3a1c796012fcc3f2c2887e2d275c51b72ed3
Author: Gabor Somogyi <ga...@apple.com>
AuthorDate: Mon Feb 24 20:09:17 2025 +0100
[FLINK-37363][state] Make state.backend.type config optional for state SQL connector
---
docs/content/docs/libs/state_processor_api.md | 14 +++++++-------
.../flink/state/table/SavepointDataStreamScanProvider.java | 12 ++++++++----
.../flink/state/table/SavepointDynamicTableSource.java | 6 ++++--
.../state/table/SavepointDynamicTableSourceFactory.java | 5 +++--
.../flink/state/table/SavepointDynamicTableSourceTest.java | 2 --
5 files changed, 22 insertions(+), 17 deletions(-)
diff --git a/docs/content/docs/libs/state_processor_api.md b/docs/content/docs/libs/state_processor_api.md
index 2ec321a05ed..0684628dfac 100644
--- a/docs/content/docs/libs/state_processor_api.md
+++ b/docs/content/docs/libs/state_processor_api.md
@@ -585,13 +585,13 @@ CREATE TABLE state_table (
### Connector options
#### General options
-| Option | Required | Default | Type | Description |
-|--------------------|----------|---------|----------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| connector | required | (none) | String | Specify what connector to use, here should be 'savepoint'. |
-| state.backend.type | required | (none) | Enum Possible values: hashmap, rocksdb | Defines the state backend which must be used for state reading. This must match with the value which was defined in Flink job which created the savepoint or checkpoint. |
-| state.path | required | (none) | String | Defines the state path which must be used for state reading. All file system that are supported by Flink can be used here. |
-| operator.uid | optional | (none) | String | Defines the operator UID which must be used for state reading (can't be used together with `operator.uid.hash`). Either `operator.uid` or `operator.uid.hash` must be specified. |
-| operator.uid.hash | optional | (none) | String | Defines the operator UID hash which must be used for state reading (can't be used together with `operator.uid`). Either `operator.uid` or `operator.uid.hash` must be specified. |
+| Option | Required | Default | Type | Description |
+|--------------------|----------|---------|----------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| connector | required | (none) | String | Specify what connector to use, here should be 'savepoint'. |
+| state.backend.type | optional | (none) | Enum Possible values: hashmap, rocksdb | Defines the state backend which must be used for state reading. This must match with the value which was defined in Flink job which created the savepoint or checkpoint. If not provided then it falls back to `state.backend.type` in flink configuration. |
+| state.path | required | (none) | String | Defines the state path which must be used for state reading. All file system that are supported by Flink can be used here. |
+| operator.uid | optional | (none) | String | Defines the operator UID which must be used for state reading (can't be used together with `operator.uid.hash`). Either `operator.uid` or `operator.uid.hash` must be specified. |
+| operator.uid.hash | optional | (none) | String | Defines the operator UID hash which must be used for state reading (can't be used together with `operator.uid`). Either `operator.uid` or `operator.uid.hash` must be specified. |
#### Connector options for column ‘#’
| Option | Required | Default | Type | Description |
diff --git a/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/table/SavepointDataStreamScanProvider.java b/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/table/SavepointDataStreamScanProvider.java
index e71bff21329..b2b12f2d201 100644
--- a/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/table/SavepointDataStreamScanProvider.java
+++ b/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/table/SavepointDataStreamScanProvider.java
@@ -36,14 +36,16 @@ import org.apache.flink.table.connector.source.DataStreamScanProvider;
import org.apache.flink.table.data.RowData;
import org.apache.flink.table.runtime.typeutils.InternalTypeInfo;
import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.util.StringUtils;
+import javax.annotation.Nullable;
import javax.naming.ConfigurationException;
import java.util.List;
/** State data stream scan provider. */
public class SavepointDataStreamScanProvider implements DataStreamScanProvider {
- private final String stateBackendType;
+ @Nullable private final String stateBackendType;
private final String statePath;
private final OperatorIdentifier operatorIdentifier;
private final String keyFormat;
@@ -51,7 +53,7 @@ public class SavepointDataStreamScanProvider implements DataStreamScanProvider {
private final RowType rowType;
public SavepointDataStreamScanProvider(
- final String stateBackendType,
+ @Nullable final String stateBackendType,
final String statePath,
final OperatorIdentifier operatorIdentifier,
final String keyFormat,
@@ -75,8 +77,10 @@ public class SavepointDataStreamScanProvider implements DataStreamScanProvider {
public DataStream<RowData> produceDataStream(
ProviderContext providerContext, StreamExecutionEnvironment execEnv) {
try {
- Configuration configuration = new Configuration();
- configuration.set(StateBackendOptions.STATE_BACKEND, stateBackendType);
+ Configuration configuration = Configuration.fromMap(execEnv.getConfiguration().toMap());
+ if (!StringUtils.isNullOrWhitespaceOnly(stateBackendType)) {
+ configuration.set(StateBackendOptions.STATE_BACKEND, stateBackendType);
+ }
StateBackend stateBackend =
StateBackendLoader.loadStateBackendFromConfig(
configuration, getClass().getClassLoader(), null);
diff --git a/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/table/SavepointDynamicTableSource.java b/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/table/SavepointDynamicTableSource.java
index e22b3ea9f5a..14847a95860 100644
--- a/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/table/SavepointDynamicTableSource.java
+++ b/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/table/SavepointDynamicTableSource.java
@@ -25,11 +25,13 @@ import org.apache.flink.table.connector.source.DynamicTableSource;
import org.apache.flink.table.connector.source.ScanTableSource;
import org.apache.flink.table.types.logical.RowType;
+import javax.annotation.Nullable;
+
import java.util.List;
/** State dynamic source. */
public class SavepointDynamicTableSource implements ScanTableSource {
- private final String stateBackendType;
+ @Nullable private final String stateBackendType;
private final String statePath;
private final OperatorIdentifier operatorIdentifier;
private final String keyFormat;
@@ -37,7 +39,7 @@ public class SavepointDynamicTableSource implements ScanTableSource {
private final RowType rowType;
public SavepointDynamicTableSource(
- final String stateBackendType,
+ @Nullable final String stateBackendType,
final String statePath,
final OperatorIdentifier operatorIdentifier,
final String keyFormat,
diff --git a/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/table/SavepointDynamicTableSourceFactory.java b/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/table/SavepointDynamicTableSourceFactory.java
index 8cdbb1a2c99..e4c2d658a1d 100644
--- a/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/table/SavepointDynamicTableSourceFactory.java
+++ b/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/table/SavepointDynamicTableSourceFactory.java
@@ -71,7 +71,7 @@ public class SavepointDynamicTableSourceFactory implements DynamicTableSourceFac
Configuration options = new Configuration();
context.getCatalogTable().getOptions().forEach(options::setString);
- final String stateBackendType = options.get(STATE_BACKEND_TYPE);
+ final String stateBackendType = options.getOptional(STATE_BACKEND_TYPE).orElse(null);
final String statePath = options.get(STATE_PATH);
final OperatorIdentifier operatorIdentifier = getOperatorIdentifier(options);
@@ -331,7 +331,6 @@ public class SavepointDynamicTableSourceFactory implements DynamicTableSourceFac
@Override
public Set<ConfigOption<?>> requiredOptions() {
final Set<ConfigOption<?>> options = new HashSet<>();
- options.add(STATE_BACKEND_TYPE);
options.add(STATE_PATH);
return options;
}
@@ -340,6 +339,8 @@ public class SavepointDynamicTableSourceFactory implements DynamicTableSourceFac
public Set<ConfigOption<?>> optionalOptions() {
final Set<ConfigOption<?>> options = new HashSet<>();
+ options.add(STATE_BACKEND_TYPE);
+
// Either UID or hash
options.add(OPERATOR_UID);
options.add(OPERATOR_UID_HASH);
diff --git a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/table/SavepointDynamicTableSourceTest.java b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/table/SavepointDynamicTableSourceTest.java
index c31efeafbeb..d8d3b14e25e 100644
--- a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/table/SavepointDynamicTableSourceTest.java
+++ b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/table/SavepointDynamicTableSourceTest.java
@@ -58,7 +58,6 @@ public class SavepointDynamicTableSourceTest {
+ ")\n"
+ "with (\n"
+ " 'connector' = 'savepoint',\n"
- + " 'state.backend.type' = 'hashmap',\n"
+ " 'state.path' = 'src/test/resources/table-state',\n"
+ " 'operator.uid' = 'keyed-state-process-uid',\n"
+ " 'fields.KeyedPojoValue.value-format' = 'com.example.state.writer.job.schema.PojoData'\n"
@@ -142,7 +141,6 @@ public class SavepointDynamicTableSourceTest {
+ ")\n"
+ "with (\n"
+ " 'connector' = 'savepoint',\n"
- + " 'state.backend.type' = 'hashmap',\n"
+ " 'state.path' = 'src/test/resources/table-state-nulls',\n"
+ " 'operator.uid' = 'keyed-state-process-uid-null',\n"
+ " 'fields.total.value-format' = 'com.example.state.writer.job.schema.PojoData'\n"