You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ro...@apache.org on 2024/07/02 01:32:15 UTC
(flink) branch release-1.20 updated (7f40c35fc74 -> d147bbf33d2)
This is an automated email from the ASF dual-hosted git repository.
ron pushed a change to branch release-1.20
in repository https://gitbox.apache.org/repos/asf/flink.git
from 7f40c35fc74 [FLINK-35699][k8s] Fix shading Jackson through fabric8
new cb3e9f4adfa [FLINK-35734][table] Fix periodic refresh job naming
new d147bbf33d2 [FLINK-35734][table] Do not override the user-defined checkpoint interval in continuous mode
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../MaterializedTableManager.java | 24 ++++++---
.../service/MaterializedTableStatementITCase.java | 62 ++++++++++++++++++++++
2 files changed, 80 insertions(+), 6 deletions(-)
(flink) 02/02: [FLINK-35734][table] Do not override the user-defined checkpoint interval in continuous mode
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ron pushed a commit to branch release-1.20
in repository https://gitbox.apache.org/repos/asf/flink.git
commit d147bbf33d2804092081b24ed7a43f244dc9580b
Author: Feng Jin <ji...@gmail.com>
AuthorDate: Mon Jul 1 16:54:17 2024 +0800
[FLINK-35734][table] Do not override the user-defined checkpoint interval in continuous mode
---
.../MaterializedTableManager.java | 13 +++--
.../service/MaterializedTableStatementITCase.java | 62 ++++++++++++++++++++++
2 files changed, 72 insertions(+), 3 deletions(-)
diff --git a/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/materializedtable/MaterializedTableManager.java b/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/materializedtable/MaterializedTableManager.java
index 2a068187ad5..aa0877265a0 100644
--- a/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/materializedtable/MaterializedTableManager.java
+++ b/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/materializedtable/MaterializedTableManager.java
@@ -541,11 +541,18 @@ public class MaterializedTableManager {
materializedTableIdentifier.asSerializableString());
customConfig.set(NAME, jobName);
customConfig.set(RUNTIME_MODE, STREAMING);
- customConfig.set(
- CheckpointingOptions.CHECKPOINTING_INTERVAL,
- catalogMaterializedTable.getFreshness());
restorePath.ifPresent(s -> customConfig.set(SAVEPOINT_PATH, s));
+ // Do not override the user-defined checkpoint interval
+ if (!operationExecutor
+ .getSessionContext()
+ .getSessionConf()
+ .contains(CheckpointingOptions.CHECKPOINTING_INTERVAL)) {
+ customConfig.set(
+ CheckpointingOptions.CHECKPOINTING_INTERVAL,
+ catalogMaterializedTable.getFreshness());
+ }
+
String insertStatement =
getInsertStatement(
materializedTableIdentifier,
diff --git a/flink-table/flink-sql-gateway/src/test/java/org/apache/flink/table/gateway/service/MaterializedTableStatementITCase.java b/flink-table/flink-sql-gateway/src/test/java/org/apache/flink/table/gateway/service/MaterializedTableStatementITCase.java
index 6ed91b954af..ef67d803246 100644
--- a/flink-table/flink-sql-gateway/src/test/java/org/apache/flink/table/gateway/service/MaterializedTableStatementITCase.java
+++ b/flink-table/flink-sql-gateway/src/test/java/org/apache/flink/table/gateway/service/MaterializedTableStatementITCase.java
@@ -73,6 +73,7 @@ import java.util.Map;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
+import static org.apache.flink.configuration.CheckpointingOptions.CHECKPOINTING_INTERVAL;
import static org.apache.flink.table.api.config.TableConfigOptions.RESOURCES_DOWNLOAD_DIR;
import static org.apache.flink.table.factories.FactoryUtil.WORKFLOW_SCHEDULER_TYPE;
import static org.apache.flink.table.gateway.service.utils.SqlGatewayServiceTestUtil.awaitOperationTermination;
@@ -170,6 +171,67 @@ public class MaterializedTableStatementITCase extends AbstractMaterializedTableS
assertThat(checkpointInterval).isEqualTo(30 * 1000);
}
+ @Test
+ void testCreateMaterializedTableInContinuousModeWithCustomCheckpointInterval()
+ throws Exception {
+
+ // set checkpoint interval to 60 seconds
+ long checkpointInterval = 60 * 1000;
+
+ OperationHandle checkpointSetHandle =
+ service.executeStatement(
+ sessionHandle,
+ String.format(
+ "SET '%s' = '%d'",
+ CHECKPOINTING_INTERVAL.key(), checkpointInterval),
+ -1,
+ new Configuration());
+ awaitOperationTermination(service, sessionHandle, checkpointSetHandle);
+
+ String materializedTableDDL =
+ "CREATE MATERIALIZED TABLE users_shops"
+ + " PARTITIONED BY (ds)\n"
+ + " WITH(\n"
+ + " 'format' = 'debezium-json'\n"
+ + " )\n"
+ + " FRESHNESS = INTERVAL '30' SECOND\n"
+ + " AS SELECT \n"
+ + " user_id,\n"
+ + " shop_id,\n"
+ + " ds,\n"
+ + " SUM (payment_amount_cents) AS payed_buy_fee_sum,\n"
+ + " SUM (1) AS pv\n"
+ + " FROM (\n"
+ + " SELECT user_id, shop_id, DATE_FORMAT(order_created_at, 'yyyy-MM-dd') AS ds, payment_amount_cents FROM datagenSource"
+ + " ) AS tmp\n"
+ + " GROUP BY (user_id, shop_id, ds)";
+ OperationHandle materializedTableHandle =
+ service.executeStatement(
+ sessionHandle, materializedTableDDL, -1, new Configuration());
+ awaitOperationTermination(service, sessionHandle, materializedTableHandle);
+
+ ResolvedCatalogMaterializedTable actualMaterializedTable =
+ (ResolvedCatalogMaterializedTable)
+ service.getTable(
+ sessionHandle,
+ ObjectIdentifier.of(
+ fileSystemCatalogName,
+ TEST_DEFAULT_DATABASE,
+ "users_shops"));
+
+ ContinuousRefreshHandler activeRefreshHandler =
+ ContinuousRefreshHandlerSerializer.INSTANCE.deserialize(
+ actualMaterializedTable.getSerializedRefreshHandler(),
+ getClass().getClassLoader());
+
+ waitUntilAllTasksAreRunning(
+ restClusterClient, JobID.fromHexString(activeRefreshHandler.getJobId()));
+
+ long actualCheckpointInterval =
+ getCheckpointIntervalConfig(restClusterClient, activeRefreshHandler.getJobId());
+ assertThat(actualCheckpointInterval).isEqualTo(checkpointInterval);
+ }
+
@Test
void testCreateMaterializedTableInFullMode() throws Exception {
String dataId = TestValuesTableFactory.registerData(Collections.emptyList());
(flink) 01/02: [FLINK-35734][table] Fix periodic refresh job naming
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ron pushed a commit to branch release-1.20
in repository https://gitbox.apache.org/repos/asf/flink.git
commit cb3e9f4adfa602de972464555022a0d642d6b9cf
Author: Feng Jin <ji...@gmail.com>
AuthorDate: Mon Jul 1 16:33:39 2024 +0800
[FLINK-35734][table] Fix periodic refresh job naming
---
.../service/materializedtable/MaterializedTableManager.java | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
diff --git a/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/materializedtable/MaterializedTableManager.java b/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/materializedtable/MaterializedTableManager.java
index 293634e66d1..2a068187ad5 100644
--- a/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/materializedtable/MaterializedTableManager.java
+++ b/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/materializedtable/MaterializedTableManager.java
@@ -621,9 +621,14 @@ public class MaterializedTableManager {
// Set job name, runtime mode
Configuration customConfig = new Configuration();
String jobName =
- String.format(
- "Materialized_table_%s_one_time_refresh_job",
- materializedTableIdentifier.asSerializableString());
+ isPeriodic
+ ? String.format(
+ "Materialized_table_%s_periodic_refresh_job",
+ materializedTableIdentifier.asSerializableString())
+ : String.format(
+ "Materialized_table_%s_one_time_refresh_job",
+ materializedTableIdentifier.asSerializableString());
+
customConfig.set(NAME, jobName);
customConfig.set(RUNTIME_MODE, BATCH);