You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by dw...@apache.org on 2024/09/20 14:51:57 UTC
(flink) branch release-1.19 updated (194df6c4067 -> 513ed95ffe5)
This is an automated email from the ASF dual-hosted git repository.
dwysakowicz pushed a change to branch release-1.19
in repository https://gitbox.apache.org/repos/asf/flink.git
from 194df6c4067 [FLINK-36191][tests] FsMergingCheckpointStorageLocationTest generates test data not in tmp folder
add 67643bc7bbc [hotfix] Remove unused code in InstantiationUtil
new 513ed95ffe5 [FLINK-36318] Fix deserializing from 1.18 savepoint with RAW<MAP...> types
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/flink/util/InstantiationUtil.java | 211 ++++++++-------------
...RestoreTest.java => OverWindowRestoreTest.java} | 27 ++-
.../nodes/exec/stream/OverWindowTestPrograms.java | 61 ++++++
.../plan/nodes/exec/testutils/RestoreTestBase.java | 61 ++++--
.../1.18/savepoint/OverWindowRestoreTest | 176 +++++++++++++++++
.../over-aggregate-lag/1.18/savepoint/_metadata | Bin 0 -> 24310 bytes
.../plan/over-aggregate-lag.json} | 211 +++++++++++----------
.../over-aggregate-lag/savepoint/_metadata | Bin 0 -> 24310 bytes
.../runtime/typeutils/ArrayDataSerializer.java | 6 +-
.../table/runtime/typeutils/MapDataSerializer.java | 13 +-
.../table/runtime/typeutils/RowDataSerializer.java | 3 +-
11 files changed, 517 insertions(+), 252 deletions(-)
copy flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/{ExpandRestoreTest.java => OverWindowRestoreTest.java} (50%)
create mode 100644 flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/OverWindowTestPrograms.java
create mode 100644 flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-over-aggregate_1/over-aggregate-lag/1.18/savepoint/OverWindowRestoreTest
create mode 100644 flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-over-aggregate_1/over-aggregate-lag/1.18/savepoint/_metadata
copy flink-table/flink-table-planner/src/test/resources/restore-tests/{stream-exec-temporal-sort_1/temporal-sort-rowtime/plan/temporal-sort-rowtime.json => stream-exec-over-aggregate_1/over-aggregate-lag/plan/over-aggregate-lag.json} (60%)
create mode 100644 flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-over-aggregate_1/over-aggregate-lag/savepoint/_metadata
(flink) 01/01: [FLINK-36318] Fix deserializing from 1.18 savepoint with RAW types
Posted by dw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dwysakowicz pushed a commit to branch release-1.19
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 513ed95ffe5cac5f0f229ec66c0d8039fe8ba095
Author: Dawid Wysakowicz <dw...@apache.org>
AuthorDate: Wed Sep 18 12:12:19 2024 +0200
[FLINK-36318] Fix deserializing from 1.18 savepoint with RAW<MAP...> types
The commit fixes restoring from savepoints which were created for SQL queries which used functions such as LAG/LEAD/ARRAY_AGG which use a RAW type for accumulators.
This closes #25357
---
.../org/apache/flink/util/InstantiationUtil.java | 109 ++++++-
.../nodes/exec/stream/OverWindowRestoreTest.java | 55 ++++
.../nodes/exec/stream/OverWindowTestPrograms.java | 61 ++++
.../plan/nodes/exec/testutils/RestoreTestBase.java | 61 +++-
.../1.18/savepoint/OverWindowRestoreTest | 176 ++++++++++++
.../over-aggregate-lag/1.18/savepoint/_metadata | Bin 0 -> 24310 bytes
.../plan/over-aggregate-lag.json | 318 +++++++++++++++++++++
.../over-aggregate-lag/savepoint/_metadata | Bin 0 -> 24310 bytes
.../runtime/typeutils/ArrayDataSerializer.java | 6 +-
.../table/runtime/typeutils/MapDataSerializer.java | 13 +-
.../table/runtime/typeutils/RowDataSerializer.java | 3 +-
11 files changed, 776 insertions(+), 26 deletions(-)
diff --git a/flink-core/src/main/java/org/apache/flink/util/InstantiationUtil.java b/flink-core/src/main/java/org/apache/flink/util/InstantiationUtil.java
index ce471df0b67..2b48c6838ad 100644
--- a/flink-core/src/main/java/org/apache/flink/util/InstantiationUtil.java
+++ b/flink-core/src/main/java/org/apache/flink/util/InstantiationUtil.java
@@ -20,7 +20,6 @@ package org.apache.flink.util;
import org.apache.flink.annotation.Internal;
import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.api.common.typeutils.base.MapSerializer;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.io.IOReadableWritable;
import org.apache.flink.core.memory.DataInputView;
@@ -46,9 +45,8 @@ import java.lang.reflect.Modifier;
import java.lang.reflect.Proxy;
import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
+import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.zip.DeflaterOutputStream;
import java.util.zip.InflaterInputStream;
@@ -145,6 +143,97 @@ public final class InstantiationUtil {
}
}
+ /**
+ * Workaround for bugs like e.g. FLINK-36318 where we serialize a class into a snapshot and then
+ * its serialVersionUID is changed in an uncontrolled way. This lets us deserialize the old
+ * snapshot assuming the binary representation of the faulty class has not changed.
+ */
+ private static final class VersionMismatchHandler {
+
+ private final Map<String, Map<Long, List<Long>>> supportedSerialVersionUidsPerClass =
+ new HashMap<>();
+
+ void addVersionsMatch(
+ String className, long localVersionUID, List<Long> streamVersionUIDs) {
+ supportedSerialVersionUidsPerClass
+ .computeIfAbsent(className, k -> new HashMap<>())
+ .put(localVersionUID, streamVersionUIDs);
+ }
+
+ /**
+ * Checks if the local version of the given class can safely deserialize the class of a
+ * different version from the object stream.
+ */
+ boolean shouldTolerateSerialVersionMismatch(
+ String className, long localVersionUID, long streamVersionUID) {
+ return supportedSerialVersionUidsPerClass
+ .getOrDefault(className, Collections.emptyMap())
+ .getOrDefault(localVersionUID, Collections.emptyList())
+ .contains(streamVersionUID);
+ }
+
+ /**
+ * Checks if there are any rules for the given class. This lets us decide early if we need
+ * to look up the local class.
+ */
+ boolean haveRulesForClass(String className) {
+ return supportedSerialVersionUidsPerClass.containsKey(className);
+ }
+ }
+
+ private static final VersionMismatchHandler versionMismatchHandler =
+ new VersionMismatchHandler();
+
+ static {
+ // See FLINK-36318
+ versionMismatchHandler.addVersionsMatch(
+ "org.apache.flink.table.runtime.typeutils.MapDataSerializer",
+ 4073842523628732956L,
+ Collections.singletonList(2533002123505507000L));
+ }
+
+ /**
+ * An {@link ObjectInputStream} that ignores certain serialVersionUID mismatches. This is a
+ * workaround for uncontrolled serialVersionUIDs changes.
+ */
+ public static class FailureTolerantObjectInputStream
+ extends InstantiationUtil.ClassLoaderObjectInputStream {
+
+ public FailureTolerantObjectInputStream(InputStream in, ClassLoader cl) throws IOException {
+ super(in, cl);
+ }
+
+ @Override
+ protected ObjectStreamClass readClassDescriptor()
+ throws IOException, ClassNotFoundException {
+ ObjectStreamClass streamClassDescriptor = super.readClassDescriptor();
+
+ final Class localClass = resolveClass(streamClassDescriptor);
+ final String name = localClass.getName();
+ if (versionMismatchHandler.haveRulesForClass(name)) {
+ final ObjectStreamClass localClassDescriptor = ObjectStreamClass.lookup(localClass);
+ if (localClassDescriptor != null
+ && localClassDescriptor.getSerialVersionUID()
+ != streamClassDescriptor.getSerialVersionUID()) {
+ if (versionMismatchHandler.shouldTolerateSerialVersionMismatch(
+ name,
+ localClassDescriptor.getSerialVersionUID(),
+ streamClassDescriptor.getSerialVersionUID())) {
+ LOG.warn(
+ "Ignoring serialVersionUID mismatch for class {}; was {}, now {}.",
+ streamClassDescriptor.getName(),
+ streamClassDescriptor.getSerialVersionUID(),
+ localClassDescriptor.getSerialVersionUID());
+
+ streamClassDescriptor = localClassDescriptor;
+ }
+ }
+ }
+
+ return streamClassDescriptor;
+ }
+ }
+
/**
* Creates a new instance of the given class name and type using the provided {@link
* ClassLoader}.
@@ -379,20 +468,28 @@ public final class InstantiationUtil {
return serializer.deserialize(reuse, inputViewWrapper);
}
- @SuppressWarnings("unchecked")
public static <T> T deserializeObject(byte[] bytes, ClassLoader cl)
throws IOException, ClassNotFoundException {
return deserializeObject(new ByteArrayInputStream(bytes), cl);
}
- @SuppressWarnings("unchecked")
public static <T> T deserializeObject(InputStream in, ClassLoader cl)
throws IOException, ClassNotFoundException {
+ return deserializeObject(in, cl, false);
+ }
+
+ @SuppressWarnings("unchecked")
+ public static <T> T deserializeObject(
+ InputStream in, ClassLoader cl, boolean tolerateKnownVersionMismatch)
+ throws IOException, ClassNotFoundException {
final ClassLoader old = Thread.currentThread().getContextClassLoader();
// not using resource try to avoid AutoClosable's close() on the given stream
try {
- ObjectInputStream oois = new InstantiationUtil.ClassLoaderObjectInputStream(in, cl);
+ ObjectInputStream oois =
+ tolerateKnownVersionMismatch
+ ? new InstantiationUtil.FailureTolerantObjectInputStream(in, cl)
+ : new InstantiationUtil.ClassLoaderObjectInputStream(in, cl);
Thread.currentThread().setContextClassLoader(cl);
return (T) oois.readObject();
} finally {
diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/OverWindowRestoreTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/OverWindowRestoreTest.java
new file mode 100644
index 00000000000..cb681e4bff5
--- /dev/null
+++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/OverWindowRestoreTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.planner.plan.nodes.exec.stream;
+
+import org.apache.flink.FlinkVersion;
+import org.apache.flink.table.planner.plan.nodes.exec.ExecNodeMetadata;
+import org.apache.flink.table.planner.plan.nodes.exec.testutils.RestoreTestBase;
+import org.apache.flink.table.test.program.TableTestProgram;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Stream;
+
+/** Tests for verifying {@link StreamExecOverAggregate}. */
+public class OverWindowRestoreTest extends RestoreTestBase {
+ public OverWindowRestoreTest() {
+ super(StreamExecOverAggregate.class);
+ }
+
+ @Override
+ protected Stream<String> getSavepointPaths(
+ TableTestProgram program, ExecNodeMetadata metadata) {
+ if (metadata.version() == 1) {
+ return Stream.concat(
+ super.getSavepointPaths(program, metadata),
+ // See src/test/resources/restore-tests/stream-exec-over-aggregate_1/over
+ // -aggregate-lag/1.18/savepoint/OverWindowRestoreTest how the savepoint was
+ // generated
+ Stream.of(getSavepointPath(program, metadata, FlinkVersion.v1_18)));
+ } else {
+ return super.getSavepointPaths(program, metadata);
+ }
+ }
+
+ @Override
+ public List<TableTestProgram> programs() {
+ return Collections.singletonList(OverWindowTestPrograms.LAG_OVER_FUNCTION);
+ }
+}
diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/OverWindowTestPrograms.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/OverWindowTestPrograms.java
new file mode 100644
index 00000000000..e9fdbb03a42
--- /dev/null
+++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/OverWindowTestPrograms.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.planner.plan.nodes.exec.stream;
+
+import org.apache.flink.table.test.program.SinkTestStep;
+import org.apache.flink.table.test.program.SourceTestStep;
+import org.apache.flink.table.test.program.TableTestProgram;
+import org.apache.flink.types.Row;
+
+import java.util.Collections;
+
+/** Programs for verifying {@link StreamExecOverAggregate}. */
+public class OverWindowTestPrograms {
+ static final TableTestProgram LAG_OVER_FUNCTION =
+ TableTestProgram.of("over-aggregate-lag", "validates restoring a lag function")
+ .setupTableSource(
+ SourceTestStep.newBuilder("t")
+ .addSchema(
+ "ts STRING",
+ "b MAP<DOUBLE, DOUBLE>",
+ "`r_time` AS TO_TIMESTAMP(`ts`)",
+ "WATERMARK for `r_time` AS `r_time`")
+ .producedBeforeRestore(
+ Row.of(
+ "2020-04-15 08:00:05",
+ Collections.singletonMap(42.0, 42.0)))
+ .producedAfterRestore(
+ Row.of(
+ "2020-04-15 08:00:06",
+ Collections.singletonMap(42.1, 42.1)))
+ .build())
+ .setupTableSink(
+ SinkTestStep.newBuilder("sink_t")
+ .addSchema("ts STRING", "b MAP<DOUBLE, DOUBLE>")
+ .consumedBeforeRestore(Row.of("2020-04-15 08:00:05", null))
+ .consumedAfterRestore(
+ Row.of(
+ "2020-04-15 08:00:06",
+ Collections.singletonMap(42.0, 42.0)))
+ .build())
+ .runSql(
+ "INSERT INTO sink_t SELECT ts, LAG(b, 1) over (order by r_time) AS "
+ + "bLag FROM t")
+ .build();
+}
diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/testutils/RestoreTestBase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/testutils/RestoreTestBase.java
index 7114cd8dfe9..7e4d3eed8d8 100644
--- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/testutils/RestoreTestBase.java
+++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/testutils/RestoreTestBase.java
@@ -18,6 +18,7 @@
package org.apache.flink.table.planner.plan.nodes.exec.testutils;
+import org.apache.flink.FlinkVersion;
import org.apache.flink.api.common.JobStatus;
import org.apache.flink.configuration.StateBackendOptions;
import org.apache.flink.core.execution.JobClient;
@@ -57,6 +58,8 @@ import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
+import javax.annotation.Nullable;
+
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -87,6 +90,9 @@ import static org.assertj.core.api.Assertions.assertThat;
@TestMethodOrder(OrderAnnotation.class)
public abstract class RestoreTestBase implements TableTestProgramRunner {
+ // This version can be set to generate savepoints for a particular Flink version.
+ // By default, the savepoint is generated for the current version in the default directory.
+ private static final FlinkVersion FLINK_VERSION = null;
private final Class<? extends ExecNode<?>> execNodeUnderTest;
private final AfterRestoreSource afterRestoreSource;
@@ -144,7 +150,42 @@ public abstract class RestoreTestBase implements TableTestProgramRunner {
return getAllMetadata().stream()
.flatMap(
metadata ->
- supportedPrograms().stream().map(p -> Arguments.of(p, metadata)));
+ supportedPrograms().stream()
+ .flatMap(
+ program ->
+ getSavepointPaths(program, metadata)
+ .map(
+ savepointPath ->
+ Arguments.of(
+ program,
+ getPlanPath(
+ program,
+ metadata),
+ savepointPath))));
+ }
+
+ /**
+ * The method can be overridden in a subclass to test multiple savepoint files for a given
+ * program and a node in a particular version. This can be useful e.g. to test a node against
+ * savepoint generated in different Flink versions.
+ */
+ protected Stream<String> getSavepointPaths(
+ TableTestProgram program, ExecNodeMetadata metadata) {
+ return Stream.of(getSavepointPath(program, metadata, null));
+ }
+
+ protected final String getSavepointPath(
+ TableTestProgram program,
+ ExecNodeMetadata metadata,
+ @Nullable FlinkVersion flinkVersion) {
+ StringBuilder builder = new StringBuilder();
+ builder.append(getTestResourceDirectory(program, metadata));
+ if (flinkVersion != null) {
+ builder.append("/").append(flinkVersion);
+ }
+ builder.append("/savepoint/");
+
+ return builder.toString();
}
private void registerSinkObserver(
@@ -231,7 +272,8 @@ public abstract class RestoreTestBase implements TableTestProgramRunner {
.get();
CommonTestUtils.waitForJobStatus(jobClient, Collections.singletonList(JobStatus.FINISHED));
final Path savepointPath = Paths.get(new URI(savepoint));
- final Path savepointDirPath = getSavepointPath(program, getLatestMetadata());
+ final Path savepointDirPath =
+ Paths.get(getSavepointPath(program, getLatestMetadata(), FLINK_VERSION));
Files.createDirectories(savepointDirPath);
Files.move(savepointPath, savepointDirPath, StandardCopyOption.ATOMIC_MOVE);
}
@@ -239,13 +281,11 @@ public abstract class RestoreTestBase implements TableTestProgramRunner {
@ParameterizedTest
@MethodSource("createSpecs")
@Order(1)
- void testRestore(TableTestProgram program, ExecNodeMetadata metadata) throws Exception {
+ void testRestore(TableTestProgram program, Path planPath, String savepointPath)
+ throws Exception {
final EnvironmentSettings settings = EnvironmentSettings.inStreamingMode();
final SavepointRestoreSettings restoreSettings =
- SavepointRestoreSettings.forPath(
- getSavepointPath(program, metadata).toString(),
- false,
- RestoreMode.NO_CLAIM);
+ SavepointRestoreSettings.forPath(savepointPath, false, RestoreMode.NO_CLAIM);
SavepointRestoreSettings.toConfiguration(restoreSettings, settings.getConfiguration());
settings.getConfiguration().set(StateBackendOptions.STATE_BACKEND, "rocksdb");
final TableEnvironment tEnv = TableEnvironment.create(settings);
@@ -284,8 +324,7 @@ public abstract class RestoreTestBase implements TableTestProgramRunner {
program.getSetupFunctionTestSteps().forEach(s -> s.apply(tEnv));
program.getSetupTemporalFunctionTestSteps().forEach(s -> s.apply(tEnv));
- final CompiledPlan compiledPlan =
- tEnv.loadPlan(PlanReference.fromFile(getPlanPath(program, metadata)));
+ final CompiledPlan compiledPlan = tEnv.loadPlan(PlanReference.fromFile(planPath));
if (afterRestoreSource == AfterRestoreSource.INFINITE) {
final TableResult tableResult = compiledPlan.execute();
@@ -312,10 +351,6 @@ public abstract class RestoreTestBase implements TableTestProgramRunner {
getTestResourceDirectory(program, metadata) + "/plan/" + program.id + ".json");
}
- private Path getSavepointPath(TableTestProgram program, ExecNodeMetadata metadata) {
- return Paths.get(getTestResourceDirectory(program, metadata) + "/savepoint/");
- }
-
private String getTestResourceDirectory(TableTestProgram program, ExecNodeMetadata metadata) {
return String.format(
"%s/src/test/resources/restore-tests/%s_%d/%s",
diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-over-aggregate_1/over-aggregate-lag/1.18/savepoint/OverWindowRestoreTest b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-over-aggregate_1/over-aggregate-lag/1.18/savepoint/OverWindowRestoreTest
new file mode 100644
index 00000000000..08e2541de98
--- /dev/null
+++ b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-over-aggregate_1/over-aggregate-lag/1.18/savepoint/OverWindowRestoreTest
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.planner.plan.nodes.exec.stream;
+
+import org.apache.commons.collections.CollectionUtils;
+
+import org.apache.flink.api.common.JobStatus;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.connector.source.Boundedness;
+import org.apache.flink.api.connector.source.Source;
+import org.apache.flink.api.connector.source.SourceReader;
+import org.apache.flink.api.connector.source.SourceReaderContext;
+import org.apache.flink.api.connector.source.SplitEnumerator;
+import org.apache.flink.api.connector.source.SplitEnumeratorContext;
+import org.apache.flink.configuration.StateBackendOptions;
+import org.apache.flink.connector.source.enumerator.NoOpEnumState;
+import org.apache.flink.connector.source.enumerator.NoOpEnumStateSerializer;
+import org.apache.flink.connector.source.enumerator.ValuesSourceEnumerator;
+import org.apache.flink.connector.source.split.ValuesSourceSplit;
+import org.apache.flink.connector.source.split.ValuesSourceSplitSerializer;
+import org.apache.flink.core.execution.JobClient;
+import org.apache.flink.core.execution.SavepointFormatType;
+import org.apache.flink.core.io.SimpleVersionedSerializer;
+import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
+import org.apache.flink.runtime.testutils.CommonTestUtils;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.table.api.CompiledPlan;
+import org.apache.flink.table.api.EnvironmentSettings;
+import org.apache.flink.table.api.TableEnvironment;
+
+import org.apache.flink.table.api.TableException;
+import org.apache.flink.table.api.TableResult;
+import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
+import org.apache.flink.table.api.config.TableConfigOptions;
+
+import org.apache.flink.table.data.GenericMapData;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.MapData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.StringData;
+import org.apache.flink.table.planner.factories.TestValuesTableFactory;
+
+import org.apache.flink.test.junit5.MiniClusterExtension;
+import org.apache.flink.types.Row;
+import org.apache.flink.util.Preconditions;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.io.TempDir;
+
+import javax.annotation.Nullable;
+
+import java.io.ByteArrayOutputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.concurrent.CompletableFuture;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+// Run this test to generate a savepoint in Flink 1.18.
+// In order to run the test, you need to cherry-pick commits:
+// - dcce3764a4500b2006cd260677169d14c553a3eb
+// - e914eb7fc3f31286ed7e33cc93e7ffbca785b731
+// - a5b4e60e563bf145afede43dda8510833d2932e4
+@ExtendWith(MiniClusterExtension.class)
+class OverWindowRestoreTest {
+
+ private @TempDir Path tmpDir;
+
+ private final Path savepointDirPath = Paths.get(String.format(
+ "%s/src/test/resources/restore-tests/%s_%d/%s/savepoint/",
+ System.getProperty("user.dir"), "stream-exec-over-aggregate", 1,
+ "lag-function"));
+
+ @Test
+ void testOverWindowRestore() throws Exception {
+ final EnvironmentSettings settings = EnvironmentSettings.inStreamingMode();
+ settings.getConfiguration().set(StateBackendOptions.STATE_BACKEND, "rocksdb");
+ final TableEnvironment tEnv = TableEnvironment.create(settings);
+ tEnv.getConfig()
+ .set(
+ TableConfigOptions.PLAN_COMPILE_CATALOG_OBJECTS,
+ TableConfigOptions.CatalogPlanCompilation.SCHEMA);
+
+ final String id = TestValuesTableFactory.registerData(
+ Collections.singletonList(
+ Row.of("2020-04-15 08:00:05", Collections.singletonMap(42.0, 42.0))
+ )
+ );
+
+ tEnv.executeSql(
+ "CREATE TABLE t(\n"
+ + "ts STRING,\n"
+ + "b MAP<DOUBLE, DOUBLE>,\n"
+ + "r_time AS TO_TIMESTAMP(`ts`),\n"
+ + "WATERMARK for `r_time` AS `r_time`\n"
+ + ") WITH(\n"
+ + "'connector' = 'values',\n"
+ + String.format("'data-id' = '%s',\n", id)
+ + "'terminating' = 'false',\n"
+ + "'runtime-source' = 'NewSource'\n"
+ + ")\n"
+ );
+
+ CompletableFuture<?> future = new CompletableFuture<>();
+ TestValuesTableFactory.registerLocalRawResultsObserver(
+ "sink_t",
+ (integer, strings) -> {
+ List<String> results = Collections.singletonList("+I[2020-04-15 08:00:05,"
+ + " null]");
+ List<String> currentResults = TestValuesTableFactory.getRawResultsAsStrings("sink_t");
+ final boolean shouldComplete =
+ CollectionUtils.isEqualCollection(currentResults, results);
+ if (shouldComplete) {
+ future.complete(null);
+ }
+ }
+ );
+
+ tEnv.executeSql(
+ "CREATE TABLE sink_t(\n"
+ + "ts STRING,\n"
+ + "b MAP<DOUBLE, DOUBLE>\n"
+ + ") WITH(\n"
+ + "'connector' = 'values',\n"
+ + "'sink-insert-only' = 'false'\n"
+ + ")\n"
+ );
+
+ final CompiledPlan compiledPlan = tEnv.compilePlanSql(
+ "INSERT INTO sink_t SELECT ts, LAG(b, 1) over (order by r_time) AS bLag FROM t");
+
+ final TableResult tableResult = compiledPlan.execute();
+ future.get();
+ final JobClient jobClient = tableResult.getJobClient().get();
+ final String savepoint =
+ jobClient
+ .stopWithSavepoint(false, tmpDir.toString(), SavepointFormatType.DEFAULT)
+ .get();
+ CommonTestUtils.waitForJobStatus(jobClient, Collections.singletonList(JobStatus.FINISHED));
+ final Path savepointPath = Paths.get(new URI(savepoint));
+ Files.createDirectories(savepointDirPath);
+ Files.move(savepointPath, savepointDirPath, StandardCopyOption.ATOMIC_MOVE);
+ }
+}
diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-over-aggregate_1/over-aggregate-lag/1.18/savepoint/_metadata b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-over-aggregate_1/over-aggregate-lag/1.18/savepoint/_metadata
new file mode 100644
index 00000000000..91cb0c70428
Binary files /dev/null and b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-over-aggregate_1/over-aggregate-lag/1.18/savepoint/_metadata differ
diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-over-aggregate_1/over-aggregate-lag/plan/over-aggregate-lag.json b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-over-aggregate_1/over-aggregate-lag/plan/over-aggregate-lag.json
new file mode 100644
index 00000000000..c09e4e1171c
--- /dev/null
+++ b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-over-aggregate_1/over-aggregate-lag/plan/over-aggregate-lag.json
@@ -0,0 +1,318 @@
+{
+ "flinkVersion" : "1.19",
+ "nodes" : [ {
+ "id" : 1,
+ "type" : "stream-exec-table-source-scan_1",
+ "scanTableSource" : {
+ "table" : {
+ "identifier" : "`default_catalog`.`default_database`.`t`",
+ "resolvedTable" : {
+ "schema" : {
+ "columns" : [ {
+ "name" : "ts",
+ "dataType" : "VARCHAR(2147483647)"
+ }, {
+ "name" : "b",
+ "dataType" : "MAP<DOUBLE, DOUBLE>"
+ }, {
+ "name" : "r_time",
+ "kind" : "COMPUTED",
+ "expression" : {
+ "rexNode" : {
+ "kind" : "CALL",
+ "internalName" : "$TO_TIMESTAMP$1",
+ "operands" : [ {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 0,
+ "type" : "VARCHAR(2147483647)"
+ } ],
+ "type" : "TIMESTAMP(3)"
+ },
+ "serializableString" : "TO_TIMESTAMP(`ts`)"
+ }
+ } ],
+ "watermarkSpecs" : [ {
+ "rowtimeAttribute" : "r_time",
+ "expression" : {
+ "rexNode" : {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 2,
+ "type" : "TIMESTAMP(3)"
+ },
+ "serializableString" : "`r_time`"
+ }
+ } ]
+ },
+ "partitionKeys" : [ ]
+ }
+ }
+ },
+ "outputType" : "ROW<`ts` VARCHAR(2147483647), `b` MAP<DOUBLE, DOUBLE>>",
+ "description" : "TableSourceScan(table=[[default_catalog, default_database, t]], fields=[ts, b])",
+ "inputProperties" : [ ]
+ }, {
+ "id" : 2,
+ "type" : "stream-exec-calc_1",
+ "projection" : [ {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 0,
+ "type" : "VARCHAR(2147483647)"
+ }, {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 1,
+ "type" : "MAP<DOUBLE, DOUBLE>"
+ }, {
+ "kind" : "CALL",
+ "internalName" : "$TO_TIMESTAMP$1",
+ "operands" : [ {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 0,
+ "type" : "VARCHAR(2147483647)"
+ } ],
+ "type" : "TIMESTAMP(3)"
+ } ],
+ "condition" : null,
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "UNKNOWN"
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : "ROW<`ts` VARCHAR(2147483647), `b` MAP<DOUBLE, DOUBLE>, `r_time` TIMESTAMP(3)>",
+ "description" : "Calc(select=[ts, b, TO_TIMESTAMP(ts) AS r_time])"
+ }, {
+ "id" : 3,
+ "type" : "stream-exec-watermark-assigner_1",
+ "watermarkExpr" : {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 2,
+ "type" : "TIMESTAMP(3)"
+ },
+ "rowtimeFieldIndex" : 2,
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "UNKNOWN"
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : {
+ "type" : "ROW",
+ "fields" : [ {
+ "name" : "ts",
+ "fieldType" : "VARCHAR(2147483647)"
+ }, {
+ "name" : "b",
+ "fieldType" : "MAP<DOUBLE, DOUBLE>"
+ }, {
+ "name" : "r_time",
+ "fieldType" : {
+ "type" : "TIMESTAMP_WITHOUT_TIME_ZONE",
+ "precision" : 3,
+ "kind" : "ROWTIME"
+ }
+ } ]
+ },
+ "description" : "WatermarkAssigner(rowtime=[r_time], watermark=[r_time])"
+ }, {
+ "id" : 4,
+ "type" : "stream-exec-exchange_1",
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "SINGLETON"
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : {
+ "type" : "ROW",
+ "fields" : [ {
+ "name" : "ts",
+ "fieldType" : "VARCHAR(2147483647)"
+ }, {
+ "name" : "b",
+ "fieldType" : "MAP<DOUBLE, DOUBLE>"
+ }, {
+ "name" : "r_time",
+ "fieldType" : {
+ "type" : "TIMESTAMP_WITHOUT_TIME_ZONE",
+ "precision" : 3,
+ "kind" : "ROWTIME"
+ }
+ } ]
+ },
+ "description" : "Exchange(distribution=[single])"
+ }, {
+ "id" : 5,
+ "type" : "stream-exec-over-aggregate_1",
+ "overSpec" : {
+ "partition" : {
+ "fields" : [ ]
+ },
+ "groups" : [ {
+ "orderBy" : {
+ "fields" : [ {
+ "index" : 2,
+ "isAscending" : true,
+ "nullIsLast" : false
+ } ]
+ },
+ "isRows" : false,
+ "lowerBound" : {
+ "kind" : "UNBOUNDED_PRECEDING"
+ },
+ "upperBound" : {
+ "kind" : "CURRENT_ROW"
+ },
+ "aggCalls" : [ {
+ "name" : "w0$o0",
+ "internalName" : "$LAG$1",
+ "argList" : [ 1, 3 ],
+ "filterArg" : -1,
+ "distinct" : false,
+ "approximate" : false,
+ "ignoreNulls" : false,
+ "type" : "MAP<DOUBLE, DOUBLE>"
+ } ]
+ } ],
+ "constants" : [ {
+ "kind" : "LITERAL",
+ "value" : 1,
+ "type" : "INT NOT NULL"
+ } ],
+ "originalInputFields" : 3
+ },
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "UNKNOWN"
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : {
+ "type" : "ROW",
+ "fields" : [ {
+ "name" : "ts",
+ "fieldType" : "VARCHAR(2147483647)"
+ }, {
+ "name" : "b",
+ "fieldType" : "MAP<DOUBLE, DOUBLE>"
+ }, {
+ "name" : "r_time",
+ "fieldType" : {
+ "type" : "TIMESTAMP_WITHOUT_TIME_ZONE",
+ "precision" : 3,
+ "kind" : "ROWTIME"
+ }
+ }, {
+ "name" : "w0$o0",
+ "fieldType" : "MAP<DOUBLE, DOUBLE>"
+ } ]
+ },
+ "description" : "OverAggregate(orderBy=[r_time ASC], window=[ RANG BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], select=[ts, b, r_time, LAG(b, 1) AS w0$o0])"
+ }, {
+ "id" : 6,
+ "type" : "stream-exec-calc_1",
+ "projection" : [ {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 0,
+ "type" : "VARCHAR(2147483647)"
+ }, {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 3,
+ "type" : "MAP<DOUBLE, DOUBLE>"
+ } ],
+ "condition" : null,
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "UNKNOWN"
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : "ROW<`ts` VARCHAR(2147483647), `$1` MAP<DOUBLE, DOUBLE>>",
+ "description" : "Calc(select=[ts, w0$o0 AS $1])"
+ }, {
+ "id" : 7,
+ "type" : "stream-exec-sink_1",
+ "configuration" : {
+ "table.exec.sink.keyed-shuffle" : "AUTO",
+ "table.exec.sink.not-null-enforcer" : "ERROR",
+ "table.exec.sink.rowtime-inserter" : "ENABLED",
+ "table.exec.sink.type-length-enforcer" : "IGNORE",
+ "table.exec.sink.upsert-materialize" : "AUTO"
+ },
+ "dynamicTableSink" : {
+ "table" : {
+ "identifier" : "`default_catalog`.`default_database`.`sink_t`",
+ "resolvedTable" : {
+ "schema" : {
+ "columns" : [ {
+ "name" : "ts",
+ "dataType" : "VARCHAR(2147483647)"
+ }, {
+ "name" : "b",
+ "dataType" : "MAP<DOUBLE, DOUBLE>"
+ } ],
+ "watermarkSpecs" : [ ]
+ },
+ "partitionKeys" : [ ]
+ }
+ }
+ },
+ "inputChangelogMode" : [ "INSERT" ],
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "UNKNOWN"
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : "ROW<`ts` VARCHAR(2147483647), `$1` MAP<DOUBLE, DOUBLE>>",
+ "description" : "Sink(table=[default_catalog.default_database.sink_t], fields=[ts, $1])"
+ } ],
+ "edges" : [ {
+ "source" : 1,
+ "target" : 2,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ }, {
+ "source" : 2,
+ "target" : 3,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ }, {
+ "source" : 3,
+ "target" : 4,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ }, {
+ "source" : 4,
+ "target" : 5,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ }, {
+ "source" : 5,
+ "target" : 6,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ }, {
+ "source" : 6,
+ "target" : 7,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ } ]
+}
\ No newline at end of file
diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-over-aggregate_1/over-aggregate-lag/savepoint/_metadata b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-over-aggregate_1/over-aggregate-lag/savepoint/_metadata
new file mode 100644
index 00000000000..6743d9776ce
Binary files /dev/null and b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-over-aggregate_1/over-aggregate-lag/savepoint/_metadata differ
diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/typeutils/ArrayDataSerializer.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/typeutils/ArrayDataSerializer.java
index 98c6bd8d297..bf9783789fe 100644
--- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/typeutils/ArrayDataSerializer.java
+++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/typeutils/ArrayDataSerializer.java
@@ -310,8 +310,10 @@ public class ArrayDataSerializer extends TypeSerializer<ArrayData> {
throws IOException {
try {
DataInputViewStream inStream = new DataInputViewStream(in);
- this.eleType = InstantiationUtil.deserializeObject(inStream, userCodeClassLoader);
- this.eleSer = InstantiationUtil.deserializeObject(inStream, userCodeClassLoader);
+ this.eleType =
+ InstantiationUtil.deserializeObject(inStream, userCodeClassLoader, true);
+ this.eleSer =
+ InstantiationUtil.deserializeObject(inStream, userCodeClassLoader, true);
} catch (ClassNotFoundException e) {
throw new IOException(e);
}
diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/typeutils/MapDataSerializer.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/typeutils/MapDataSerializer.java
index c667777c896..cd58a4bc0a3 100644
--- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/typeutils/MapDataSerializer.java
+++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/typeutils/MapDataSerializer.java
@@ -44,6 +44,9 @@ import java.io.IOException;
@Internal
public class MapDataSerializer extends TypeSerializer<MapData> {
+ // version used since 1.19
+ private static final long serialVersionUID = 4073842523628732956L;
+
private final LogicalType keyType;
private final LogicalType valueType;
@@ -293,12 +296,14 @@ public class MapDataSerializer extends TypeSerializer<MapData> {
throws IOException {
try {
DataInputViewStream inStream = new DataInputViewStream(in);
- this.keyType = InstantiationUtil.deserializeObject(inStream, userCodeClassLoader);
- this.valueType = InstantiationUtil.deserializeObject(inStream, userCodeClassLoader);
+ this.keyType =
+ InstantiationUtil.deserializeObject(inStream, userCodeClassLoader, true);
+ this.valueType =
+ InstantiationUtil.deserializeObject(inStream, userCodeClassLoader, true);
this.keySerializer =
- InstantiationUtil.deserializeObject(inStream, userCodeClassLoader);
+ InstantiationUtil.deserializeObject(inStream, userCodeClassLoader, true);
this.valueSerializer =
- InstantiationUtil.deserializeObject(inStream, userCodeClassLoader);
+ InstantiationUtil.deserializeObject(inStream, userCodeClassLoader, true);
} catch (ClassNotFoundException e) {
throw new IOException(e);
}
diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/typeutils/RowDataSerializer.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/typeutils/RowDataSerializer.java
index 178f4b0e8aa..3106d9e65d5 100644
--- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/typeutils/RowDataSerializer.java
+++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/typeutils/RowDataSerializer.java
@@ -314,7 +314,8 @@ public class RowDataSerializer extends AbstractRowDataSerializer<RowData> {
types = new LogicalType[length];
for (int i = 0; i < length; i++) {
try {
- types[i] = InstantiationUtil.deserializeObject(stream, userCodeClassLoader);
+ types[i] =
+ InstantiationUtil.deserializeObject(stream, userCodeClassLoader, true);
} catch (ClassNotFoundException e) {
throw new IOException(e);
}