You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tz...@apache.org on 2019/02/22 07:45:31 UTC

[flink] branch master updated (c2c45d6 -> 04ede2c)

This is an automated email from the ASF dual-hosted git repository.

tzulitai pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git.


    from c2c45d6  [hotfix] [tests] Fix EnumValueSerializerSnapshotMigrationTest for Scala 2.12 builds
     new 633ed95  [FLINK-11323] Add OptionalMap to be used by Kryo
     new 6685d6a  [FLINK-11323] [tests] Adjust serializer tests to respect new COMPATIBLE_WITH_RECONFIGURED_SERIALIZER option
     new f436fbf  [FLINK-11323] [core] Migrate KryoSerializer to use new serialization compatibility abstractions
     new 22a7ecc  [FLINK-11323] [tests] Add more test cases to TypeSerializerSnapshotMigrationTestBase
     new 9d8a2e2  [FLINK-11323] Migrate WritableSerializer to use new serialization compatibility abstractions
     new 4fa8288  [FLINK-11323] Migrate ValueSerializer to use new serialization compatibility abstractions
     new 186b8df  [FLINK-11323] Migrate TrySerializer to use new serialization compatibility abstractions
     new a3822f2  [FLINK-11485] [core] Move OptionalMap to org.apache.flink.util
     new cf7b86d  [FLINK-11485] [core] Rename OptionalMap to LinkedOptionalMap
     new d5d45c8  [FLINK-11485] [core] Refactor multiple nested serializer compatibility resolution logic to CompositeTypeSerializerUtil
     new 7621c57  [FLINK-11485] [core] Implement new PojoSerializerSnapshot class
     new 1850193  [FLINK-11485] [core] Migrate PojoSerializer to use new serialization compatibility abstractions
     new 69bf1d6  [FLINK-11485] [tests] Adjust existing PojoSerializer upgrade tests
     new 52b49c8  [hotfix] [core] LinkedOptionalMap should collect keys in LinkedHashSet to guarantee ordering
     new 9c02593  [FLINK-10897] [docs] Document POJO schema evolution in /dev/stream/state/schema_evolution.md
     new cf65720  [FLINK-6768] [core] More efficient field serializer lookups in PojoSerializerSnapshot
     new 04ede2c  [hotfix] [core] Subclass serializer cache in PojoSerializer should be a Map

The 17 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 docs/dev/stream/state/schema_evolution.md          |  22 +-
 .../java/typeutils/runtime/WritableSerializer.java |  46 +-
 .../runtime/WritableSerializerMigrationTest.java   |  88 ++++
 .../resources/flink-1.6-writeable-serializer-data  | Bin 0 -> 150 bytes
 .../flink-1.6-writeable-serializer-snapshot        | Bin 0 -> 506 bytes
 .../resources/flink-1.7-writeable-serializer-data  | Bin 0 -> 150 bytes
 .../flink-1.7-writeable-serializer-snapshot        | Bin 0 -> 494 bytes
 .../typeutils/CompositeTypeSerializerSnapshot.java |  54 +--
 .../typeutils/CompositeTypeSerializerUtil.java     | 146 ++++++
 .../api/common/typeutils/TypeSerializerUtils.java  |  14 +-
 .../api/java/typeutils/runtime/PojoFieldUtils.java |  93 ++++
 .../api/java/typeutils/runtime/PojoSerializer.java | 369 ++++++----------
 .../typeutils/runtime/PojoSerializerSnapshot.java  | 488 +++++++++++++++++++++
 .../runtime/PojoSerializerSnapshotData.java        | 326 ++++++++++++++
 .../java/typeutils/runtime/ValueSerializer.java    | 106 +++--
 .../typeutils/runtime/kryo/KryoSerializer.java     |  93 ++--
 .../runtime/kryo/KryoSerializerSnapshot.java       | 166 +++++++
 .../runtime/kryo/KryoSerializerSnapshotData.java   | 379 ++++++++++++++++
 .../org/apache/flink/util/LinkedOptionalMap.java   | 309 +++++++++++++
 .../typeutils/CompositeTypeSerializerUtilTest.java | 149 +++++++
 .../api/common/typeutils/SerializerTestBase.java   |  13 +-
 .../TypeSerializerSnapshotMigrationTestBase.java   |  59 ++-
 .../PojoSerializerSnapshotMigrationTest.java       | 127 ++++++
 .../runtime/PojoSerializerSnapshotTest.java        | 385 ++++++++++++++++
 .../java/typeutils/runtime/PojoSerializerTest.java | 157 +------
 .../runtime/ValueSerializerMigrationTest.java      |  88 ++++
 .../runtime/kryo/KryoPojosForMigrationTests.java   | 154 +++++++
 .../kryo/KryoSerializerCompatibilityTest.java      |   4 +-
 .../runtime/kryo/KryoSnapshotMigrationTest.java    | 146 ++++++
 .../SchemaCompatibilityTestingSerializer.java      | 296 +++++++++++++
 .../apache/flink/util/LinkedOptionalMapTest.java   | 214 +++++++++
 .../flink-1.6-kryo-type-serializer-custom-data     |   1 +
 .../flink-1.6-kryo-type-serializer-custom-snapshot | Bin 0 -> 4808 bytes
 .../resources/flink-1.6-kryo-type-serializer-data  |   1 +
 ...link-1.6-kryo-type-serializer-empty-config-data | Bin 0 -> 180 bytes
 ...-1.6-kryo-type-serializer-empty-config-snapshot | Bin 0 -> 2759 bytes
 .../flink-1.6-kryo-type-serializer-snapshot        | Bin 0 -> 3854 bytes
 ...1.6-pojo-new-and-removed-fields-serializer-data | Bin 0 -> 320 bytes
 ...pojo-new-and-removed-fields-serializer-snapshot | Bin 0 -> 4696 bytes
 .../test/resources/flink-1.6-pojo-serializer-data  | Bin 0 -> 230 bytes
 .../resources/flink-1.6-pojo-serializer-snapshot   | Bin 0 -> 4133 bytes
 .../test/resources/flink-1.6-value-serializer-data | Bin 0 -> 150 bytes
 .../resources/flink-1.6-value-serializer-snapshot  | Bin 0 -> 1310 bytes
 .../flink-1.7-kryo-type-serializer-custom-data     |   1 +
 .../flink-1.7-kryo-type-serializer-custom-snapshot | Bin 0 -> 4796 bytes
 .../resources/flink-1.7-kryo-type-serializer-data  |   1 +
 ...link-1.7-kryo-type-serializer-empty-config-data | Bin 0 -> 892 bytes
 ...-1.7-kryo-type-serializer-empty-config-snapshot | Bin 0 -> 2747 bytes
 .../flink-1.7-kryo-type-serializer-snapshot        | Bin 0 -> 3842 bytes
 ...1.7-pojo-new-and-removed-fields-serializer-data | Bin 0 -> 320 bytes
 ...pojo-new-and-removed-fields-serializer-snapshot | Bin 0 -> 4724 bytes
 .../test/resources/flink-1.7-pojo-serializer-data  | Bin 0 -> 230 bytes
 .../resources/flink-1.7-pojo-serializer-snapshot   | Bin 0 -> 4148 bytes
 .../test/resources/flink-1.7-value-serializer-data | Bin 0 -> 150 bytes
 .../resources/flink-1.7-value-serializer-snapshot  | Bin 0 -> 1298 bytes
 .../ScalaTrySerializerConfigSnapshot.java          |  13 +
 .../typeutils/ScalaTrySerializerSnapshot.java      |  71 +++
 .../flink/api/scala/typeutils/TrySerializer.scala  |  73 ++-
 .../ScalaTrySerializerSnapshotMigrationTest.java   |  59 +++
 .../resources/flink-1.6-scala-try-serializer-data  | Bin 0 -> 1658 bytes
 .../flink-1.6-scala-try-serializer-snapshot        | Bin 0 -> 7054 bytes
 .../resources/flink-1.7-scala-try-serializer-data  | Bin 0 -> 1658 bytes
 .../flink-1.7-scala-try-serializer-snapshot        | Bin 0 -> 8932 bytes
 .../PojoSerializerUpgradeTest.java                 |  66 +--
 64 files changed, 4156 insertions(+), 621 deletions(-)
 create mode 100644 flink-connectors/flink-hadoop-compatibility/src/test/java/org/apache/flink/api/java/typeutils/runtime/WritableSerializerMigrationTest.java
 create mode 100644 flink-connectors/flink-hadoop-compatibility/src/test/resources/flink-1.6-writeable-serializer-data
 create mode 100644 flink-connectors/flink-hadoop-compatibility/src/test/resources/flink-1.6-writeable-serializer-snapshot
 create mode 100644 flink-connectors/flink-hadoop-compatibility/src/test/resources/flink-1.7-writeable-serializer-data
 create mode 100644 flink-connectors/flink-hadoop-compatibility/src/test/resources/flink-1.7-writeable-serializer-snapshot
 create mode 100644 flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoFieldUtils.java
 create mode 100644 flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerSnapshot.java
 create mode 100644 flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerSnapshotData.java
 create mode 100644 flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshot.java
 create mode 100644 flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshotData.java
 create mode 100644 flink-core/src/main/java/org/apache/flink/util/LinkedOptionalMap.java
 create mode 100644 flink-core/src/test/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerUtilTest.java
 create mode 100644 flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerSnapshotMigrationTest.java
 create mode 100644 flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerSnapshotTest.java
 create mode 100644 flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/ValueSerializerMigrationTest.java
 create mode 100644 flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoPojosForMigrationTests.java
 create mode 100644 flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSnapshotMigrationTest.java
 create mode 100644 flink-core/src/test/java/org/apache/flink/testutils/migration/SchemaCompatibilityTestingSerializer.java
 create mode 100644 flink-core/src/test/java/org/apache/flink/util/LinkedOptionalMapTest.java
 create mode 100644 flink-core/src/test/resources/flink-1.6-kryo-type-serializer-custom-data
 create mode 100644 flink-core/src/test/resources/flink-1.6-kryo-type-serializer-custom-snapshot
 create mode 100644 flink-core/src/test/resources/flink-1.6-kryo-type-serializer-data
 create mode 100644 flink-core/src/test/resources/flink-1.6-kryo-type-serializer-empty-config-data
 create mode 100644 flink-core/src/test/resources/flink-1.6-kryo-type-serializer-empty-config-snapshot
 create mode 100644 flink-core/src/test/resources/flink-1.6-kryo-type-serializer-snapshot
 create mode 100644 flink-core/src/test/resources/flink-1.6-pojo-new-and-removed-fields-serializer-data
 create mode 100644 flink-core/src/test/resources/flink-1.6-pojo-new-and-removed-fields-serializer-snapshot
 create mode 100644 flink-core/src/test/resources/flink-1.6-pojo-serializer-data
 create mode 100644 flink-core/src/test/resources/flink-1.6-pojo-serializer-snapshot
 create mode 100644 flink-core/src/test/resources/flink-1.6-value-serializer-data
 create mode 100644 flink-core/src/test/resources/flink-1.6-value-serializer-snapshot
 create mode 100644 flink-core/src/test/resources/flink-1.7-kryo-type-serializer-custom-data
 create mode 100644 flink-core/src/test/resources/flink-1.7-kryo-type-serializer-custom-snapshot
 create mode 100644 flink-core/src/test/resources/flink-1.7-kryo-type-serializer-data
 create mode 100644 flink-core/src/test/resources/flink-1.7-kryo-type-serializer-empty-config-data
 create mode 100644 flink-core/src/test/resources/flink-1.7-kryo-type-serializer-empty-config-snapshot
 create mode 100644 flink-core/src/test/resources/flink-1.7-kryo-type-serializer-snapshot
 create mode 100644 flink-core/src/test/resources/flink-1.7-pojo-new-and-removed-fields-serializer-data
 create mode 100644 flink-core/src/test/resources/flink-1.7-pojo-new-and-removed-fields-serializer-snapshot
 create mode 100644 flink-core/src/test/resources/flink-1.7-pojo-serializer-data
 create mode 100644 flink-core/src/test/resources/flink-1.7-pojo-serializer-snapshot
 create mode 100644 flink-core/src/test/resources/flink-1.7-value-serializer-data
 create mode 100644 flink-core/src/test/resources/flink-1.7-value-serializer-snapshot
 create mode 100644 flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaTrySerializerSnapshot.java
 create mode 100644 flink-scala/src/test/java/org/apache/flink/api/scala/typeutils/ScalaTrySerializerSnapshotMigrationTest.java
 create mode 100644 flink-scala/src/test/resources/flink-1.6-scala-try-serializer-data
 create mode 100644 flink-scala/src/test/resources/flink-1.6-scala-try-serializer-snapshot
 create mode 100644 flink-scala/src/test/resources/flink-1.7-scala-try-serializer-data
 create mode 100644 flink-scala/src/test/resources/flink-1.7-scala-try-serializer-snapshot


[flink] 01/17: [FLINK-11323] Add OptionalMap to be used by Kryo

Posted by tz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tzulitai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 633ed9552e193c20a5260f2ff7465a22e32d3c23
Author: Igal Shilman <ig...@data-artisans.com>
AuthorDate: Tue Jan 15 12:02:30 2019 +0100

    [FLINK-11323] Add OptionalMap to be used by Kryo
    
    An OptinalMap is a map that allows null values and null keys (as long
    as a name is assosicated with them. This would replace the usage of
    DummyUnloadableClass and DummyUnloadableSerializer within
    KryoSerializerSnapshot.
---
 .../java/typeutils/runtime/kryo/OptionalMap.java   | 249 +++++++++++++++++++++
 .../typeutils/runtime/kryo/OptionalMapTest.java    | 215 ++++++++++++++++++
 2 files changed, 464 insertions(+)

diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/OptionalMap.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/OptionalMap.java
new file mode 100644
index 0000000..afabb4b
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/OptionalMap.java
@@ -0,0 +1,249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.typeutils.runtime.kryo;
+
+import org.apache.flink.annotation.VisibleForTesting;
+
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.Map.Entry;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import javax.annotation.Nullable;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * An OptionalMap is an order preserving map (like {@link LinkedHashMap}) where keys have a unique string name, but are
+ * optionally present, and the values are optional.
+ */
+final class OptionalMap<K, V> {
+
+
+	// --------------------------------------------------------------------------------------------------------
+	// Factory
+	// --------------------------------------------------------------------------------------------------------
+
+	/**
+	 * Creates an {@code OptionalMap} from the provided map.
+	 *
+	 * <p>This method is the equivalent of {@link Optional#of(Object)} but for maps. To support more than one {@code NULL}
+	 * key, an optional map requires a unique string name to be associated with each key (provided by keyNameGetter)
+	 *
+	 * @param sourceMap     a source map to wrap as an optional map.
+	 * @param keyNameGetter function that assigns a unique name to the keys of the source map.
+	 * @param <K>           key type
+	 * @param <V>           value type
+	 * @return an {@code OptionalMap} with optional named keys, and optional values.
+	 */
+	static <K, V> OptionalMap<K, V> optionalMapOf(LinkedHashMap<K, V> sourceMap, Function<K, String> keyNameGetter) {
+
+		LinkedHashMap<String, KeyValue<K, V>> underlyingMap = new LinkedHashMap<>(sourceMap.size());
+
+		sourceMap.forEach((k, v) -> {
+			String keyName = keyNameGetter.apply(k);
+			underlyingMap.put(keyName, new KeyValue<>(k, v));
+		});
+
+		return new OptionalMap<>(underlyingMap);
+	}
+
+	/**
+	 * Tries to merges the keys and the values of @right into @left.
+	 */
+	static <K, V> MergeResult<K, V> mergeRightIntoLeft(OptionalMap<K, V> left, OptionalMap<K, V> right) {
+		OptionalMap<K, V> merged = new OptionalMap<>(left);
+		merged.putAll(right);
+
+		return new MergeResult<>(merged, isLeftPrefixOfRight(left, right));
+	}
+
+	// --------------------------------------------------------------------------------------------------------
+	// Constructor
+	// --------------------------------------------------------------------------------------------------------
+
+	private final LinkedHashMap<String, KeyValue<K, V>> underlyingMap;
+
+	OptionalMap() {
+		this(new LinkedHashMap<>());
+	}
+
+	@SuppressWarnings("CopyConstructorMissesField")
+	OptionalMap(OptionalMap<K, V> optionalMap) {
+		this(new LinkedHashMap<>(optionalMap.underlyingMap));
+	}
+
+	private OptionalMap(LinkedHashMap<String, KeyValue<K, V>> underlyingMap) {
+		this.underlyingMap = checkNotNull(underlyingMap);
+	}
+
+	// --------------------------------------------------------------------------------------------------------
+	// API
+	// --------------------------------------------------------------------------------------------------------
+
+	int size() {
+		return underlyingMap.size();
+	}
+
+	void put(String keyName, @Nullable K key, @Nullable V value) {
+		checkNotNull(keyName);
+
+		underlyingMap.compute(keyName, (unused, kv) ->
+			(kv == null) ? new KeyValue<>(key, value) : kv.merge(key, value));
+	}
+
+	void putAll(OptionalMap<K, V> right) {
+		for (Entry<String, KeyValue<K, V>> entry : right.underlyingMap.entrySet()) {
+			KeyValue<K, V> kv = entry.getValue();
+			this.put(entry.getKey(), kv.key, kv.value);
+		}
+	}
+
+	/**
+	 * Returns the key names of any keys or values that are absent.
+	 */
+	Set<String> absentKeysOrValues() {
+		return underlyingMap.entrySet()
+			.stream()
+			.filter(OptionalMap::keyOrValueIsAbsent)
+			.map(Entry::getKey)
+			.collect(Collectors.toSet());
+	}
+
+	/**
+	 * Assuming all the entries of this map are present (keys and values) this method would return
+	 * a map with these key and values, stripped from their Optional wrappers.
+	 * NOTE: please note that if any of the key or values are absent this method would throw an {@link IllegalStateException}.
+	 */
+	LinkedHashMap<K, V> unwrapOptionals() {
+		final LinkedHashMap<K, V> unwrapped = new LinkedHashMap<>(underlyingMap.size());
+
+		for (Entry<String, KeyValue<K, V>> entry : underlyingMap.entrySet()) {
+			String namedKey = entry.getKey();
+			KeyValue<K, V> kv = entry.getValue();
+			if (kv.key == null) {
+				throw new IllegalStateException("Missing key '" + namedKey + "'");
+			}
+			if (kv.value == null) {
+				throw new IllegalStateException("Missing value for the key '" + namedKey + "'");
+			}
+			unwrapped.put(kv.key, kv.value);
+		}
+
+		return unwrapped;
+	}
+
+	/**
+	 * Returns the key names added to this map.
+	 */
+	Set<String> keyNames() {
+		return underlyingMap.keySet();
+	}
+
+	// --------------------------------------------------------------------------------------------------------
+	// Static Utility Methods
+	// --------------------------------------------------------------------------------------------------------
+
+	private static <K, V> boolean keyOrValueIsAbsent(Entry<String, KeyValue<K, V>> entry) {
+		KeyValue<K, V> kv = entry.getValue();
+		return kv.key == null || kv.value == null;
+	}
+
+	@VisibleForTesting
+	static <K, V> boolean isLeftPrefixOfRight(OptionalMap<K, V> left, OptionalMap<K, V> right) {
+		Iterator<String> rightKeys = right.keyNames().iterator();
+
+		for (String leftKey : left.keyNames()) {
+			if (!rightKeys.hasNext()) {
+				return false;
+			}
+			String rightKey = rightKeys.next();
+			if (!leftKey.equals(rightKey)) {
+				return false;
+			}
+		}
+		return true;
+	}
+
+	// --------------------------------------------------------------------------------------------------------
+	// Inner Classes
+	// --------------------------------------------------------------------------------------------------------
+
+	private static final class KeyValue<K, V> {
+		K key;
+		V value;
+
+		KeyValue(K key, V value) {
+			this.key = key;
+			this.value = value;
+		}
+
+		KeyValue<K, V> merge(K key, V value) {
+			this.key = firstNonNull(key, this.key);
+			this.value = firstNonNull(value, this.value);
+			return this;
+		}
+
+		private static <T> T firstNonNull(T first, T second) {
+			if (first != null) {
+				return first;
+			}
+			return second;
+		}
+	}
+
+	// --------------------------------------------------------------------------------------------------------
+	// Merge
+	// --------------------------------------------------------------------------------------------------------
+
+	static final class MergeResult<K, V> {
+		private final OptionalMap<K, V> merged;
+		private final Set<String> missingKeys;
+		private final boolean isOrderedSubset;
+
+		MergeResult(OptionalMap<K, V> merged, boolean isOrderedSubset) {
+			this.merged = merged;
+			this.missingKeys = merged.absentKeysOrValues();
+			this.isOrderedSubset = isOrderedSubset;
+		}
+
+		boolean hasMissingKeys() {
+			return !missingKeys.isEmpty();
+		}
+
+		Set<String> missingKeys() {
+			return missingKeys;
+		}
+
+		LinkedHashMap<K, V> getMerged() {
+			return merged.unwrapOptionals();
+		}
+
+		/**
+		 * Returns {@code true} if keyNames present at @left, appearing in prefix order at @right.
+		 */
+		boolean isOrderedSubset() {
+			return isOrderedSubset;
+		}
+	}
+
+}
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/OptionalMapTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/OptionalMapTest.java
new file mode 100644
index 0000000..ff7f31f
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/OptionalMapTest.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.typeutils.runtime.kryo;
+
+import org.apache.flink.api.java.typeutils.runtime.kryo.OptionalMap.MergeResult;
+
+import org.junit.Test;
+
+import java.util.LinkedHashMap;
+
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.hasItems;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test {@link OptionalMap}.
+ */
+public class OptionalMapTest {
+
+	@Test
+	public void usageExample() {
+		OptionalMap<Class<?>, String> map = new OptionalMap<>();
+
+		map.put("java.lang.String", String.class, "a string class");
+		map.put("scala.Option", null, "a scala Option");
+		map.put("java.lang.Boolean", Boolean.class, null);
+
+		assertThat(map.keyNames(), hasItems("java.lang.String", "scala.Option"));
+		assertThat(map.absentKeysOrValues(), hasItems("scala.Option", "java.lang.Boolean"));
+	}
+
+	@Test
+	public void overridingKeyWithTheSameKeyName() {
+		OptionalMap<Class<?>, String> map = new OptionalMap<>();
+
+		map.put("java.lang.String", null, "a string class");
+		map.put("java.lang.String", String.class, "a string class");
+
+		assertThat(map.absentKeysOrValues(), is(empty()));
+	}
+
+	@Test
+	public void overridingKeysAndValuesWithTheSameKeyName() {
+		OptionalMap<Class<?>, String> map = new OptionalMap<>();
+
+		map.put("java.lang.String", null, null);
+		map.put("java.lang.String", String.class, "a string class");
+
+		assertThat(map.absentKeysOrValues(), is(empty()));
+	}
+
+	@Test
+	public void overridingAValueWithMissingKeyShouldBeConsideredAsAbsent() {
+		OptionalMap<Class<?>, String> map = new OptionalMap<>();
+
+		map.put("java.lang.String", null, null);
+		map.put("java.lang.String", null, "a string class");
+
+		assertThat(map.absentKeysOrValues(), hasItem("java.lang.String"));
+	}
+
+	@Test
+	public void mergingMapsWithPresentEntriesLeavesNoAbsentKeyNames() {
+		OptionalMap<Class<?>, String> first = new OptionalMap<>();
+		first.put("b", null, null);
+		first.put("c", String.class, null);
+
+		OptionalMap<Class<?>, String> second = new OptionalMap<>();
+		second.put("a", String.class, "aaa");
+		second.put("b", String.class, "bbb");
+		second.put("c", Void.class, "ccc");
+		second.put("d", String.class, "ddd");
+
+		first.putAll(second);
+
+		assertThat(first.absentKeysOrValues(), is(empty()));
+	}
+
+	@Test
+	public void mergingMapsPreserversTheOrderOfTheOriginalMap() {
+		OptionalMap<Class<?>, String> first = new OptionalMap<>();
+		first.put("b", null, null);
+		first.put("c", String.class, null);
+
+		OptionalMap<Class<?>, String> second = new OptionalMap<>();
+		second.put("a", String.class, "aaa");
+		second.put("b", String.class, "bbb");
+		second.put("c", Void.class, "ccc");
+		second.put("d", String.class, "ddd");
+
+		first.putAll(second);
+
+		assertThat(first.keyNames(), contains("b", "c", "a", "d"));
+	}
+
+	@Test
+	public void mergingToEmpty() {
+		OptionalMap<Class<?>, String> first = new OptionalMap<>();
+
+		OptionalMap<Class<?>, String> second = new OptionalMap<>();
+		second.put("a", String.class, "aaa");
+		second.put("b", String.class, "bbb");
+		second.put("c", Void.class, "ccc");
+		second.put("d", String.class, "ddd");
+
+		first.putAll(second);
+
+		assertThat(first.keyNames(), contains("a", "b", "c", "d"));
+	}
+
+	@Test(expected = IllegalStateException.class)
+	public void unwrapOptionalsWithMissingValueThrows() {
+		OptionalMap<Class<?>, String> map = new OptionalMap<>();
+
+		map.put("a", String.class, null);
+
+		map.unwrapOptionals();
+	}
+
+	@Test(expected = IllegalStateException.class)
+	public void unwrapOptionalsWithMissingKeyThrows() {
+		OptionalMap<Class<?>, String> map = new OptionalMap<>();
+
+		map.put("a", null, "blabla");
+
+		map.unwrapOptionals();
+	}
+
+	@Test
+	public void unwrapOptionalsPreservesOrder() {
+		OptionalMap<Class<?>, String> map = new OptionalMap<>();
+
+		map.put("a", String.class, "aaa");
+		map.put("b", Boolean.class, "bbb");
+
+		LinkedHashMap<Class<?>, String> m = map.unwrapOptionals();
+
+		assertThat(m.keySet(), contains(String.class, Boolean.class));
+		assertThat(m.values(), contains("aaa", "bbb"));
+	}
+
+	@Test
+	public void testPrefix() {
+		OptionalMap<Class<?>, String> left = new OptionalMap<>();
+
+		left.put("a", String.class, "aaa");
+		left.put("b", String.class, "aaa");
+
+		OptionalMap<Class<?>, String> right = new OptionalMap<>(left);
+
+		right.put("c", Boolean.class, "bbb");
+
+		assertTrue(OptionalMap.isLeftPrefixOfRight(left, right));
+	}
+
+	@Test
+	public void testNonPrefix() {
+		OptionalMap<Class<?>, String> left = new OptionalMap<>();
+
+		left.put("a", String.class, "aaa");
+		left.put("c", String.class, "aaa");
+
+		OptionalMap<Class<?>, String> right = new OptionalMap<>();
+
+		right.put("b", Boolean.class, "bbb");
+		right.put("c", Boolean.class, "bbb");
+
+
+		assertFalse(OptionalMap.isLeftPrefixOfRight(left, right));
+	}
+
+	@Test
+	@SuppressWarnings("unchecked")
+	public void demoMergeResult() {
+		OptionalMap<Class<?>, String> left = new OptionalMap<>();
+		left.put("b", null, null);
+		left.put("c", String.class, null);
+
+		OptionalMap<Class<?>, String> right = new OptionalMap<>();
+		right.put("b", String.class, "bbb");
+		right.put("c", Void.class, "ccc");
+		right.put("a", Boolean.class, "aaa");
+		right.put("d", Long.class, "ddd");
+
+		MergeResult<Class<?>, String> result = OptionalMap.mergeRightIntoLeft(left, right);
+
+		assertThat(result.hasMissingKeys(), is(false));
+		assertThat(result.isOrderedSubset(), is(true));
+		assertThat(result.missingKeys(), is(empty()));
+
+		LinkedHashMap<Class<?>, String> merged = result.getMerged();
+		assertThat(merged.keySet(), contains(String.class, Void.class, Boolean.class, Long.class));
+	}
+}


[flink] 02/17: [FLINK-11323] [tests] Adjust serializer tests to respect new COMPATIBLE_WITH_RECONFIGURED_SERIALIZER option

Posted by tz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tzulitai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 6685d6a14c71b5bbb8be8e53245c828ce0e284f8
Author: Igal Shilman <ig...@data-artisans.com>
AuthorDate: Tue Jan 15 12:14:52 2019 +0100

    [FLINK-11323] [tests] Adjust serializer tests to respect new COMPATIBLE_WITH_RECONFIGURED_SERIALIZER option
    
    Adjust serializer tests to use the reconfigured serializer if such
    serializer was provided.
---
 .../flink/api/common/typeutils/SerializerTestBase.java      | 13 ++++++++++---
 .../runtime/kryo/KryoSerializerCompatibilityTest.java       |  4 +++-
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java
index 151dafb..d7d7d80 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java
@@ -129,9 +129,16 @@ public abstract class SerializerTestBase<T> extends TestLogger {
 		}
 
 		TypeSerializerSchemaCompatibility<T> strategy = restoredConfig.resolveSchemaCompatibility(getSerializer());
-		assertTrue(strategy.isCompatibleAsIs());
-
-		TypeSerializer<T> restoreSerializer = restoredConfig.restoreSerializer();
+		final TypeSerializer<T> restoreSerializer;
+		if (strategy.isCompatibleAsIs()) {
+			restoreSerializer = restoredConfig.restoreSerializer();
+		}
+		else if (strategy.isCompatibleWithReconfiguredSerializer()) {
+			restoreSerializer = strategy.getReconfiguredSerializer();
+		}
+		else {
+			throw new AssertionError("Unable to restore serializer with " + strategy);
+		}
 		assertEquals(serializer.getClass(), restoreSerializer.getClass());
 	}
 
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerCompatibilityTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerCompatibilityTest.java
index 477843c..bc66294 100644
--- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerCompatibilityTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerCompatibilityTest.java
@@ -279,7 +279,9 @@ public class KryoSerializerCompatibilityTest {
 		@SuppressWarnings("unchecked")
 		TypeSerializerSchemaCompatibility<TestClass> compatResult =
 			kryoSerializerConfigSnapshot.resolveSchemaCompatibility(kryoSerializer);
-		assertTrue(compatResult.isCompatibleAsIs());
+		assertTrue(compatResult.isCompatibleWithReconfiguredSerializer());
+
+		kryoSerializer = (KryoSerializer<TestClass>) compatResult.getReconfiguredSerializer();
 		assertEquals(testClassId, kryoSerializer.getKryo().getRegistration(TestClass.class).getId());
 		assertEquals(testClassAId, kryoSerializer.getKryo().getRegistration(TestClassA.class).getId());
 		assertEquals(testClassBId, kryoSerializer.getKryo().getRegistration(TestClassB.class).getId());


[flink] 17/17: [hotfix] [core] Subclass serializer cache in PojoSerializer should be a Map

Posted by tz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tzulitai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 04ede2c66cb030a162436a1f6b1317fa4bdf34df
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
AuthorDate: Thu Feb 21 11:22:26 2019 +0800

    [hotfix] [core] Subclass serializer cache in PojoSerializer should be a Map
    
    In this case, there is no specific need to let that cache be
    specifically a HashMap. Changing this to the generic Map does not break
    contracts with how the PojoSerializer works. It also doesn't break
    compatibility w.r.t. Java serialization of the PojoSerializer.
---
 .../apache/flink/api/java/typeutils/runtime/PojoSerializer.java   | 8 ++++----
 .../flink/api/java/typeutils/runtime/PojoSerializerSnapshot.java  | 2 +-
 .../api/java/typeutils/runtime/PojoSerializerSnapshotData.java    | 5 +++--
 .../src/main/java/org/apache/flink/util/LinkedOptionalMap.java    | 4 ++--
 4 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java
index fbaedc0..dcc0a72 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java
@@ -92,7 +92,7 @@ public final class PojoSerializer<T> extends TypeSerializer<T> {
 	/**
 	 * Cache of non-registered subclasses to their serializers, created on-the-fly.
 	 */
-	private transient HashMap<Class<?>, TypeSerializer<?>> subclassSerializerCache;
+	private transient Map<Class<?>, TypeSerializer<?>> subclassSerializerCache;
 
 	// --------------------------------------------------------------------------------------------
 
@@ -145,7 +145,7 @@ public final class PojoSerializer<T> extends TypeSerializer<T> {
 			TypeSerializer<Object>[] fieldSerializers,
 			LinkedHashMap<Class<?>, Integer> registeredClasses,
 			TypeSerializer<?>[] registeredSerializers,
-			HashMap<Class<?>, TypeSerializer<?>> subclassSerializerCache,
+			Map<Class<?>, TypeSerializer<?>> subclassSerializerCache,
 			ExecutionConfig executionConfig) {
 
 		this.clazz = checkNotNull(clazz);
@@ -987,7 +987,7 @@ public final class PojoSerializer<T> extends TypeSerializer<T> {
 		return result;
 	}
 
-	HashMap<Class<?>, TypeSerializer<?>> getSubclassSerializerCache() {
+	Map<Class<?>, TypeSerializer<?>> getSubclassSerializerCache() {
 		return subclassSerializerCache;
 	}
 
@@ -1110,7 +1110,7 @@ public final class PojoSerializer<T> extends TypeSerializer<T> {
 			TypeSerializer<?>[] registeredSubclassSerializers,
 			Field[] fields,
 			TypeSerializer<?>[] fieldSerializers,
-			HashMap<Class<?>, TypeSerializer<?>> nonRegisteredSubclassSerializerCache) {
+			Map<Class<?>, TypeSerializer<?>> nonRegisteredSubclassSerializerCache) {
 
 		final LinkedHashMap<Class<?>, TypeSerializer<?>> subclassRegistry = new LinkedHashMap<>(registeredSubclassesToTags.size());
 
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerSnapshot.java
index 7130134..5610536 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerSnapshot.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerSnapshot.java
@@ -79,7 +79,7 @@ public class PojoSerializerSnapshot<T> implements TypeSerializerSnapshot<T> {
 			Field[] fields,
 			TypeSerializer<?>[] fieldSerializers,
 			LinkedHashMap<Class<?>, TypeSerializer<?>> registeredSubclassSerializers,
-			HashMap<Class<?>, TypeSerializer<?>> nonRegisteredSubclassSerializers) {
+			Map<Class<?>, TypeSerializer<?>> nonRegisteredSubclassSerializers) {
 
 		this.snapshotData = PojoSerializerSnapshotData.createFrom(
 			pojoClass,
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerSnapshotData.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerSnapshotData.java
index e2efbdb..8999d77 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerSnapshotData.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerSnapshotData.java
@@ -37,6 +37,7 @@ import java.io.IOException;
 import java.lang.reflect.Field;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
+import java.util.Map;
 
 import static org.apache.flink.util.LinkedOptionalMap.optionalMapOf;
 import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -86,7 +87,7 @@ final class PojoSerializerSnapshotData<T> {
 			Field[] fields,
 			TypeSerializer<?>[] fieldSerializers,
 			LinkedHashMap<Class<?>, TypeSerializer<?>> registeredSubclassSerializers,
-			HashMap<Class<?>, TypeSerializer<?>> nonRegisteredSubclassSerializers) {
+			Map<Class<?>, TypeSerializer<?>> nonRegisteredSubclassSerializers) {
 
 		final LinkedOptionalMap<Field, TypeSerializerSnapshot<?>> fieldSerializerSnapshots = new LinkedOptionalMap<>(fields.length);
 
@@ -99,7 +100,7 @@ final class PojoSerializerSnapshotData<T> {
 		LinkedHashMap<Class<?>, TypeSerializerSnapshot<?>> registeredSubclassSerializerSnapshots = new LinkedHashMap<>(registeredSubclassSerializers.size());
 		registeredSubclassSerializers.forEach((k, v) -> registeredSubclassSerializerSnapshots.put(k, TypeSerializerUtils.snapshotBackwardsCompatible(v)));
 
-		HashMap<Class<?>, TypeSerializerSnapshot<?>> nonRegisteredSubclassSerializerSnapshots = new HashMap<>(nonRegisteredSubclassSerializers.size());
+		Map<Class<?>, TypeSerializerSnapshot<?>> nonRegisteredSubclassSerializerSnapshots = new HashMap<>(nonRegisteredSubclassSerializers.size());
 		nonRegisteredSubclassSerializers.forEach((k, v) -> nonRegisteredSubclassSerializerSnapshots.put(k, TypeSerializerUtils.snapshotBackwardsCompatible(v)));
 
 		return new PojoSerializerSnapshotData<>(
diff --git a/flink-core/src/main/java/org/apache/flink/util/LinkedOptionalMap.java b/flink-core/src/main/java/org/apache/flink/util/LinkedOptionalMap.java
index a17dca4..db4318c 100644
--- a/flink-core/src/main/java/org/apache/flink/util/LinkedOptionalMap.java
+++ b/flink-core/src/main/java/org/apache/flink/util/LinkedOptionalMap.java
@@ -24,10 +24,10 @@ import org.apache.flink.annotation.VisibleForTesting;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
-import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
+import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Optional;
 import java.util.Set;
@@ -59,7 +59,7 @@ public final class LinkedOptionalMap<K, V> {
 	 * @param <V>           value type
 	 * @return an {@code LinkedOptionalMap} with optional named keys, and optional values.
 	 */
-	public static <K, V> LinkedOptionalMap<K, V> optionalMapOf(HashMap<K, V> sourceMap, Function<K, String> keyNameGetter) {
+	public static <K, V> LinkedOptionalMap<K, V> optionalMapOf(Map<K, V> sourceMap, Function<K, String> keyNameGetter) {
 
 		LinkedHashMap<String, KeyValue<K, V>> underlyingMap = new LinkedHashMap<>(sourceMap.size());
 


[flink] 11/17: [FLINK-11485] [core] Implement new PojoSerializerSnapshot class

Posted by tz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tzulitai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 7621c57eb199f7225f5d9a06aee52914f6896780
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
AuthorDate: Tue Feb 19 14:33:36 2019 +0800

    [FLINK-11485] [core] Implement new PojoSerializerSnapshot class
---
 .../api/common/typeutils/TypeSerializerUtils.java  |  14 +-
 .../api/java/typeutils/runtime/PojoFieldUtils.java |  93 ++++
 .../api/java/typeutils/runtime/PojoSerializer.java |  94 +++--
 .../typeutils/runtime/PojoSerializerSnapshot.java  | 468 +++++++++++++++++++++
 .../runtime/PojoSerializerSnapshotData.java        | 325 ++++++++++++++
 .../org/apache/flink/util/LinkedOptionalMap.java   |  65 ++-
 .../runtime/PojoSerializerSnapshotTest.java        | 385 +++++++++++++++++
 7 files changed, 1402 insertions(+), 42 deletions(-)

diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerUtils.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerUtils.java
index 355edd4..c167231 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerUtils.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerUtils.java
@@ -26,7 +26,7 @@ import java.util.Arrays;
 public final class TypeSerializerUtils {
 
 	/**
-	 * Takes config snapshots of the given serializers. In case where the config snapshots
+	 * Takes snapshots of the given serializers. In case where the snapshots
 	 * are still extending the old {@code TypeSerializerConfigSnapshot} class, the snapshots
 	 * are set up properly (with their originating serializer) such that the backwards
 	 * compatible code paths work.
@@ -35,11 +35,21 @@ public final class TypeSerializerUtils {
 			TypeSerializer<?>... originatingSerializers) {
 
 		return Arrays.stream(originatingSerializers)
-				.map((s) -> configureForBackwardsCompatibility(s.snapshotConfiguration(), s))
+				.map(TypeSerializerUtils::snapshotBackwardsCompatible)
 				.toArray(TypeSerializerSnapshot[]::new);
 	}
 
 	/**
+	 * Takes a snapshot of the given serializer. In case where the snapshot
+	 * is still extending the old {@code TypeSerializerConfigSnapshot} class, the snapshot
+	 * is set up properly (with its originating serializer) such that the backwards
+	 * compatible code paths work.
+	 */
+	public static TypeSerializerSnapshot<?> snapshotBackwardsCompatible(TypeSerializer<?> originatingSerializer) {
+		return configureForBackwardsCompatibility(originatingSerializer.snapshotConfiguration(), originatingSerializer);
+	}
+
+	/**
 	 * Utility method to bind the serializer and serializer snapshot to a common
 	 * generic type variable.
 	 */
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoFieldUtils.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoFieldUtils.java
new file mode 100644
index 0000000..6d78919
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoFieldUtils.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.typeutils.runtime;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.util.InstantiationUtil;
+
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.lang.reflect.Field;
+
+/**
+ * Utility class for reading, writing, and finding POJO fields.
+ */
+@Internal
+final class PojoFieldUtils {
+
+	/**
+	 * Writes a field to the given {@link DataOutputView}.
+	 *
+	 * <p>This write method avoids Java serialization, by writing only the classname of the field's declaring class
+	 * and the field name. The written field can be read using {@link #readField(DataInputView, ClassLoader)}.
+	 *
+	 * @param out the output view to write to.
+	 * @param field the field to write.
+	 */
+	static void writeField(DataOutputView out, Field field) throws IOException {
+		Class<?> declaringClass = field.getDeclaringClass();
+		out.writeUTF(declaringClass.getName());
+		out.writeUTF(field.getName());
+	}
+
+	/**
+	 * Reads a field from the given {@link DataInputView}.
+	 *
+	 * <p>This read methods avoids Java serialization, by reading the classname of the field's declaring class
+	 * and dynamically loading it. The field is also read by field name and obtained via reflection.
+	 *
+	 * @param in the input view to read from.
+	 * @param userCodeClassLoader the user classloader.
+	 *
+	 * @return the read field.
+	 */
+	static Field readField(DataInputView in, ClassLoader userCodeClassLoader) throws IOException {
+		Class<?> declaringClass = InstantiationUtil.resolveClassByName(in, userCodeClassLoader);
+		String fieldName = in.readUTF();
+		return getField(fieldName, declaringClass);
+	}
+
+	/**
+	 * Finds a field by name from its declaring class. This also searches for the
+	 * field in super classes.
+	 *
+	 * @param fieldName the name of the field to find.
+	 * @param declaringClass the declaring class of the field.
+	 *
+	 * @return the field.
+	 */
+	@Nullable
+	static Field getField(String fieldName, Class<?> declaringClass) {
+		Class<?> clazz = declaringClass;
+
+		while (clazz != null) {
+			try {
+				Field field = clazz.getDeclaredField(fieldName);
+				field.setAccessible(true);
+				return field;
+			} catch (NoSuchFieldException e) {
+				clazz = clazz.getSuperclass();
+			}
+		}
+
+		return null;
+	}
+}
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java
index 882f7bd..8abe0d9 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java
@@ -19,7 +19,6 @@
 package org.apache.flink.api.java.typeutils.runtime;
 
 import org.apache.flink.annotation.Internal;
-import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.typeutils.CompatibilityResult;
 import org.apache.flink.api.common.typeutils.CompatibilityUtil;
@@ -115,6 +114,9 @@ public final class PojoSerializer<T> extends TypeSerializer<T> {
 
 	private transient ClassLoader cl;
 
+	/**
+	 * Constructor to create a new {@link PojoSerializer}.
+	 */
 	@SuppressWarnings("unchecked")
 	public PojoSerializer(
 			Class<T> clazz,
@@ -144,13 +146,18 @@ public final class PojoSerializer<T> extends TypeSerializer<T> {
 		this.subclassSerializerCache = new HashMap<>();
 	}
 
-	public PojoSerializer(
+	/**
+	 * Constructor to create a restore serializer or a reconfigured serializer
+	 * from a {@link PojoSerializerSnapshot}.
+	 */
+	PojoSerializer(
 			Class<T> clazz,
 			Field[] fields,
 			TypeSerializer<Object>[] fieldSerializers,
 			LinkedHashMap<Class<?>, Integer> registeredClasses,
 			TypeSerializer<?>[] registeredSerializers,
-			HashMap<Class<?>, TypeSerializer<?>> subclassSerializerCache) {
+			HashMap<Class<?>, TypeSerializer<?>> subclassSerializerCache,
+			ExecutionConfig executionConfig) {
 
 		this.clazz = checkNotNull(clazz);
 		this.fields = checkNotNull(fields);
@@ -159,8 +166,7 @@ public final class PojoSerializer<T> extends TypeSerializer<T> {
 		this.registeredClasses = checkNotNull(registeredClasses);
 		this.registeredSerializers = checkNotNull(registeredSerializers);
 		this.subclassSerializerCache = checkNotNull(subclassSerializerCache);
-
-		this.executionConfig = null;
+		this.executionConfig = checkNotNull(executionConfig);
 	}
 	
 	@Override
@@ -737,7 +743,8 @@ public final class PojoSerializer<T> extends TypeSerializer<T> {
 								reorderedFieldSerializers,
 								reorderedRegisteredSubclassesToClasstags,
 								reorderedRegisteredSubclassSerializers,
-								rebuiltCache));
+								rebuiltCache,
+								null));
 					}
 				}
 			}
@@ -1054,6 +1061,52 @@ public final class PojoSerializer<T> extends TypeSerializer<T> {
 	}
 
 	// --------------------------------------------------------------------------------------------
+	// Configuration access
+	// --------------------------------------------------------------------------------------------
+
+	Class<T> getPojoClass() {
+		return clazz;
+	}
+
+	Field[] getFields() {
+		return fields;
+	}
+
+	TypeSerializer<?>[] getFieldSerializers() {
+		return fieldSerializers;
+	}
+
+	TypeSerializer<?> getFieldSerializer(Field targetField) {
+		int fieldIndex = findField(targetField.getName());
+		if (fieldIndex == -1) {
+			return null;
+		}
+		return fieldSerializers[fieldIndex];
+	}
+
+	ExecutionConfig getExecutionConfig() {
+		return executionConfig;
+	}
+
+	LinkedHashMap<Class<?>, Integer> getRegisteredClasses() {
+		return registeredClasses;
+	}
+
+	TypeSerializer<?>[] getRegisteredSerializers() {
+		return registeredSerializers;
+	}
+
+	LinkedHashMap<Class<?>, TypeSerializer<?>> getBundledSubclassSerializerRegistry() {
+		final LinkedHashMap<Class<?>, TypeSerializer<?>> result = new LinkedHashMap<>(registeredClasses.size());
+		registeredClasses.forEach((registeredClass, id) -> result.put(registeredClass, registeredSerializers[id]));
+		return result;
+	}
+
+	HashMap<Class<?>, TypeSerializer<?>> getSubclassSerializerCache() {
+		return subclassSerializerCache;
+	}
+
+	// --------------------------------------------------------------------------------------------
 	// Utilities
 	// --------------------------------------------------------------------------------------------
 
@@ -1209,33 +1262,4 @@ public final class PojoSerializer<T> extends TypeSerializer<T> {
 				registeredSubclassesToSerializerConfigSnapshots,
 				nonRegisteredSubclassesToSerializerConfigSnapshots);
 	}
-
-	// --------------------------------------------------------------------------------------------
-	// Test utilities
-	// --------------------------------------------------------------------------------------------
-
-	@VisibleForTesting
-	Field[] getFields() {
-		return fields;
-	}
-
-	@VisibleForTesting
-	TypeSerializer<?>[] getFieldSerializers() {
-		return fieldSerializers;
-	}
-
-	@VisibleForTesting
-	LinkedHashMap<Class<?>, Integer> getRegisteredClasses() {
-		return registeredClasses;
-	}
-
-	@VisibleForTesting
-	TypeSerializer<?>[] getRegisteredSerializers() {
-		return registeredSerializers;
-	}
-
-	@VisibleForTesting
-	HashMap<Class<?>, TypeSerializer<?>> getSubclassSerializerCache() {
-		return subclassSerializerCache;
-	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerSnapshot.java
new file mode 100644
index 0000000..5843168
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerSnapshot.java
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.typeutils.runtime;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.typeutils.CompositeTypeSerializerUtil;
+import org.apache.flink.api.common.typeutils.CompositeTypeSerializerUtil.IntermediateCompatibilityResult;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
+import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.util.LinkedOptionalMap;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.flink.util.Preconditions.checkArgument;
+import static org.apache.flink.util.Preconditions.checkState;
+
+/**
+ * Snapshot class for the {@link PojoSerializer}.
+ */
+@Internal
+public class PojoSerializerSnapshot<T> implements TypeSerializerSnapshot<T> {
+
+	/**
+	 * We start from version {@code 2}. {@code 1} is retained for {@link PojoSerializer.PojoSerializerConfigSnapshot}.
+	 */
+	private static final int VERSION = 2;
+
+	/**
+	 * Contains the actual content for the serializer snapshot.
+	 */
+	private PojoSerializerSnapshotData<T> snapshotData;
+
+	/**
+	 * Constructor for reading the snapshot.
+	 */
+	public PojoSerializerSnapshot() {}
+
+	/**
+	 * Constructor for writing the snapshot.
+	 *
+	 * @param pojoClass the Pojo type class.
+	 * @param fields array of fields. Fields may be {@code null} if the originating {@link PojoSerializer} is a
+	 *               restored one with already missing fields, and was never replaced by a new {@link PojoSerializer}.
+	 * @param fieldSerializers array of field serializers.
+	 * @param registeredSubclassSerializers map of registered subclasses to their corresponding serializers.
+	 * @param nonRegisteredSubclassSerializers map of non-registered subclasses to their corresponding serializers.
+	 */
+	PojoSerializerSnapshot(
+			Class<T> pojoClass,
+			Field[] fields,
+			TypeSerializer<?>[] fieldSerializers,
+			LinkedHashMap<Class<?>, TypeSerializer<?>> registeredSubclassSerializers,
+			HashMap<Class<?>, TypeSerializer<?>> nonRegisteredSubclassSerializers) {
+
+		this.snapshotData = PojoSerializerSnapshotData.createFrom(
+			pojoClass,
+			fields,
+			fieldSerializers,
+			registeredSubclassSerializers,
+			nonRegisteredSubclassSerializers);
+	}
+
+	/**
+	 * Constructor for backwards compatibility paths with the {@link PojoSerializer.PojoSerializerConfigSnapshot}.
+	 * This is used in {@link PojoSerializer.PojoSerializerConfigSnapshot#resolveSchemaCompatibility(TypeSerializer)}
+	 * to delegate the compatibility check to this snapshot class.
+	 */
+	PojoSerializerSnapshot(
+			Class<T> pojoClass,
+			Field[] fields,
+			TypeSerializerSnapshot<?>[] existingFieldSerializerSnapshots,
+			LinkedHashMap<Class<?>, TypeSerializerSnapshot<?>> existingRegisteredSubclassSerializerSnapshots,
+			Map<Class<?>, TypeSerializerSnapshot<?>> existingNonRegisteredSubclassSerializerSnapshots) {
+
+		this.snapshotData = PojoSerializerSnapshotData.createFrom(
+			pojoClass,
+			fields,
+			existingFieldSerializerSnapshots,
+			existingRegisteredSubclassSerializerSnapshots,
+			existingNonRegisteredSubclassSerializerSnapshots);
+	}
+
+	@Override
+	public int getCurrentVersion() {
+		return VERSION;
+	}
+
+	@Override
+	public void writeSnapshot(DataOutputView out) throws IOException {
+		snapshotData.writeSnapshotData(out);
+	}
+
+	@Override
+	public void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException {
+		checkArgument(readVersion == 2, "unrecognized read version %d", readVersion);
+		snapshotData = PojoSerializerSnapshotData.createFrom(in, userCodeClassLoader);
+	}
+
+	@Override
+	@SuppressWarnings("unchecked")
+	public TypeSerializer<T> restoreSerializer() {
+		final int numFields = snapshotData.getFieldSerializerSnapshots().size();
+
+		final ArrayList<Field> restoredFields = new ArrayList<>(numFields);
+		final ArrayList<TypeSerializer<?>> restoredFieldSerializers = new ArrayList<>(numFields);
+		snapshotData.getFieldSerializerSnapshots().forEach((fieldName, field, fieldSerializerSnapshot) -> {
+			restoredFields.add(field);
+			checkState(fieldSerializerSnapshot != null, "field serializer snapshots should be present.");
+			restoredFieldSerializers.add(fieldSerializerSnapshot.restoreSerializer());
+		});
+
+		final LinkedHashMap<Class<?>, TypeSerializer<?>> registeredSubclassSerializers = restoreSerializers(
+			snapshotData.getRegisteredSubclassSerializerSnapshots().unwrapOptionals());
+		final Tuple2<LinkedHashMap<Class<?>, Integer>, TypeSerializer<Object>[]> decomposedSubclassSerializerRegistry =
+			decomposeSubclassSerializerRegistry(registeredSubclassSerializers);
+
+		final LinkedHashMap<Class<?>, TypeSerializer<?>> nonRegisteredSubclassSerializers = restoreSerializers(
+			snapshotData.getNonRegisteredSubclassSerializerSnapshots().unwrapOptionals());
+
+		return new PojoSerializer<>(
+			snapshotData.getPojoClass(),
+			restoredFields.toArray(new Field[numFields]),
+			restoredFieldSerializers.toArray(new TypeSerializer[numFields]),
+			decomposedSubclassSerializerRegistry.f0,
+			decomposedSubclassSerializerRegistry.f1,
+			nonRegisteredSubclassSerializers,
+			new ExecutionConfig());
+	}
+
+	@Override
+	public TypeSerializerSchemaCompatibility<T> resolveSchemaCompatibility(TypeSerializer<T> newSerializer) {
+		if (newSerializer.getClass() != PojoSerializer.class) {
+			return TypeSerializerSchemaCompatibility.incompatible();
+		}
+
+		final PojoSerializer<T> newPojoSerializer = (PojoSerializer<T>) newSerializer;
+
+		final Class<T> previousPojoClass = snapshotData.getPojoClass();
+		final LinkedOptionalMap<Field, TypeSerializerSnapshot<?>> fieldSerializerSnapshots = snapshotData.getFieldSerializerSnapshots();
+		final LinkedOptionalMap<Class<?>, TypeSerializerSnapshot<?>> registeredSubclassSerializerSnapshots = snapshotData.getRegisteredSubclassSerializerSnapshots();
+		final LinkedOptionalMap<Class<?>, TypeSerializerSnapshot<?>> nonRegisteredSubclassSerializerSnapshots = snapshotData.getNonRegisteredSubclassSerializerSnapshots();
+
+		if (previousPojoClass != newPojoSerializer.getPojoClass()) {
+			return TypeSerializerSchemaCompatibility.incompatible();
+		}
+
+		if (registeredSubclassSerializerSnapshots.hasAbsentKeysOrValues()) {
+			return TypeSerializerSchemaCompatibility.incompatible();
+		}
+
+		if (nonRegisteredSubclassSerializerSnapshots.hasAbsentKeysOrValues()) {
+			return TypeSerializerSchemaCompatibility.incompatible();
+		}
+
+		final IntermediateCompatibilityResult<T> preExistingFieldSerializersCompatibility =
+			getCompatibilityOfPreExistingFields(newPojoSerializer, fieldSerializerSnapshots);
+
+		if (preExistingFieldSerializersCompatibility.isIncompatible()) {
+			return TypeSerializerSchemaCompatibility.incompatible();
+		}
+
+		final IntermediateCompatibilityResult<T> preExistingRegistrationsCompatibility =
+			getCompatibilityOfPreExistingRegisteredSubclasses(newPojoSerializer, registeredSubclassSerializerSnapshots);
+
+		if (preExistingRegistrationsCompatibility.isIncompatible()) {
+			return TypeSerializerSchemaCompatibility.incompatible();
+		}
+
+		if (newPojoSerializerIsCompatibleAfterMigration(
+				newPojoSerializer,
+				preExistingFieldSerializersCompatibility,
+				preExistingRegistrationsCompatibility,
+				fieldSerializerSnapshots)) {
+
+			return TypeSerializerSchemaCompatibility.compatibleAfterMigration();
+		}
+
+		if (newPojoSerializerIsCompatibleWithReconfiguredSerializer(
+				newPojoSerializer,
+				preExistingFieldSerializersCompatibility,
+				preExistingRegistrationsCompatibility,
+				registeredSubclassSerializerSnapshots,
+				nonRegisteredSubclassSerializerSnapshots)) {
+
+			return TypeSerializerSchemaCompatibility.compatibleWithReconfiguredSerializer(
+				constructReconfiguredPojoSerializer(
+					newPojoSerializer,
+					preExistingFieldSerializersCompatibility,
+					registeredSubclassSerializerSnapshots,
+					preExistingRegistrationsCompatibility,
+					nonRegisteredSubclassSerializerSnapshots));
+		}
+
+		return TypeSerializerSchemaCompatibility.compatibleAsIs();
+	}
+
+	// ---------------------------------------------------------------------------------------------
+	//  Utility methods
+	// ---------------------------------------------------------------------------------------------
+
+	/**
+	 * Transforms a {@link LinkedHashMap} with {@link TypeSerializerSnapshot}s as
+	 * the value to {@link TypeSerializer} as the value by restoring the snapshot.
+	 */
+	private static <K> LinkedHashMap<K, TypeSerializer<?>> restoreSerializers(LinkedHashMap<K, TypeSerializerSnapshot<?>> snapshotsMap) {
+		final LinkedHashMap<K, TypeSerializer<?>> restoredSerializersMap = new LinkedHashMap<>(snapshotsMap.size());
+		snapshotsMap.forEach((key, snapshot) -> restoredSerializersMap.put(key, snapshot.restoreSerializer()));
+		return restoredSerializersMap;
+	}
+
+	/**
+	 * Transforms the subclass serializer registry structure, {@code LinkedHashMap<Class<?>, TypeSerializer<?>>}
+	 * to 2 separate structures: a map containing with registered classes as key and their corresponding ids (order
+	 * in the original map) as value, as well as a separate array of the corresponding subclass serializers.
+	 */
+	@SuppressWarnings("unchecked")
+	private static Tuple2<LinkedHashMap<Class<?>, Integer>, TypeSerializer<Object>[]> decomposeSubclassSerializerRegistry(
+		LinkedHashMap<Class<?>, TypeSerializer<?>> subclassSerializerRegistry) {
+
+		final LinkedHashMap<Class<?>, Integer> subclassIds = new LinkedHashMap<>(subclassSerializerRegistry.size());
+		final TypeSerializer[] subclassSerializers = new TypeSerializer[subclassSerializerRegistry.size()];
+
+		subclassSerializerRegistry.forEach((registeredSubclassClass, serializer) -> {
+			int id = subclassIds.size();
+			subclassIds.put(registeredSubclassClass, id);
+			subclassSerializers[id] = serializer;
+		});
+
+		return Tuple2.of(subclassIds, subclassSerializers);
+	}
+
+	/**
+	 * Finds which Pojo fields exists both in the new {@link PojoSerializer} as well as in the previous one
+	 * (represented by this snapshot), and returns an {@link IntermediateCompatibilityResult}
+	 * of the serializers of those preexisting fields.
+	 */
+	private static <T> IntermediateCompatibilityResult<T> getCompatibilityOfPreExistingFields(
+			PojoSerializer<T> newPojoSerializer,
+			LinkedOptionalMap<Field, TypeSerializerSnapshot<?>> fieldSerializerSnapshots) {
+
+		// the present entries dictates the preexisting fields, because removed fields would be
+		// represented as absent keys in the optional map.
+		final Set<LinkedOptionalMap.KeyValue<Field, TypeSerializerSnapshot<?>>> presentFieldSnapshots =
+			fieldSerializerSnapshots.getPresentEntries();
+
+		final ArrayList<TypeSerializerSnapshot<?>> associatedFieldSerializerSnapshots = new ArrayList<>(presentFieldSnapshots.size());
+		final ArrayList<TypeSerializer<?>> associatedNewFieldSerializers = new ArrayList<>(presentFieldSnapshots.size());
+
+		for (LinkedOptionalMap.KeyValue<Field, TypeSerializerSnapshot<?>> presentFieldEntry : presentFieldSnapshots) {
+			TypeSerializer<?> associatedNewFieldSerializer = newPojoSerializer.getFieldSerializer(presentFieldEntry.getKey());
+			checkState(
+				associatedNewFieldSerializer != null,
+				"a present field should have its associated new field serializer available.");
+
+			associatedFieldSerializerSnapshots.add(presentFieldEntry.getValue());
+			associatedNewFieldSerializers.add(associatedNewFieldSerializer);
+		}
+
+		return CompositeTypeSerializerUtil.constructIntermediateCompatibilityResult(
+			associatedNewFieldSerializers.toArray(new TypeSerializer<?>[associatedNewFieldSerializers.size()]),
+			associatedFieldSerializerSnapshots.toArray(new TypeSerializerSnapshot<?>[associatedFieldSerializerSnapshots.size()]));
+	}
+
+	/**
+	 * Finds which registered subclasses exists both in the new {@link PojoSerializer} as well as in the previous one
+	 * (represented by this snapshot), and returns an {@link IntermediateCompatibilityResult}
+	 * of the serializers of this preexisting registered subclasses.
+	 */
+	private static <T> IntermediateCompatibilityResult<T> getCompatibilityOfPreExistingRegisteredSubclasses(
+			PojoSerializer<T> newPojoSerializer,
+			LinkedOptionalMap<Class<?>, TypeSerializerSnapshot<?>> registeredSubclassSerializerSnapshots) {
+
+		final LinkedHashMap<Class<?>, TypeSerializerSnapshot<?>> unwrappedSerializerSnapshots = registeredSubclassSerializerSnapshots.unwrapOptionals();
+
+		final ArrayList<TypeSerializerSnapshot<?>> associatedSubclassSerializerSnapshots = new ArrayList<>();
+		final ArrayList<TypeSerializer<?>> associatedNewSubclassSerializers = new ArrayList<>();
+
+		final LinkedHashMap<Class<?>, TypeSerializer<?>> newSubclassSerializerRegistry = newPojoSerializer.getBundledSubclassSerializerRegistry();
+
+		for (Map.Entry<Class<?>, TypeSerializerSnapshot<?>> entry : unwrappedSerializerSnapshots.entrySet()) {
+			TypeSerializer<?> newRegisteredSerializer = newSubclassSerializerRegistry.get(entry.getKey());
+			if (newRegisteredSerializer != null) {
+				associatedSubclassSerializerSnapshots.add(entry.getValue());
+				associatedNewSubclassSerializers.add(newRegisteredSerializer);
+			}
+		}
+
+		return CompositeTypeSerializerUtil.constructIntermediateCompatibilityResult(
+			associatedNewSubclassSerializers.toArray(new TypeSerializer<?>[associatedNewSubclassSerializers.size()]),
+			associatedSubclassSerializerSnapshots.toArray(new TypeSerializerSnapshot<?>[associatedSubclassSerializerSnapshots.size()]));
+	}
+
+	/**
+	 * Checks if the new {@link PojoSerializer} is compatible after migration.
+	 */
+	private static <T> boolean newPojoSerializerIsCompatibleAfterMigration(
+			PojoSerializer<T> newPojoSerializer,
+			IntermediateCompatibilityResult<T> fieldSerializerCompatibility,
+			IntermediateCompatibilityResult<T> preExistingRegistrationsCompatibility,
+			LinkedOptionalMap<Field, TypeSerializerSnapshot<?>> fieldSerializerSnapshots) {
+		return newPojoHasNewOrRemovedFields(fieldSerializerSnapshots, newPojoSerializer)
+			|| fieldSerializerCompatibility.isCompatibleAfterMigration()
+			|| preExistingRegistrationsCompatibility.isCompatibleAfterMigration();
+	}
+
+	/**
+	 * Checks if the new {@link PojoSerializer} is compatible with a reconfigured instance.
+	 */
+	private static <T> boolean newPojoSerializerIsCompatibleWithReconfiguredSerializer(
+			PojoSerializer<T> newPojoSerializer,
+			IntermediateCompatibilityResult<T> fieldSerializerCompatibility,
+			IntermediateCompatibilityResult<T> preExistingRegistrationsCompatibility,
+			LinkedOptionalMap<Class<?>, TypeSerializerSnapshot<?>> registeredSubclassSerializerSnapshots,
+			LinkedOptionalMap<Class<?>, TypeSerializerSnapshot<?>> nonRegisteredSubclassSerializerSnapshots) {
+		return newPojoHasDifferentSubclassRegistrationOrder(registeredSubclassSerializerSnapshots, newPojoSerializer)
+			|| previousSerializerHasNonRegisteredSubclasses(nonRegisteredSubclassSerializerSnapshots)
+			|| fieldSerializerCompatibility.isCompatibleWithReconfiguredSerializer()
+			|| preExistingRegistrationsCompatibility.isCompatibleWithReconfiguredSerializer();
+	}
+
+	/**
+	 * Checks whether the new {@link PojoSerializer} has new or removed fields compared to the previous one.
+	 */
+	private static boolean newPojoHasNewOrRemovedFields(
+			LinkedOptionalMap<Field, TypeSerializerSnapshot<?>> fieldSerializerSnapshots,
+			PojoSerializer<?> newPojoSerializer) {
+		int numRemovedFields = fieldSerializerSnapshots.absentKeysOrValues().size();
+		int numPreexistingFields = fieldSerializerSnapshots.size() - numRemovedFields;
+
+		boolean hasRemovedFields = numRemovedFields > 0;
+		boolean hasNewFields = newPojoSerializer.getFields().length - numPreexistingFields > 0;
+		return hasRemovedFields || hasNewFields;
+	}
+
+	/**
+	 * Checks whether the new {@link PojoSerializer} has a different subclass registration order
+	 * compared to the previous one.
+	 */
+	private static boolean newPojoHasDifferentSubclassRegistrationOrder(
+			LinkedOptionalMap<Class<?>, TypeSerializerSnapshot<?>> registeredSubclassSerializerSnapshots,
+			PojoSerializer<?> newPojoSerializer) {
+		Set<Class<?>> previousRegistrationOrder = registeredSubclassSerializerSnapshots.unwrapOptionals().keySet();
+		Set<Class<?>> newRegistrationOrder = newPojoSerializer.getRegisteredClasses().keySet();
+		return !isPreviousRegistrationPrefixOfNewRegistration(previousRegistrationOrder, newRegistrationOrder);
+	}
+
+	private static boolean isPreviousRegistrationPrefixOfNewRegistration(
+		Set<Class<?>> previousRegistrationOrder,
+		Set<Class<?>> newRegistrationOrder) {
+		Iterator<Class<?>> newRegistrationItr = newRegistrationOrder.iterator();
+
+		for (Class<?> previousRegisteredClass : previousRegistrationOrder) {
+			if (!newRegistrationItr.hasNext()) {
+				return false;
+			}
+			Class<?> newRegisteredClass = newRegistrationItr.next();
+			if (!previousRegisteredClass.equals(newRegisteredClass)) {
+				return false;
+			}
+		}
+		return true;
+	}
+
+	/**
+	 * Checks whether the previous serializer, represented by this snapshot, has
+	 * non-registered subclasses.
+	 */
+	private static boolean previousSerializerHasNonRegisteredSubclasses(
+			LinkedOptionalMap<Class<?>, TypeSerializerSnapshot<?>> nonRegisteredSubclassSerializerSnapshots) {
+		return nonRegisteredSubclassSerializerSnapshots.size() > 0;
+	}
+
+	/**
+	 * Creates a reconfigured version of the {@link PojoSerializer}.
+	 *
+	 * @param originalNewPojoSerializer the original new {@link PojoSerializer} to create a reconfigured version of.
+	 * @param fieldSerializerCompatibility compatibility of preexisting fields' serializers.
+	 * @param registeredSerializerSnapshots snapshot of previous registered subclasses' serializers.
+	 * @param preExistingRegistrationsCompatibility compatibility of preexisting subclasses' serializers.
+	 * @param nonRegisteredSubclassSerializerSnapshots snapshot of previous non-registered subclasses' serializers.
+	 *
+	 * @return a reconfigured version of the original new {@link PojoSerializer}.
+	 */
+	private static <T> PojoSerializer<T> constructReconfiguredPojoSerializer(
+			PojoSerializer<T> originalNewPojoSerializer,
+			IntermediateCompatibilityResult<T> fieldSerializerCompatibility,
+			LinkedOptionalMap<Class<?>, TypeSerializerSnapshot<?>> registeredSerializerSnapshots,
+			IntermediateCompatibilityResult<T> preExistingRegistrationsCompatibility,
+			LinkedOptionalMap<Class<?>, TypeSerializerSnapshot<?>> nonRegisteredSubclassSerializerSnapshots) {
+
+		@SuppressWarnings("unchecked")
+		final TypeSerializer<Object>[] reconfiguredFieldSerializers = constructReconfiguredFieldSerializers(fieldSerializerCompatibility);
+
+		Tuple2<LinkedHashMap<Class<?>, Integer>, TypeSerializer<Object>[]> reconfiguredSubclassRegistry = constructReconfiguredSubclassRegistry(
+			originalNewPojoSerializer.getBundledSubclassSerializerRegistry(),
+			registeredSerializerSnapshots,
+			preExistingRegistrationsCompatibility);
+
+		return new PojoSerializer<>(
+			originalNewPojoSerializer.getPojoClass(),
+			originalNewPojoSerializer.getFields(),
+			reconfiguredFieldSerializers,
+			reconfiguredSubclassRegistry.f0,
+			reconfiguredSubclassRegistry.f1,
+			restoreSerializers(nonRegisteredSubclassSerializerSnapshots.unwrapOptionals()),
+			originalNewPojoSerializer.getExecutionConfig());
+	}
+
+	private static TypeSerializer[] constructReconfiguredFieldSerializers(
+			IntermediateCompatibilityResult<?> fieldSerializerCompatibility) {
+		checkArgument(!fieldSerializerCompatibility.isIncompatible() && !fieldSerializerCompatibility.isCompatibleAfterMigration());
+		return fieldSerializerCompatibility.getNestedSerializers();
+	}
+
+	private static Tuple2<LinkedHashMap<Class<?>, Integer>, TypeSerializer<Object>[]> constructReconfiguredSubclassRegistry(
+			LinkedHashMap<Class<?>, TypeSerializer<?>> newSubclassRegistrations,
+			LinkedOptionalMap<Class<?>, TypeSerializerSnapshot<?>> registeredSerializerSnapshots,
+			IntermediateCompatibilityResult<?> preExistingRegistrationsCompatibility) {
+
+		checkArgument(!preExistingRegistrationsCompatibility.isIncompatible() && !preExistingRegistrationsCompatibility.isCompatibleAfterMigration());
+
+		LinkedHashMap<Class<?>, TypeSerializer<?>> reconfiguredSubclassSerializerRegistry =
+			restoreSerializers(registeredSerializerSnapshots.unwrapOptionals());
+
+		Iterator<TypeSerializer<?>> serializersForPreexistingRegistrations =
+			Arrays.asList(preExistingRegistrationsCompatibility.getNestedSerializers()).iterator();
+
+		for (Map.Entry<Class<?>, TypeSerializer<?>> registration : newSubclassRegistrations.entrySet()) {
+			// new registrations should simply be appended to the subclass serializer registry with their new serializers;
+			// preexisting registrations should use the compatibility-checked serializer
+			TypeSerializer<?> newRegistration = (reconfiguredSubclassSerializerRegistry.containsKey(registration.getKey()))
+				? serializersForPreexistingRegistrations.next()
+				: registration.getValue();
+			reconfiguredSubclassSerializerRegistry.put(registration.getKey(), newRegistration);
+		}
+
+		return decomposeSubclassSerializerRegistry(reconfiguredSubclassSerializerRegistry);
+	}
+}
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerSnapshotData.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerSnapshotData.java
new file mode 100644
index 0000000..e2efbdb
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerSnapshotData.java
@@ -0,0 +1,325 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.typeutils.runtime;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
+import org.apache.flink.api.common.typeutils.TypeSerializerUtils;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.util.InstantiationUtil;
+import org.apache.flink.util.LinkedOptionalMap;
+import org.apache.flink.util.function.BiConsumerWithException;
+import org.apache.flink.util.function.BiFunctionWithException;
+import org.apache.flink.util.function.FunctionWithException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+
+import static org.apache.flink.util.LinkedOptionalMap.optionalMapOf;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * This class holds the snapshot content for the {@link PojoSerializer}.
+ *
+ * <h2>Serialization Format</hr>
+ *
+ * <p>The serialization format defined by this class is as follows:
+ *
+ * <pre>{@code
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * |                                            POJO class name                                          |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * |      number of fields      |                (field name, field serializer snapshot)                 |
+ * |                            |                                pairs                                   |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * |         number of          |       (registered subclass name, subclass serializer snapshot)         |
+ * |   registered subclasses    |                                pairs                                   |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * |         number of          |     (non-registered subclass name, subclass serializer snapshot)       |
+ * | non-registered subclasses  |                                pairs                                   |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * }</pre>
+ */
+@Internal
+final class PojoSerializerSnapshotData<T> {
+
+	private static final Logger LOG = LoggerFactory.getLogger(PojoSerializerSnapshotData.class);
+
+	// ---------------------------------------------------------------------------------------------
+	//  Factory methods
+	// ---------------------------------------------------------------------------------------------
+
+	/**
+	 * Creates a {@link PojoSerializerSnapshotData} from configuration of a {@link PojoSerializer}.
+	 *
+	 * <p>This factory method is meant to be used in regular write paths, i.e. when taking a snapshot
+	 * of the {@link PojoSerializer}. All registered subclass classes, and non-registered
+	 * subclass classes are all present. Some POJO fields may be absent, if the originating
+	 * {@link PojoSerializer} was a restored one with already missing fields, and was never replaced
+	 * by a new {@link PojoSerializer} (i.e. because the serialized old data was never accessed).
+	 */
+	static <T> PojoSerializerSnapshotData<T> createFrom(
+			Class<T> pojoClass,
+			Field[] fields,
+			TypeSerializer<?>[] fieldSerializers,
+			LinkedHashMap<Class<?>, TypeSerializer<?>> registeredSubclassSerializers,
+			HashMap<Class<?>, TypeSerializer<?>> nonRegisteredSubclassSerializers) {
+
+		final LinkedOptionalMap<Field, TypeSerializerSnapshot<?>> fieldSerializerSnapshots = new LinkedOptionalMap<>(fields.length);
+
+		for (int i = 0; i < fields.length; i++) {
+			Field field = fields[i];
+			String fieldName = (field == null) ? getDummyNameForMissingField(i) : field.getName();
+			fieldSerializerSnapshots.put(fieldName, field, TypeSerializerUtils.snapshotBackwardsCompatible(fieldSerializers[i]));
+		}
+
+		LinkedHashMap<Class<?>, TypeSerializerSnapshot<?>> registeredSubclassSerializerSnapshots = new LinkedHashMap<>(registeredSubclassSerializers.size());
+		registeredSubclassSerializers.forEach((k, v) -> registeredSubclassSerializerSnapshots.put(k, TypeSerializerUtils.snapshotBackwardsCompatible(v)));
+
+		HashMap<Class<?>, TypeSerializerSnapshot<?>> nonRegisteredSubclassSerializerSnapshots = new HashMap<>(nonRegisteredSubclassSerializers.size());
+		nonRegisteredSubclassSerializers.forEach((k, v) -> nonRegisteredSubclassSerializerSnapshots.put(k, TypeSerializerUtils.snapshotBackwardsCompatible(v)));
+
+		return new PojoSerializerSnapshotData<>(
+			pojoClass,
+			fieldSerializerSnapshots,
+			optionalMapOf(registeredSubclassSerializerSnapshots, Class::getName),
+			optionalMapOf(nonRegisteredSubclassSerializerSnapshots, Class::getName));
+	}
+
+	/**
+	 * Creates a {@link PojoSerializerSnapshotData} from serialized data stream.
+	 *
+	 * <p>This factory method is meant to be used in regular read paths, i.e. when reading back a snapshot
+	 * of the {@link PojoSerializer}. POJO fields, registered subclass classes, and non-registered subclass
+	 * classes may no longer be present anymore.
+	 */
+	static <T> PojoSerializerSnapshotData<T> createFrom(DataInputView in, ClassLoader userCodeClassLoader) throws IOException {
+		return PojoSerializerSnapshotData.readSnapshotData(in, userCodeClassLoader);
+	}
+
+	/**
+	 * Creates a {@link PojoSerializerSnapshotData} from existing snapshotted configuration of a {@link PojoSerializer}.
+	 */
+	static <T> PojoSerializerSnapshotData<T> createFrom(
+			Class<T> pojoClass,
+			Field[] fields,
+			TypeSerializerSnapshot<?>[] existingFieldSerializerSnapshots,
+			LinkedHashMap<Class<?>, TypeSerializerSnapshot<?>> existingRegisteredSubclassSerializerSnapshots,
+			Map<Class<?>, TypeSerializerSnapshot<?>> existingNonRegisteredSubclassSerializerSnapshots) {
+
+		final LinkedOptionalMap<Field, TypeSerializerSnapshot<?>> fieldSerializerSnapshots = new LinkedOptionalMap<>(fields.length);
+		for (int i = 0; i < fields.length; i++) {
+			Field field = fields[i];
+			String fieldName = (field == null) ? getDummyNameForMissingField(i) : field.getName();
+			fieldSerializerSnapshots.put(fieldName, field, existingFieldSerializerSnapshots[i]);
+		}
+
+		return new PojoSerializerSnapshotData<>(
+			pojoClass,
+			fieldSerializerSnapshots,
+			optionalMapOf(existingRegisteredSubclassSerializerSnapshots, Class::getName),
+			optionalMapOf(existingNonRegisteredSubclassSerializerSnapshots, Class::getName));
+	}
+
+	private Class<T> pojoClass;
+	private LinkedOptionalMap<Field, TypeSerializerSnapshot<?>> fieldSerializerSnapshots;
+	private LinkedOptionalMap<Class<?>, TypeSerializerSnapshot<?>> registeredSubclassSerializerSnapshots;
+	private LinkedOptionalMap<Class<?>, TypeSerializerSnapshot<?>> nonRegisteredSubclassSerializerSnapshots;
+
+	private PojoSerializerSnapshotData(
+			Class<T> typeClass,
+			LinkedOptionalMap<Field, TypeSerializerSnapshot<?>> fieldSerializerSnapshots,
+			LinkedOptionalMap<Class<?>, TypeSerializerSnapshot<?>> registeredSubclassSerializerSnapshots,
+			LinkedOptionalMap<Class<?>, TypeSerializerSnapshot<?>> nonRegisteredSubclassSerializerSnapshots) {
+
+		this.pojoClass = checkNotNull(typeClass);
+		this.fieldSerializerSnapshots = checkNotNull(fieldSerializerSnapshots);
+		this.registeredSubclassSerializerSnapshots = checkNotNull(registeredSubclassSerializerSnapshots);
+		this.nonRegisteredSubclassSerializerSnapshots = checkNotNull(nonRegisteredSubclassSerializerSnapshots);
+	}
+
+	// ---------------------------------------------------------------------------------------------
+	//  Snapshot data read / write methods
+	// ---------------------------------------------------------------------------------------------
+
+	void writeSnapshotData(DataOutputView out) throws IOException {
+		out.writeUTF(pojoClass.getName());
+		writeOptionalMap(out, fieldSerializerSnapshots, PojoFieldUtils::writeField, TypeSerializerSnapshot::writeVersionedSnapshot);
+		writeOptionalMap(out, registeredSubclassSerializerSnapshots, NoOpWriter.noopWriter(), TypeSerializerSnapshot::writeVersionedSnapshot);
+		writeOptionalMap(out, nonRegisteredSubclassSerializerSnapshots, NoOpWriter.noopWriter(), TypeSerializerSnapshot::writeVersionedSnapshot);
+	}
+
+	private static <T> PojoSerializerSnapshotData<T> readSnapshotData(DataInputView in, ClassLoader userCodeClassLoader) throws IOException {
+		Class<T> pojoClass = InstantiationUtil.resolveClassByName(in, userCodeClassLoader);
+
+		LinkedOptionalMap<Field, TypeSerializerSnapshot<?>> fieldSerializerSnapshots = readOptionalMap(
+			in,
+			fieldReader(userCodeClassLoader),
+			snapshotReader(userCodeClassLoader));
+		LinkedOptionalMap<Class<?>, TypeSerializerSnapshot<?>> registeredSubclassSerializerSnapshots = readOptionalMap(
+			in,
+			classReader(userCodeClassLoader),
+			snapshotReader(userCodeClassLoader));
+		LinkedOptionalMap<Class<?>, TypeSerializerSnapshot<?>> nonRegisteredSubclassSerializerSnapshots = readOptionalMap(
+			in,
+			classReader(userCodeClassLoader),
+			snapshotReader(userCodeClassLoader));
+
+		return new PojoSerializerSnapshotData<>(pojoClass, fieldSerializerSnapshots, registeredSubclassSerializerSnapshots, nonRegisteredSubclassSerializerSnapshots);
+	}
+
+	// ---------------------------------------------------------------------------------------------
+	//  Snapshot data accessors
+	// ---------------------------------------------------------------------------------------------
+
+	Class<T> getPojoClass() {
+		return pojoClass;
+	}
+
+	LinkedOptionalMap<Field, TypeSerializerSnapshot<?>> getFieldSerializerSnapshots() {
+		return fieldSerializerSnapshots;
+	}
+
+	LinkedOptionalMap<Class<?>, TypeSerializerSnapshot<?>> getRegisteredSubclassSerializerSnapshots() {
+		return registeredSubclassSerializerSnapshots;
+	}
+
+	LinkedOptionalMap<Class<?>, TypeSerializerSnapshot<?>> getNonRegisteredSubclassSerializerSnapshots() {
+		return nonRegisteredSubclassSerializerSnapshots;
+	}
+
+	// ---------------------------------------------------------------------------------------------
+	//  Utilities
+	// ---------------------------------------------------------------------------------------------
+
+	private static String getDummyNameForMissingField(int fieldIndex) {
+		return String.format("missing-field-at-%d", fieldIndex);
+	}
+
+	private static <K, V> void writeOptionalMap(
+		DataOutputView out,
+		LinkedOptionalMap<K, V> map,
+		BiConsumerWithException<DataOutputView, K, IOException> keyWriter,
+		BiConsumerWithException<DataOutputView, V, IOException> valueWriter) throws IOException {
+
+		out.writeInt(map.size());
+		map.forEach(((keyName, key, value) -> {
+			out.writeUTF(keyName);
+
+			if (key == null) {
+				out.writeBoolean(false);
+			} else {
+				out.writeBoolean(true);
+				keyWriter.accept(out, key);
+			}
+
+			if (value == null) {
+				out.writeBoolean(false);
+			} else {
+				out.writeBoolean(true);
+				valueWriter.accept(out, value);
+			}
+		}));
+	}
+
+	private static <K, V> LinkedOptionalMap<K, V> readOptionalMap(
+		DataInputView in,
+		BiFunctionWithException<DataInputView, String, K, IOException> keyReader,
+		FunctionWithException<DataInputView, V, IOException> valueReader) throws IOException {
+
+		long mapSize = in.readInt();
+
+		LinkedOptionalMap<K, V> map = new LinkedOptionalMap<>();
+		for (int i = 0; i < mapSize; i++) {
+			String keyName = in.readUTF();
+
+			final K key;
+			if (in.readBoolean()) {
+				key = keyReader.apply(in, keyName);
+			} else {
+				key = null;
+			}
+
+			final V value;
+			if (in.readBoolean()) {
+				value = valueReader.apply(in);
+			} else {
+				value = null;
+			}
+
+			map.put(keyName, key, value);
+		}
+		return map;
+	}
+
+	private enum NoOpWriter implements BiConsumerWithException<DataOutputView, Object, IOException> {
+		INSTANCE;
+
+		@Override
+		public void accept(DataOutputView dataOutputView, Object o) throws IOException {}
+
+		@SuppressWarnings("unchecked")
+		static <K> BiConsumerWithException<DataOutputView, K, IOException> noopWriter() {
+			return (BiConsumerWithException<DataOutputView, K, IOException>) INSTANCE;
+		}
+	}
+
+	private static BiFunctionWithException<DataInputView, String, Field, IOException> fieldReader(ClassLoader cl) {
+		return (input, fieldName) -> {
+			try {
+				return PojoFieldUtils.readField(input, cl);
+			}
+			catch (Throwable t) {
+				LOG.warn(String.format("Exception while reading field %s", fieldName), t);
+				return null;
+			}
+		};
+	}
+
+	private static FunctionWithException<DataInputView, TypeSerializerSnapshot<?>, IOException> snapshotReader(ClassLoader cl) {
+		return input -> {
+			try {
+				return TypeSerializerSnapshot.readVersionedSnapshot(input, cl);
+			}
+			catch (Throwable t) {
+				LOG.warn("Exception while reading serializer snapshot.", t);
+				return null;
+			}
+		};
+	}
+
+	private static BiFunctionWithException<DataInputView, String, Class<?>, IOException> classReader(ClassLoader cl) {
+		return (input, className) -> {
+			try {
+				// input is ignored because we don't write the actual class as value.
+				return Class.forName(className, false, cl);
+			} catch (Throwable t) {
+				LOG.warn(String.format("Exception while reading class %s", className), t);
+				return null;
+			}
+		};
+	}
+}
diff --git a/flink-core/src/main/java/org/apache/flink/util/LinkedOptionalMap.java b/flink-core/src/main/java/org/apache/flink/util/LinkedOptionalMap.java
index 0697ec6..30c7b54 100644
--- a/flink-core/src/main/java/org/apache/flink/util/LinkedOptionalMap.java
+++ b/flink-core/src/main/java/org/apache/flink/util/LinkedOptionalMap.java
@@ -21,16 +21,19 @@ package org.apache.flink.util;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.annotation.VisibleForTesting;
 
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
 import java.util.Map.Entry;
 import java.util.Optional;
 import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
-import javax.annotation.Nullable;
-
 import static org.apache.flink.util.Preconditions.checkNotNull;
 
 /**
@@ -56,7 +59,7 @@ public final class LinkedOptionalMap<K, V> {
 	 * @param <V>           value type
 	 * @return an {@code LinkedOptionalMap} with optional named keys, and optional values.
 	 */
-	public static <K, V> LinkedOptionalMap<K, V> optionalMapOf(LinkedHashMap<K, V> sourceMap, Function<K, String> keyNameGetter) {
+	public static <K, V> LinkedOptionalMap<K, V> optionalMapOf(HashMap<K, V> sourceMap, Function<K, String> keyNameGetter) {
 
 		LinkedHashMap<String, KeyValue<K, V>> underlyingMap = new LinkedHashMap<>(sourceMap.size());
 
@@ -88,6 +91,10 @@ public final class LinkedOptionalMap<K, V> {
 		this(new LinkedHashMap<>());
 	}
 
+	public LinkedOptionalMap(int initialSize) {
+		this(new LinkedHashMap<>(initialSize));
+	}
+
 	@SuppressWarnings("CopyConstructorMissesField")
 	LinkedOptionalMap(LinkedOptionalMap<K, V> linkedOptionalMap) {
 		this(new LinkedHashMap<>(linkedOptionalMap.underlyingMap));
@@ -131,6 +138,41 @@ public final class LinkedOptionalMap<K, V> {
 	}
 
 	/**
+	 * Checks whether there are entries with absent keys or values.
+	 */
+	public boolean hasAbsentKeysOrValues() {
+		for (Entry<String, KeyValue<K, V>> entry : underlyingMap.entrySet()) {
+			if (keyOrValueIsAbsent(entry)) {
+				return true;
+			}
+		}
+		return false;
+	}
+
+	/**
+	 * A {@link java.util.function.Consumer} that throws exceptions.
+	 */
+	@FunctionalInterface
+	public interface ConsumerWithException<K, V, E extends Throwable> {
+		void accept(@Nonnull String keyName, @Nullable K key , @Nullable V value) throws E;
+	}
+
+	public <E extends Throwable> void forEach(ConsumerWithException<K, V, E> consumer) throws E {
+		for (Entry<String, KeyValue<K, V>> entry : underlyingMap.entrySet()) {
+			KeyValue<K, V> kv = entry.getValue();
+			consumer.accept(entry.getKey(), kv.key, kv.value);
+		}
+	}
+
+	public Set<KeyValue<K, V>> getPresentEntries() {
+		return underlyingMap.entrySet()
+			.stream()
+			.filter(entry -> !LinkedOptionalMap.keyOrValueIsAbsent(entry))
+			.map(Entry::getValue)
+			.collect(Collectors.toCollection(LinkedHashSet::new));
+	}
+
+	/**
 	 * Assuming all the entries of this map are present (keys and values) this method would return
 	 * a map with these key and values, stripped from their Optional wrappers.
 	 * NOTE: please note that if any of the key or values are absent this method would throw an {@link IllegalStateException}.
@@ -149,7 +191,6 @@ public final class LinkedOptionalMap<K, V> {
 			}
 			unwrapped.put(kv.key, kv.value);
 		}
-
 		return unwrapped;
 	}
 
@@ -189,7 +230,13 @@ public final class LinkedOptionalMap<K, V> {
 	// Inner Classes
 	// --------------------------------------------------------------------------------------------------------
 
-	private static final class KeyValue<K, V> {
+	/**
+	 * Key-value pairs stored by the underlying map.
+	 *
+	 * @param <K> key type.
+	 * @param <V> value type.
+	 */
+	public static final class KeyValue<K, V> {
 		K key;
 		V value;
 
@@ -198,6 +245,14 @@ public final class LinkedOptionalMap<K, V> {
 			this.value = value;
 		}
 
+		public K getKey() {
+			return key;
+		}
+
+		public V getValue() {
+			return value;
+		}
+
 		KeyValue<K, V> merge(K key, V value) {
 			this.key = firstNonNull(key, this.key);
 			this.value = firstNonNull(value, this.value);
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerSnapshotTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerSnapshotTest.java
new file mode 100644
index 0000000..91a3414
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerSnapshotTest.java
@@ -0,0 +1,385 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.typeutils.runtime;
+
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
+import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
+import org.apache.flink.api.common.typeutils.base.DoubleSerializer;
+import org.apache.flink.api.common.typeutils.base.IntSerializer;
+import org.apache.flink.api.common.typeutils.base.StringSerializer;
+import org.apache.flink.testutils.migration.SchemaCompatibilityTestingSerializer;
+
+import org.junit.Test;
+
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests for the {@link PojoSerializerSnapshot}.
+ */
+public class PojoSerializerSnapshotTest {
+
+	public static class TestPojo {
+		public int id;
+		public String name;
+		public double height;
+
+		public TestPojo() {}
+
+		public TestPojo(int id, String name, double height) {
+			this.id = id;
+			this.name = name;
+			this.height = height;
+		}
+	}
+
+	private static class TestPojoField {
+		String name;
+		Field field;
+		TypeSerializer<?> serializer;
+		TypeSerializerSnapshot<?> serializerSnapshot;
+
+		TestPojoField(String name, TypeSerializer<?> serializer) throws Exception {
+			this.name = name;
+			this.field = TestPojo.class.getDeclaredField(name);
+			this.serializer = serializer;
+			this.serializerSnapshot = serializer.snapshotConfiguration();
+		}
+
+		TestPojoField(String name, Field field, TypeSerializerSnapshot<?> serializerSnapshot) {
+			this.name = name;
+			this.field = field;
+			this.serializerSnapshot = serializerSnapshot;
+		}
+
+		TestPojoField shallowCopy() {
+			return new TestPojoField(name, field, serializerSnapshot);
+		}
+	}
+
+	private final static Map<String, Field> FIELDS = new HashMap<>(3);
+	static {
+		try {
+			FIELDS.put("id", TestPojo.class.getDeclaredField("id"));
+			FIELDS.put("name", TestPojo.class.getDeclaredField("name"));
+			FIELDS.put("height", TestPojo.class.getDeclaredField("height"));
+		} catch (Exception e) {
+			throw new RuntimeException(e);
+		}
+	}
+
+	private static TestPojoField ID_FIELD;
+	private static TestPojoField NAME_FIELD;
+	private static TestPojoField HEIGHT_FIELD;
+	static {
+		try {
+			ID_FIELD = new TestPojoField("id", IntSerializer.INSTANCE);
+			NAME_FIELD = new TestPojoField("name", StringSerializer.INSTANCE);
+			HEIGHT_FIELD = new TestPojoField("height", DoubleSerializer.INSTANCE);
+		} catch (Exception e) {
+			throw new RuntimeException(e);
+		}
+	}
+	// ------------------------------------------------------------------------------------------------
+	//  Tests for PojoSerializerSnapshot#restoreSerializer
+	// ------------------------------------------------------------------------------------------------
+
+	@Test
+	public void testRestoreSerializerWithSameFields() {
+		final PojoSerializerSnapshot<TestPojo> testSnapshot = buildTestSnapshot(Arrays.asList(
+			ID_FIELD,
+			NAME_FIELD,
+			HEIGHT_FIELD
+		));
+
+		final TypeSerializer<TestPojo> restoredSerializer = testSnapshot.restoreSerializer();
+		assertTrue(restoredSerializer.getClass() == PojoSerializer.class);
+		final PojoSerializer<TestPojo> restoredPojoSerializer = (PojoSerializer<TestPojo>) restoredSerializer;
+
+		final Field[] restoredFields = restoredPojoSerializer.getFields();
+		assertArrayEquals(
+			new Field[] { ID_FIELD.field, NAME_FIELD.field, HEIGHT_FIELD.field },
+			restoredFields);
+
+		final TypeSerializer<?>[] restoredFieldSerializers = restoredPojoSerializer.getFieldSerializers();
+		assertArrayEquals(
+			new TypeSerializer[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE, DoubleSerializer.INSTANCE },
+			restoredFieldSerializers);
+	}
+
+	@Test
+	public void testRestoreSerializerWithRemovedFields() {
+		final PojoSerializerSnapshot<TestPojo> testSnapshot = buildTestSnapshot(Arrays.asList(
+			mockRemovedField(ID_FIELD),
+			NAME_FIELD,
+			mockRemovedField(HEIGHT_FIELD)
+		));
+
+		final TypeSerializer<TestPojo> restoredSerializer = testSnapshot.restoreSerializer();
+		assertTrue(restoredSerializer.getClass() == PojoSerializer.class);
+		final PojoSerializer<TestPojo> restoredPojoSerializer = (PojoSerializer<TestPojo>) restoredSerializer;
+
+		final Field[] restoredFields = restoredPojoSerializer.getFields();
+		assertArrayEquals(
+			new Field[] { null, NAME_FIELD.field, null },
+			restoredFields);
+
+		final TypeSerializer<?>[] restoredFieldSerializers = restoredPojoSerializer.getFieldSerializers();
+		assertArrayEquals(
+			new TypeSerializer[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE, DoubleSerializer.INSTANCE },
+			restoredFieldSerializers);
+	}
+
+	@Test
+	public void testRestoreSerializerWithNewFields() {
+		final PojoSerializerSnapshot<TestPojo> testSnapshot = buildTestSnapshot(Collections.singletonList(HEIGHT_FIELD));
+
+		final TypeSerializer<TestPojo> restoredSerializer = testSnapshot.restoreSerializer();
+		assertTrue(restoredSerializer.getClass() == PojoSerializer.class);
+		final PojoSerializer<TestPojo> restoredPojoSerializer = (PojoSerializer<TestPojo>) restoredSerializer;
+
+		final Field[] restoredFields = restoredPojoSerializer.getFields();
+		assertArrayEquals(
+			new Field[] { HEIGHT_FIELD.field },
+			restoredFields);
+
+		final TypeSerializer<?>[] restoredFieldSerializers = restoredPojoSerializer.getFieldSerializers();
+		assertArrayEquals(
+			new TypeSerializer[] { DoubleSerializer.INSTANCE },
+			restoredFieldSerializers);
+	}
+
+	// ------------------------------------------------------------------------------------------------
+	//  Tests for PojoSerializerSnapshot#resolveSchemaCompatibility
+	// ------------------------------------------------------------------------------------------------
+
+	@Test
+	public void testResolveSchemaCompatibilityWithSameFields() {
+		final PojoSerializerSnapshot<TestPojo> testSnapshot = buildTestSnapshot(Arrays.asList(
+			ID_FIELD,
+			NAME_FIELD,
+			HEIGHT_FIELD
+		));
+
+		final PojoSerializer<TestPojo> newPojoSerializer = buildTestNewPojoSerializer(Arrays.asList(
+			ID_FIELD,
+			NAME_FIELD,
+			HEIGHT_FIELD
+		));
+
+		final TypeSerializerSchemaCompatibility<TestPojo> resultCompatibility =
+			testSnapshot.resolveSchemaCompatibility(newPojoSerializer);
+
+		assertTrue(resultCompatibility.isCompatibleAsIs());
+	}
+
+	@Test
+	public void testResolveSchemaCompatibilityWithRemovedFields() {
+		final PojoSerializerSnapshot<TestPojo> testSnapshot = buildTestSnapshot(Arrays.asList(
+			mockRemovedField(ID_FIELD),
+			NAME_FIELD,
+			mockRemovedField(HEIGHT_FIELD)
+		));
+
+		final PojoSerializer<TestPojo> newPojoSerializer = buildTestNewPojoSerializer(Collections.singletonList(NAME_FIELD));
+
+		final TypeSerializerSchemaCompatibility<TestPojo> resultCompatibility =
+			testSnapshot.resolveSchemaCompatibility(newPojoSerializer);
+
+		assertTrue(resultCompatibility.isCompatibleAfterMigration());
+	}
+
+	@Test
+	public void testResolveSchemaCompatibilityWithNewFields() {
+		final PojoSerializerSnapshot<TestPojo> testSnapshot = buildTestSnapshot(Collections.singletonList(HEIGHT_FIELD));
+
+		final PojoSerializer<TestPojo> newPojoSerializer = buildTestNewPojoSerializer(Arrays.asList(
+			ID_FIELD,
+			NAME_FIELD,
+			HEIGHT_FIELD
+		));
+
+		final TypeSerializerSchemaCompatibility<TestPojo> resultCompatibility =
+			testSnapshot.resolveSchemaCompatibility(newPojoSerializer);
+
+		assertTrue(resultCompatibility.isCompatibleAfterMigration());
+	}
+
+	@Test
+	public void testResolveSchemaCompatibilityWithNewAndRemovedFields() {
+		final PojoSerializerSnapshot<TestPojo> testSnapshot = buildTestSnapshot(
+			Collections.singletonList(mockRemovedField(ID_FIELD)));
+
+		final PojoSerializer<TestPojo> newPojoSerializer = buildTestNewPojoSerializer(Arrays.asList(
+			NAME_FIELD,
+			HEIGHT_FIELD
+		));
+
+		final TypeSerializerSchemaCompatibility<TestPojo> resultCompatibility =
+			testSnapshot.resolveSchemaCompatibility(newPojoSerializer);
+
+		assertTrue(resultCompatibility.isCompatibleAfterMigration());
+	}
+
+	@Test
+	public void testResolveSchemaCompatibilityWithIncompatibleFieldSerializers() {
+		final PojoSerializerSnapshot<TestPojo> testSnapshot = buildTestSnapshot(Arrays.asList(
+			ID_FIELD,
+			mockFieldSerializerSnapshot(
+				NAME_FIELD,
+				new SchemaCompatibilityTestingSerializer.InitialSerializer<>().snapshotConfiguration()),
+			HEIGHT_FIELD
+		));
+
+		final PojoSerializer<TestPojo> newPojoSerializer = buildTestNewPojoSerializer(Arrays.asList(
+			ID_FIELD,
+			mockFieldSerializer(NAME_FIELD, new SchemaCompatibilityTestingSerializer.IncompatibleSerializer<>()),
+			HEIGHT_FIELD
+		));
+
+		final TypeSerializerSchemaCompatibility<TestPojo> resultCompatibility =
+			testSnapshot.resolveSchemaCompatibility(newPojoSerializer);
+
+		assertTrue(resultCompatibility.isIncompatible());
+	}
+
+	@Test
+	public void testResolveSchemaCompatibilityWithCompatibleAfterMigrationFieldSerializers() {
+		final PojoSerializerSnapshot<TestPojo> testSnapshot = buildTestSnapshot(Arrays.asList(
+			ID_FIELD,
+			NAME_FIELD,
+			mockFieldSerializerSnapshot(
+				HEIGHT_FIELD,
+				new SchemaCompatibilityTestingSerializer.InitialSerializer<>().snapshotConfiguration())
+		));
+
+		final PojoSerializer<TestPojo> newPojoSerializer = buildTestNewPojoSerializer(Arrays.asList(
+			ID_FIELD,
+			NAME_FIELD,
+			mockFieldSerializer(HEIGHT_FIELD, new SchemaCompatibilityTestingSerializer.UpgradedSchemaSerializer<>())
+		));
+
+		final TypeSerializerSchemaCompatibility<TestPojo> resultCompatibility =
+			testSnapshot.resolveSchemaCompatibility(newPojoSerializer);
+
+		assertTrue(resultCompatibility.isCompatibleAfterMigration());
+	}
+
+	@Test
+	public void testResolveSchemaCompatibilityWithCompatibleWithReconfigurationFieldSerializers() {
+		final PojoSerializerSnapshot<TestPojo> testSnapshot = buildTestSnapshot(Arrays.asList(
+			mockFieldSerializerSnapshot(
+				ID_FIELD,
+				new SchemaCompatibilityTestingSerializer.InitialSerializer<>().snapshotConfiguration()),
+			NAME_FIELD,
+			HEIGHT_FIELD
+		));
+
+		final PojoSerializer<TestPojo> newPojoSerializer = buildTestNewPojoSerializer(Arrays.asList(
+			mockFieldSerializer(ID_FIELD, new SchemaCompatibilityTestingSerializer.ReconfigurationRequiringSerializer<>()),
+			NAME_FIELD,
+			HEIGHT_FIELD
+		));
+
+		final TypeSerializerSchemaCompatibility<TestPojo> resultCompatibility =
+			testSnapshot.resolveSchemaCompatibility(newPojoSerializer);
+
+		assertTrue(resultCompatibility.isCompatibleWithReconfiguredSerializer());
+
+		final TypeSerializer<TestPojo> reconfiguredSerializer = resultCompatibility.getReconfiguredSerializer();
+		assertTrue(reconfiguredSerializer.getClass() == PojoSerializer.class);
+		final PojoSerializer<TestPojo> reconfiguredPojoSerializer = (PojoSerializer<TestPojo>) reconfiguredSerializer;
+
+		final TypeSerializer<?>[] reconfiguredFieldSerializers = reconfiguredPojoSerializer.getFieldSerializers();
+		assertArrayEquals(
+			new TypeSerializer[] {
+				new SchemaCompatibilityTestingSerializer.InitialSerializer(),
+				StringSerializer.INSTANCE,
+				DoubleSerializer.INSTANCE },
+			reconfiguredFieldSerializers);
+	}
+
+	// ------------------------------------------------------------------------------------------------
+	//  Test utilities
+	// ------------------------------------------------------------------------------------------------
+
+	private static PojoSerializerSnapshot<TestPojo> buildTestSnapshot(List<TestPojoField> fieldsToContainInSnapshot) {
+
+		int numFields = fieldsToContainInSnapshot.size();
+		ArrayList<Field> fields = new ArrayList<>(numFields);
+		ArrayList<TypeSerializerSnapshot<?>> fieldSerializerSnapshots = new ArrayList<>(numFields);
+		fieldsToContainInSnapshot.forEach(testPojoField -> {
+			fields.add(testPojoField.field);
+			fieldSerializerSnapshots.add(testPojoField.serializerSnapshot);
+		});
+
+		return new PojoSerializerSnapshot<>(
+			TestPojo.class,
+			fields.toArray(new Field[numFields]),
+			fieldSerializerSnapshots.toArray(new TypeSerializerSnapshot[numFields]),
+			new LinkedHashMap<>(),
+			new LinkedHashMap<>());
+	}
+
+	private static PojoSerializer<TestPojo> buildTestNewPojoSerializer(List<TestPojoField> fieldsForNewPojo) {
+		int numFields = fieldsForNewPojo.size();
+
+		final ArrayList<Field> fields = new ArrayList<>(numFields);
+		final ArrayList<TypeSerializer<?>> fieldSerializers = new ArrayList<>(numFields);
+		fieldsForNewPojo.forEach(fieldForNewPojo -> {
+			fields.add(fieldForNewPojo.field);
+			fieldSerializers.add(fieldForNewPojo.serializer);
+		});
+
+		return new PojoSerializer<>(
+			TestPojo.class,
+			fieldSerializers.toArray(new TypeSerializer[numFields]),
+			fields.toArray(new Field[numFields]),
+			new ExecutionConfig());
+	}
+
+	private static TestPojoField mockRemovedField(TestPojoField original) {
+		TestPojoField copy = original.shallowCopy();
+		copy.field = null;
+		return copy;
+	}
+
+	private static TestPojoField mockFieldSerializer(TestPojoField original, TypeSerializer<?> mockSerializer) {
+		TestPojoField copy = original.shallowCopy();
+		copy.serializer = mockSerializer;
+		return copy;
+	}
+
+	private static TestPojoField mockFieldSerializerSnapshot(TestPojoField original, TypeSerializerSnapshot<?> mockSnapshot) {
+		TestPojoField copy = original.shallowCopy();
+		copy.serializerSnapshot = mockSnapshot;
+		return copy;
+	}
+}


[flink] 06/17: [FLINK-11323] Migrate ValueSerializer to use new serialization compatibility abstractions

Posted by tz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tzulitai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 4fa828810c9c9eeada8ba64e294905e06a90b63a
Author: Igal Shilman <ig...@data-artisans.com>
AuthorDate: Sat Feb 9 21:12:54 2019 +0100

    [FLINK-11323] Migrate ValueSerializer to use new serialization compatibility abstractions
---
 .../java/typeutils/runtime/ValueSerializer.java    | 106 +++++++++++++--------
 .../runtime/ValueSerializerMigrationTest.java      |  88 +++++++++++++++++
 .../test/resources/flink-1.6-value-serializer-data | Bin 0 -> 150 bytes
 .../resources/flink-1.6-value-serializer-snapshot  | Bin 0 -> 1310 bytes
 .../test/resources/flink-1.7-value-serializer-data | Bin 0 -> 150 bytes
 .../resources/flink-1.7-value-serializer-snapshot  | Bin 0 -> 1298 bytes
 6 files changed, 155 insertions(+), 39 deletions(-)

diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/ValueSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/ValueSerializer.java
index 5758f3f..edeb4c9 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/ValueSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/ValueSerializer.java
@@ -18,23 +18,23 @@
 
 package org.apache.flink.api.java.typeutils.runtime;
 
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.util.LinkedHashMap;
-
 import org.apache.flink.annotation.Internal;
-import org.apache.flink.api.common.typeutils.CompatibilityResult;
+import org.apache.flink.api.common.typeutils.GenericTypeSerializerSnapshot;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
+import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
+import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 import org.apache.flink.types.Value;
 import org.apache.flink.util.InstantiationUtil;
 
 import com.esotericsoftware.kryo.Kryo;
-import org.apache.flink.util.Preconditions;
 import org.objenesis.strategy.StdInstantiatorStrategy;
 
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.util.LinkedHashMap;
+
 import static org.apache.flink.util.Preconditions.checkNotNull;
 
 /**
@@ -47,7 +47,7 @@ import static org.apache.flink.util.Preconditions.checkNotNull;
 public final class ValueSerializer<T extends Value> extends TypeSerializer<T> {
 
 	private static final long serialVersionUID = 1L;
-	
+
 	private final Class<T> type;
 
 	/**
@@ -62,18 +62,18 @@ public final class ValueSerializer<T extends Value> extends TypeSerializer<T> {
 	private LinkedHashMap<String, KryoRegistration> kryoRegistrations;
 
 	private transient Kryo kryo;
-	
+
 	private transient T copyInstance;
-	
+
 	// --------------------------------------------------------------------------------------------
-	
+
 	public ValueSerializer(Class<T> type) {
 		this.type = checkNotNull(type);
 		this.kryoRegistrations = asKryoRegistrations(type);
 	}
 
 	// --------------------------------------------------------------------------------------------
-	
+
 	@Override
 	public boolean isImmutableType() {
 		return false;
@@ -83,7 +83,7 @@ public final class ValueSerializer<T extends Value> extends TypeSerializer<T> {
 	public ValueSerializer<T> duplicate() {
 		return new ValueSerializer<T>(type);
 	}
-	
+
 	@Override
 	public T createInstance() {
 		return InstantiationUtil.instantiate(this.type);
@@ -95,7 +95,7 @@ public final class ValueSerializer<T extends Value> extends TypeSerializer<T> {
 
 		return KryoUtils.copy(from, kryo, this);
 	}
-	
+
 	@Override
 	public T copy(T from, T reuse) {
 		checkKryoInitialized();
@@ -117,7 +117,7 @@ public final class ValueSerializer<T extends Value> extends TypeSerializer<T> {
 	public T deserialize(DataInputView source) throws IOException {
 		return deserialize(createInstance(), source);
 	}
-	
+
 	@Override
 	public T deserialize(T reuse, DataInputView source) throws IOException {
 		reuse.read(source);
@@ -129,11 +129,11 @@ public final class ValueSerializer<T extends Value> extends TypeSerializer<T> {
 		if (this.copyInstance == null) {
 			this.copyInstance = InstantiationUtil.instantiate(type);
 		}
-		
+
 		this.copyInstance.read(source);
 		this.copyInstance.write(target);
 	}
-	
+
 	private void checkKryoInitialized() {
 		if (this.kryo == null) {
 			this.kryo = new Kryo();
@@ -147,14 +147,14 @@ public final class ValueSerializer<T extends Value> extends TypeSerializer<T> {
 			KryoUtils.applyRegistrations(this.kryo, kryoRegistrations.values());
 		}
 	}
-	
+
 	// --------------------------------------------------------------------------------------------
-	
+
 	@Override
 	public int hashCode() {
 		return this.type.hashCode();
 	}
-	
+
 	@Override
 	public boolean equals(Object obj) {
 		if (obj instanceof ValueSerializer) {
@@ -166,33 +166,24 @@ public final class ValueSerializer<T extends Value> extends TypeSerializer<T> {
 		}
 	}
 
+	private  Class<T> getValueType() {
+		return type;
+	}
+
 	// --------------------------------------------------------------------------------------------
 	// Serializer configuration snapshotting & compatibility
 	// --------------------------------------------------------------------------------------------
 
 	@Override
-	public ValueSerializerConfigSnapshot<T> snapshotConfiguration() {
-		return new ValueSerializerConfigSnapshot<>(type);
-	}
-
-	@SuppressWarnings("unchecked")
-	@Override
-	public CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot<?> configSnapshot) {
-		if (configSnapshot instanceof ValueSerializerConfigSnapshot) {
-			final ValueSerializerConfigSnapshot<T> config = (ValueSerializerConfigSnapshot<T>) configSnapshot;
-
-			if (type.equals(config.getTypeClass())) {
-				// currently, simply checking the type of the value class is sufficient;
-				// in the future, if there are more Kryo registrations, we should try to resolve that
-				return CompatibilityResult.compatible();
-			}
-		}
-
-		return CompatibilityResult.requiresMigration();
+	public TypeSerializerSnapshot<T> snapshotConfiguration() {
+		return new ValueSerializerSnapshot<>(type);
 	}
 
+	@Deprecated
 	public static class ValueSerializerConfigSnapshot<T extends Value> extends KryoRegistrationSerializerConfigSnapshot<T> {
 
+		public static final long serialVersionUID = 2277251654485371327L;
+
 		private static final int VERSION = 1;
 
 		/** This empty nullary constructor is required for deserializing the configuration. */
@@ -206,6 +197,43 @@ public final class ValueSerializer<T extends Value> extends TypeSerializer<T> {
 		public int getVersion() {
 			return VERSION;
 		}
+
+		@Override
+		public TypeSerializerSchemaCompatibility<T> resolveSchemaCompatibility(TypeSerializer<T> newSerializer) {
+			return new ValueSerializerSnapshot<>(getTypeClass())
+				.resolveSchemaCompatibility(newSerializer);
+		}
+	}
+
+	/**
+	 * {@link ValueSerializer} snapshot class.
+	 */
+	public static final class ValueSerializerSnapshot<T extends Value>
+		extends GenericTypeSerializerSnapshot<T, ValueSerializer> {
+
+		@SuppressWarnings("unused")
+		public ValueSerializerSnapshot() {
+		}
+
+		ValueSerializerSnapshot(Class<T> typeClass) {
+			super(typeClass);
+		}
+
+		@Override
+		protected TypeSerializer<T> createSerializer(Class<T> typeClass) {
+			return new ValueSerializer<>(typeClass);
+		}
+
+		@SuppressWarnings("unchecked")
+		@Override
+		protected Class<T> getTypeClass(ValueSerializer serializer) {
+			return serializer.type;
+		}
+
+		@Override
+		protected Class<?> serializerClass() {
+			return ValueSerializer.class;
+		}
 	}
 
 	// --------------------------------------------------------------------------------------------
@@ -220,7 +248,7 @@ public final class ValueSerializer<T extends Value> extends TypeSerializer<T> {
 	}
 
 	private static LinkedHashMap<String, KryoRegistration> asKryoRegistrations(Class<?> type) {
-		Preconditions.checkNotNull(type);
+		checkNotNull(type);
 
 		LinkedHashMap<String, KryoRegistration> registration = new LinkedHashMap<>(1);
 		registration.put(type.getClass().getName(), new KryoRegistration(type));
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/ValueSerializerMigrationTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/ValueSerializerMigrationTest.java
new file mode 100644
index 0000000..55a1bae
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/ValueSerializerMigrationTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.typeutils.runtime;
+
+import org.apache.flink.api.common.typeutils.TypeSerializerSnapshotMigrationTestBase;
+import org.apache.flink.api.java.typeutils.runtime.ValueSerializer.ValueSerializerSnapshot;
+import org.apache.flink.api.java.typeutils.runtime.ValueSerializerMigrationTest.NameValue;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.testutils.migration.MigrationVersion;
+import org.apache.flink.types.Value;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.io.IOException;
+import java.util.Collection;
+
+/**
+ * State migration test for {@link RowSerializer}.
+ */
+@RunWith(Parameterized.class)
+public class ValueSerializerMigrationTest extends TypeSerializerSnapshotMigrationTestBase<NameValue> {
+
+	public ValueSerializerMigrationTest(TestSpecification<NameValue> testSpecification) {
+		super(testSpecification);
+	}
+
+	@SuppressWarnings("unchecked")
+	@Parameterized.Parameters(name = "Test Specification = {0}")
+	public static Collection<TestSpecification<?>> testSpecifications() {
+
+		TestSpecifications testSpecifications = new TestSpecifications(MigrationVersion.v1_6, MigrationVersion.v1_7);
+
+		testSpecifications.add(
+			"value-serializer",
+			ValueSerializer.class,
+			ValueSerializerSnapshot.class,
+			() -> new ValueSerializer<>(NameValue.class));
+
+		return testSpecifications.get();
+	}
+
+	/**
+	 * A dummy class used for this test.
+	 */
+	public static final class NameValue implements Value {
+
+		public static final long serialVersionUID = 2277251654485371327L;
+
+		private String name;
+
+		public String getName() {
+			return name;
+		}
+
+		public void setName(String name) {
+			this.name = name;
+		}
+
+		@Override
+		public void write(DataOutputView out) throws IOException {
+			out.writeUTF(name);
+		}
+
+		@Override
+		public void read(DataInputView in) throws IOException {
+			name = in.readUTF();
+		}
+	}
+
+}
diff --git a/flink-core/src/test/resources/flink-1.6-value-serializer-data b/flink-core/src/test/resources/flink-1.6-value-serializer-data
new file mode 100644
index 0000000..f2bb149
Binary files /dev/null and b/flink-core/src/test/resources/flink-1.6-value-serializer-data differ
diff --git a/flink-core/src/test/resources/flink-1.6-value-serializer-snapshot b/flink-core/src/test/resources/flink-1.6-value-serializer-snapshot
new file mode 100644
index 0000000..90cbdcf
Binary files /dev/null and b/flink-core/src/test/resources/flink-1.6-value-serializer-snapshot differ
diff --git a/flink-core/src/test/resources/flink-1.7-value-serializer-data b/flink-core/src/test/resources/flink-1.7-value-serializer-data
new file mode 100644
index 0000000..cbd0d18
Binary files /dev/null and b/flink-core/src/test/resources/flink-1.7-value-serializer-data differ
diff --git a/flink-core/src/test/resources/flink-1.7-value-serializer-snapshot b/flink-core/src/test/resources/flink-1.7-value-serializer-snapshot
new file mode 100644
index 0000000..5194920
Binary files /dev/null and b/flink-core/src/test/resources/flink-1.7-value-serializer-snapshot differ


[flink] 09/17: [FLINK-11485] [core] Rename OptionalMap to LinkedOptionalMap

Posted by tz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tzulitai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit cf7b86de436c8714414f563e8637ceb36ea7aabe
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
AuthorDate: Thu Jan 31 17:31:58 2019 +0800

    [FLINK-11485] [core] Rename OptionalMap to LinkedOptionalMap
    
    Having "Linked" in the name better reflects the fact that the map
    preserves entry ordering.
---
 .../runtime/kryo/KryoSerializerSnapshot.java       |  6 +--
 .../runtime/kryo/KryoSerializerSnapshotData.java   | 49 +++++++++++----------
 .../{OptionalMap.java => LinkedOptionalMap.java}   | 37 ++++++++--------
 ...onalMapTest.java => LinkedOptionalMapTest.java} | 51 +++++++++++-----------
 4 files changed, 72 insertions(+), 71 deletions(-)

diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshot.java
index 77242e6..b49fa35 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshot.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshot.java
@@ -25,7 +25,7 @@ import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
 import org.apache.flink.api.java.typeutils.runtime.KryoRegistration;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
-import org.apache.flink.util.OptionalMap.MergeResult;
+import org.apache.flink.util.LinkedOptionalMap.MergeResult;
 
 import com.esotericsoftware.kryo.Serializer;
 import org.slf4j.Logger;
@@ -36,8 +36,8 @@ import java.util.LinkedHashMap;
 import java.util.function.Function;
 
 import static org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializerSnapshotData.createFrom;
-import static org.apache.flink.util.OptionalMap.mergeRightIntoLeft;
-import static org.apache.flink.util.OptionalMap.optionalMapOf;
+import static org.apache.flink.util.LinkedOptionalMap.mergeRightIntoLeft;
+import static org.apache.flink.util.LinkedOptionalMap.optionalMapOf;
 
 /**
  * {@link TypeSerializerSnapshot} for {@link KryoSerializer}.
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshotData.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshotData.java
index 40b12a4..f04c8e8 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshotData.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshotData.java
@@ -25,7 +25,7 @@ import org.apache.flink.api.java.typeutils.runtime.KryoRegistration;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 import org.apache.flink.util.InstantiationUtil;
-import org.apache.flink.util.OptionalMap;
+import org.apache.flink.util.LinkedOptionalMap;
 
 import com.esotericsoftware.kryo.Serializer;
 import org.slf4j.Logger;
@@ -37,7 +37,7 @@ import java.util.LinkedHashMap;
 import java.util.Map.Entry;
 import java.util.function.Function;
 
-import static org.apache.flink.util.OptionalMap.optionalMapOf;
+import static org.apache.flink.util.LinkedOptionalMap.optionalMapOf;
 import static org.apache.flink.util.Preconditions.checkNotNull;
 
 final class KryoSerializerSnapshotData<T> {
@@ -63,9 +63,9 @@ final class KryoSerializerSnapshotData<T> {
 
 	static <T> KryoSerializerSnapshotData<T> createFrom(DataInputView in, ClassLoader cl) throws IOException {
 		Class<T> typeClass = readTypeClass(in, cl);
-		OptionalMap<String, KryoRegistration> kryoRegistrations = readKryoRegistrations(in, cl);
-		OptionalMap<Class<?>, SerializableSerializer<?>> defaultSerializer = readDefaultKryoSerializers(in, cl);
-		OptionalMap<Class<?>, Class<? extends Serializer<?>>> defaultSerializerClasses =
+		LinkedOptionalMap<String, KryoRegistration> kryoRegistrations = readKryoRegistrations(in, cl);
+		LinkedOptionalMap<Class<?>, SerializableSerializer<?>> defaultSerializer = readDefaultKryoSerializers(in, cl);
+		LinkedOptionalMap<Class<?>, Class<? extends Serializer<?>>> defaultSerializerClasses =
 			readDefaultKryoSerializerClasses(in, cl);
 
 		return new KryoSerializerSnapshotData<>(
@@ -80,14 +80,15 @@ final class KryoSerializerSnapshotData<T> {
 	// --------------------------------------------------------------------------------------------
 
 	private final Class<T> typeClass;
-	private final OptionalMap<Class<?>, SerializableSerializer<?>> defaultKryoSerializers;
-	private final OptionalMap<Class<?>, Class<? extends Serializer<?>>> defaultKryoSerializerClasses;
-	private final OptionalMap<String, KryoRegistration> kryoRegistrations;
+	private final LinkedOptionalMap<Class<?>, SerializableSerializer<?>> defaultKryoSerializers;
+	private final LinkedOptionalMap<Class<?>, Class<? extends Serializer<?>>> defaultKryoSerializerClasses;
+	private final LinkedOptionalMap<String, KryoRegistration> kryoRegistrations;
 
-	private KryoSerializerSnapshotData(Class<T> typeClass,
-			OptionalMap<Class<?>, SerializableSerializer<?>> defaultKryoSerializers,
-			OptionalMap<Class<?>, Class<? extends Serializer<?>>> defaultKryoSerializerClasses,
-			OptionalMap<String, KryoRegistration> kryoRegistrations) {
+	private KryoSerializerSnapshotData(
+		Class<T> typeClass,
+		LinkedOptionalMap<Class<?>, SerializableSerializer<?>> defaultKryoSerializers,
+		LinkedOptionalMap<Class<?>, Class<? extends Serializer<?>>> defaultKryoSerializerClasses,
+		LinkedOptionalMap<String, KryoRegistration> kryoRegistrations) {
 
 		this.typeClass = typeClass;
 		this.defaultKryoSerializers = defaultKryoSerializers;
@@ -103,15 +104,15 @@ final class KryoSerializerSnapshotData<T> {
 		return typeClass;
 	}
 
-	OptionalMap<Class<?>, SerializableSerializer<?>> getDefaultKryoSerializers() {
+	LinkedOptionalMap<Class<?>, SerializableSerializer<?>> getDefaultKryoSerializers() {
 		return defaultKryoSerializers;
 	}
 
-	OptionalMap<Class<?>, Class<? extends Serializer<?>>> getDefaultKryoSerializerClasses() {
+	LinkedOptionalMap<Class<?>, Class<? extends Serializer<?>>> getDefaultKryoSerializerClasses() {
 		return defaultKryoSerializerClasses;
 	}
 
-	OptionalMap<String, KryoRegistration> getKryoRegistrations() {
+	LinkedOptionalMap<String, KryoRegistration> getKryoRegistrations() {
 		return kryoRegistrations;
 	}
 
@@ -132,7 +133,7 @@ final class KryoSerializerSnapshotData<T> {
 
 	private static void writeKryoRegistrations(
 		DataOutputView out,
-		OptionalMap<String, KryoRegistration> kryoRegistrations) throws IOException {
+		LinkedOptionalMap<String, KryoRegistration> kryoRegistrations) throws IOException {
 
 		out.writeInt(kryoRegistrations.size());
 		for (Entry<String, KryoRegistration> entry : kryoRegistrations.unwrapOptionals().entrySet()) {
@@ -143,7 +144,7 @@ final class KryoSerializerSnapshotData<T> {
 
 	private void writeDefaultKryoSerializers(
 		DataOutputView out,
-		OptionalMap<Class<?>,
+		LinkedOptionalMap<Class<?>,
 			SerializableSerializer<?>> defaultKryoSerializers) throws IOException {
 
 		out.writeInt(defaultKryoSerializers.size());
@@ -160,7 +161,7 @@ final class KryoSerializerSnapshotData<T> {
 
 	private static void writeDefaultKryoSerializerClasses(
 		DataOutputView out,
-		OptionalMap<Class<?>, Class<? extends Serializer<?>>> defaultKryoSerializerClasses)
+		LinkedOptionalMap<Class<?>, Class<? extends Serializer<?>>> defaultKryoSerializerClasses)
 		throws IOException {
 
 		out.writeInt(defaultKryoSerializerClasses.size());
@@ -181,11 +182,11 @@ final class KryoSerializerSnapshotData<T> {
 		return InstantiationUtil.resolveClassByName(in, userCodeClassLoader);
 	}
 
-	private static OptionalMap<String, KryoRegistration> readKryoRegistrations(
+	private static LinkedOptionalMap<String, KryoRegistration> readKryoRegistrations(
 		DataInputView in,
 		ClassLoader userCodeClassLoader) throws IOException {
 
-		OptionalMap<String, KryoRegistration> registrations = new OptionalMap<>();
+		LinkedOptionalMap<String, KryoRegistration> registrations = new LinkedOptionalMap<>();
 		final int size = in.readInt();
 		for (int i = 0; i < size; i++) {
 			final String name = in.readUTF();
@@ -198,8 +199,8 @@ final class KryoSerializerSnapshotData<T> {
 		return registrations;
 	}
 
-	private static OptionalMap<Class<?>, SerializableSerializer<?>> readDefaultKryoSerializers(DataInputView in, ClassLoader cl) throws IOException {
-		OptionalMap<Class<?>, SerializableSerializer<?>> kryoSerializers = new OptionalMap<>();
+	private static LinkedOptionalMap<Class<?>, SerializableSerializer<?>> readDefaultKryoSerializers(DataInputView in, ClassLoader cl) throws IOException {
+		LinkedOptionalMap<Class<?>, SerializableSerializer<?>> kryoSerializers = new LinkedOptionalMap<>();
 		final int size = in.readInt();
 		for (int i = 0; i < size; i++) {
 			final String className = in.readUTF();
@@ -225,11 +226,11 @@ final class KryoSerializerSnapshotData<T> {
 	}
 
 	@SuppressWarnings("unchecked")
-	private static OptionalMap<Class<?>, Class<? extends Serializer<?>>> readDefaultKryoSerializerClasses(
+	private static LinkedOptionalMap<Class<?>, Class<? extends Serializer<?>>> readDefaultKryoSerializerClasses(
 		DataInputView in,
 		ClassLoader cl) throws IOException {
 
-		OptionalMap<Class<?>, Class<? extends Serializer<?>>> kryoSerializerClasses = new OptionalMap<>();
+		LinkedOptionalMap<Class<?>, Class<? extends Serializer<?>>> kryoSerializerClasses = new LinkedOptionalMap<>();
 		final int size = in.readInt();
 		for (int i = 0; i < size; i++) {
 			final String className = in.readUTF();
diff --git a/flink-core/src/main/java/org/apache/flink/util/OptionalMap.java b/flink-core/src/main/java/org/apache/flink/util/LinkedOptionalMap.java
similarity index 84%
rename from flink-core/src/main/java/org/apache/flink/util/OptionalMap.java
rename to flink-core/src/main/java/org/apache/flink/util/LinkedOptionalMap.java
index 4552102..0697ec6 100644
--- a/flink-core/src/main/java/org/apache/flink/util/OptionalMap.java
+++ b/flink-core/src/main/java/org/apache/flink/util/LinkedOptionalMap.java
@@ -18,6 +18,7 @@
 
 package org.apache.flink.util;
 
+import org.apache.flink.annotation.Internal;
 import org.apache.flink.annotation.VisibleForTesting;
 
 import java.util.Iterator;
@@ -33,18 +34,18 @@ import javax.annotation.Nullable;
 import static org.apache.flink.util.Preconditions.checkNotNull;
 
 /**
- * An OptionalMap is an order preserving map (like {@link LinkedHashMap}) where keys have a unique string name, but are
+ * A LinkedOptionalMap is an order preserving map (like {@link LinkedHashMap}) where keys have a unique string name, but are
  * optionally present, and the values are optional.
  */
-public final class OptionalMap<K, V> {
-
+@Internal
+public final class LinkedOptionalMap<K, V> {
 
 	// --------------------------------------------------------------------------------------------------------
 	// Factory
 	// --------------------------------------------------------------------------------------------------------
 
 	/**
-	 * Creates an {@code OptionalMap} from the provided map.
+	 * Creates an {@code LinkedOptionalMap} from the provided map.
 	 *
 	 * <p>This method is the equivalent of {@link Optional#of(Object)} but for maps. To support more than one {@code NULL}
 	 * key, an optional map requires a unique string name to be associated with each key (provided by keyNameGetter)
@@ -53,9 +54,9 @@ public final class OptionalMap<K, V> {
 	 * @param keyNameGetter function that assigns a unique name to the keys of the source map.
 	 * @param <K>           key type
 	 * @param <V>           value type
-	 * @return an {@code OptionalMap} with optional named keys, and optional values.
+	 * @return an {@code LinkedOptionalMap} with optional named keys, and optional values.
 	 */
-	public static <K, V> OptionalMap<K, V> optionalMapOf(LinkedHashMap<K, V> sourceMap, Function<K, String> keyNameGetter) {
+	public static <K, V> LinkedOptionalMap<K, V> optionalMapOf(LinkedHashMap<K, V> sourceMap, Function<K, String> keyNameGetter) {
 
 		LinkedHashMap<String, KeyValue<K, V>> underlyingMap = new LinkedHashMap<>(sourceMap.size());
 
@@ -64,14 +65,14 @@ public final class OptionalMap<K, V> {
 			underlyingMap.put(keyName, new KeyValue<>(k, v));
 		});
 
-		return new OptionalMap<>(underlyingMap);
+		return new LinkedOptionalMap<>(underlyingMap);
 	}
 
 	/**
 	 * Tries to merges the keys and the values of @right into @left.
 	 */
-	public static <K, V> MergeResult<K, V> mergeRightIntoLeft(OptionalMap<K, V> left, OptionalMap<K, V> right) {
-		OptionalMap<K, V> merged = new OptionalMap<>(left);
+	public static <K, V> MergeResult<K, V> mergeRightIntoLeft(LinkedOptionalMap<K, V> left, LinkedOptionalMap<K, V> right) {
+		LinkedOptionalMap<K, V> merged = new LinkedOptionalMap<>(left);
 		merged.putAll(right);
 
 		return new MergeResult<>(merged, isLeftPrefixOfRight(left, right));
@@ -83,16 +84,16 @@ public final class OptionalMap<K, V> {
 
 	private final LinkedHashMap<String, KeyValue<K, V>> underlyingMap;
 
-	public OptionalMap() {
+	public LinkedOptionalMap() {
 		this(new LinkedHashMap<>());
 	}
 
 	@SuppressWarnings("CopyConstructorMissesField")
-	OptionalMap(OptionalMap<K, V> optionalMap) {
-		this(new LinkedHashMap<>(optionalMap.underlyingMap));
+	LinkedOptionalMap(LinkedOptionalMap<K, V> linkedOptionalMap) {
+		this(new LinkedHashMap<>(linkedOptionalMap.underlyingMap));
 	}
 
-	private OptionalMap(LinkedHashMap<String, KeyValue<K, V>> underlyingMap) {
+	private LinkedOptionalMap(LinkedHashMap<String, KeyValue<K, V>> underlyingMap) {
 		this.underlyingMap = checkNotNull(underlyingMap);
 	}
 
@@ -111,7 +112,7 @@ public final class OptionalMap<K, V> {
 			(kv == null) ? new KeyValue<>(key, value) : kv.merge(key, value));
 	}
 
-	void putAll(OptionalMap<K, V> right) {
+	void putAll(LinkedOptionalMap<K, V> right) {
 		for (Entry<String, KeyValue<K, V>> entry : right.underlyingMap.entrySet()) {
 			KeyValue<K, V> kv = entry.getValue();
 			this.put(entry.getKey(), kv.key, kv.value);
@@ -124,7 +125,7 @@ public final class OptionalMap<K, V> {
 	public Set<String> absentKeysOrValues() {
 		return underlyingMap.entrySet()
 			.stream()
-			.filter(OptionalMap::keyOrValueIsAbsent)
+			.filter(LinkedOptionalMap::keyOrValueIsAbsent)
 			.map(Entry::getKey)
 			.collect(Collectors.toSet());
 	}
@@ -169,7 +170,7 @@ public final class OptionalMap<K, V> {
 	}
 
 	@VisibleForTesting
-	static <K, V> boolean isLeftPrefixOfRight(OptionalMap<K, V> left, OptionalMap<K, V> right) {
+	static <K, V> boolean isLeftPrefixOfRight(LinkedOptionalMap<K, V> left, LinkedOptionalMap<K, V> right) {
 		Iterator<String> rightKeys = right.keyNames().iterator();
 
 		for (String leftKey : left.keyNames()) {
@@ -220,11 +221,11 @@ public final class OptionalMap<K, V> {
 	 * {@link #mergeRightIntoLeft(LinkedOptionalMap, LinkedOptionalMap)}.
 	 */
 	public static final class MergeResult<K, V> {
-		private final OptionalMap<K, V> merged;
+		private final LinkedOptionalMap<K, V> merged;
 		private final Set<String> missingKeys;
 		private final boolean isOrderedSubset;
 
-		MergeResult(OptionalMap<K, V> merged, boolean isOrderedSubset) {
+		MergeResult(LinkedOptionalMap<K, V> merged, boolean isOrderedSubset) {
 			this.merged = merged;
 			this.missingKeys = merged.absentKeysOrValues();
 			this.isOrderedSubset = isOrderedSubset;
diff --git a/flink-core/src/test/java/org/apache/flink/util/OptionalMapTest.java b/flink-core/src/test/java/org/apache/flink/util/LinkedOptionalMapTest.java
similarity index 74%
rename from flink-core/src/test/java/org/apache/flink/util/OptionalMapTest.java
rename to flink-core/src/test/java/org/apache/flink/util/LinkedOptionalMapTest.java
index 050dba8..a12bf6c 100644
--- a/flink-core/src/test/java/org/apache/flink/util/OptionalMapTest.java
+++ b/flink-core/src/test/java/org/apache/flink/util/LinkedOptionalMapTest.java
@@ -18,7 +18,7 @@
 
 package org.apache.flink.util;
 
-import org.apache.flink.util.OptionalMap.MergeResult;
+import org.apache.flink.util.LinkedOptionalMap.MergeResult;
 
 import org.junit.Test;
 
@@ -34,13 +34,13 @@ import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 
 /**
- * Test {@link OptionalMap}.
+ * Test {@link LinkedOptionalMap}.
  */
-public class OptionalMapTest {
+public class LinkedOptionalMapTest {
 
 	@Test
 	public void usageExample() {
-		OptionalMap<Class<?>, String> map = new OptionalMap<>();
+		LinkedOptionalMap<Class<?>, String> map = new LinkedOptionalMap<>();
 
 		map.put("java.lang.String", String.class, "a string class");
 		map.put("scala.Option", null, "a scala Option");
@@ -52,7 +52,7 @@ public class OptionalMapTest {
 
 	@Test
 	public void overridingKeyWithTheSameKeyName() {
-		OptionalMap<Class<?>, String> map = new OptionalMap<>();
+		LinkedOptionalMap<Class<?>, String> map = new LinkedOptionalMap<>();
 
 		map.put("java.lang.String", null, "a string class");
 		map.put("java.lang.String", String.class, "a string class");
@@ -62,7 +62,7 @@ public class OptionalMapTest {
 
 	@Test
 	public void overridingKeysAndValuesWithTheSameKeyName() {
-		OptionalMap<Class<?>, String> map = new OptionalMap<>();
+		LinkedOptionalMap<Class<?>, String> map = new LinkedOptionalMap<>();
 
 		map.put("java.lang.String", null, null);
 		map.put("java.lang.String", String.class, "a string class");
@@ -72,7 +72,7 @@ public class OptionalMapTest {
 
 	@Test
 	public void overridingAValueWithMissingKeyShouldBeConsideredAsAbsent() {
-		OptionalMap<Class<?>, String> map = new OptionalMap<>();
+		LinkedOptionalMap<Class<?>, String> map = new LinkedOptionalMap<>();
 
 		map.put("java.lang.String", null, null);
 		map.put("java.lang.String", null, "a string class");
@@ -82,11 +82,11 @@ public class OptionalMapTest {
 
 	@Test
 	public void mergingMapsWithPresentEntriesLeavesNoAbsentKeyNames() {
-		OptionalMap<Class<?>, String> first = new OptionalMap<>();
+		LinkedOptionalMap<Class<?>, String> first = new LinkedOptionalMap<>();
 		first.put("b", null, null);
 		first.put("c", String.class, null);
 
-		OptionalMap<Class<?>, String> second = new OptionalMap<>();
+		LinkedOptionalMap<Class<?>, String> second = new LinkedOptionalMap<>();
 		second.put("a", String.class, "aaa");
 		second.put("b", String.class, "bbb");
 		second.put("c", Void.class, "ccc");
@@ -99,11 +99,11 @@ public class OptionalMapTest {
 
 	@Test
 	public void mergingMapsPreserversTheOrderOfTheOriginalMap() {
-		OptionalMap<Class<?>, String> first = new OptionalMap<>();
+		LinkedOptionalMap<Class<?>, String> first = new LinkedOptionalMap<>();
 		first.put("b", null, null);
 		first.put("c", String.class, null);
 
-		OptionalMap<Class<?>, String> second = new OptionalMap<>();
+		LinkedOptionalMap<Class<?>, String> second = new LinkedOptionalMap<>();
 		second.put("a", String.class, "aaa");
 		second.put("b", String.class, "bbb");
 		second.put("c", Void.class, "ccc");
@@ -116,9 +116,9 @@ public class OptionalMapTest {
 
 	@Test
 	public void mergingToEmpty() {
-		OptionalMap<Class<?>, String> first = new OptionalMap<>();
+		LinkedOptionalMap<Class<?>, String> first = new LinkedOptionalMap<>();
 
-		OptionalMap<Class<?>, String> second = new OptionalMap<>();
+		LinkedOptionalMap<Class<?>, String> second = new LinkedOptionalMap<>();
 		second.put("a", String.class, "aaa");
 		second.put("b", String.class, "bbb");
 		second.put("c", Void.class, "ccc");
@@ -131,7 +131,7 @@ public class OptionalMapTest {
 
 	@Test(expected = IllegalStateException.class)
 	public void unwrapOptionalsWithMissingValueThrows() {
-		OptionalMap<Class<?>, String> map = new OptionalMap<>();
+		LinkedOptionalMap<Class<?>, String> map = new LinkedOptionalMap<>();
 
 		map.put("a", String.class, null);
 
@@ -140,7 +140,7 @@ public class OptionalMapTest {
 
 	@Test(expected = IllegalStateException.class)
 	public void unwrapOptionalsWithMissingKeyThrows() {
-		OptionalMap<Class<?>, String> map = new OptionalMap<>();
+		LinkedOptionalMap<Class<?>, String> map = new LinkedOptionalMap<>();
 
 		map.put("a", null, "blabla");
 
@@ -149,7 +149,7 @@ public class OptionalMapTest {
 
 	@Test
 	public void unwrapOptionalsPreservesOrder() {
-		OptionalMap<Class<?>, String> map = new OptionalMap<>();
+		LinkedOptionalMap<Class<?>, String> map = new LinkedOptionalMap<>();
 
 		map.put("a", String.class, "aaa");
 		map.put("b", Boolean.class, "bbb");
@@ -162,48 +162,47 @@ public class OptionalMapTest {
 
 	@Test
 	public void testPrefix() {
-		OptionalMap<Class<?>, String> left = new OptionalMap<>();
+		LinkedOptionalMap<Class<?>, String> left = new LinkedOptionalMap<>();
 
 		left.put("a", String.class, "aaa");
 		left.put("b", String.class, "aaa");
 
-		OptionalMap<Class<?>, String> right = new OptionalMap<>(left);
+		LinkedOptionalMap<Class<?>, String> right = new LinkedOptionalMap<>(left);
 
 		right.put("c", Boolean.class, "bbb");
 
-		assertTrue(OptionalMap.isLeftPrefixOfRight(left, right));
+		assertTrue(LinkedOptionalMap.isLeftPrefixOfRight(left, right));
 	}
 
 	@Test
 	public void testNonPrefix() {
-		OptionalMap<Class<?>, String> left = new OptionalMap<>();
+		LinkedOptionalMap<Class<?>, String> left = new LinkedOptionalMap<>();
 
 		left.put("a", String.class, "aaa");
 		left.put("c", String.class, "aaa");
 
-		OptionalMap<Class<?>, String> right = new OptionalMap<>();
+		LinkedOptionalMap<Class<?>, String> right = new LinkedOptionalMap<>();
 
 		right.put("b", Boolean.class, "bbb");
 		right.put("c", Boolean.class, "bbb");
 
-
-		assertFalse(OptionalMap.isLeftPrefixOfRight(left, right));
+		assertFalse(LinkedOptionalMap.isLeftPrefixOfRight(left, right));
 	}
 
 	@Test
 	@SuppressWarnings("unchecked")
 	public void demoMergeResult() {
-		OptionalMap<Class<?>, String> left = new OptionalMap<>();
+		LinkedOptionalMap<Class<?>, String> left = new LinkedOptionalMap<>();
 		left.put("b", null, null);
 		left.put("c", String.class, null);
 
-		OptionalMap<Class<?>, String> right = new OptionalMap<>();
+		LinkedOptionalMap<Class<?>, String> right = new LinkedOptionalMap<>();
 		right.put("b", String.class, "bbb");
 		right.put("c", Void.class, "ccc");
 		right.put("a", Boolean.class, "aaa");
 		right.put("d", Long.class, "ddd");
 
-		MergeResult<Class<?>, String> result = OptionalMap.mergeRightIntoLeft(left, right);
+		MergeResult<Class<?>, String> result = LinkedOptionalMap.mergeRightIntoLeft(left, right);
 
 		assertThat(result.hasMissingKeys(), is(false));
 		assertThat(result.isOrderedSubset(), is(true));


[flink] 14/17: [hotfix] [core] LinkedOptionalMap should collect keys in LinkedHashSet to guarantee ordering

Posted by tz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tzulitai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 52b49c8c38b5c3bf86db723c0e4a125271709750
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
AuthorDate: Wed Feb 20 13:14:45 2019 +0800

    [hotfix] [core] LinkedOptionalMap should collect keys in LinkedHashSet to guarantee ordering
---
 flink-core/src/main/java/org/apache/flink/util/LinkedOptionalMap.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flink-core/src/main/java/org/apache/flink/util/LinkedOptionalMap.java b/flink-core/src/main/java/org/apache/flink/util/LinkedOptionalMap.java
index 30c7b54..a17dca4 100644
--- a/flink-core/src/main/java/org/apache/flink/util/LinkedOptionalMap.java
+++ b/flink-core/src/main/java/org/apache/flink/util/LinkedOptionalMap.java
@@ -134,7 +134,7 @@ public final class LinkedOptionalMap<K, V> {
 			.stream()
 			.filter(LinkedOptionalMap::keyOrValueIsAbsent)
 			.map(Entry::getKey)
-			.collect(Collectors.toSet());
+			.collect(Collectors.toCollection(LinkedHashSet::new));
 	}
 
 	/**


[flink] 04/17: [FLINK-11323] [tests] Add more test cases to TypeSerializerSnapshotMigrationTestBase

Posted by tz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tzulitai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 22a7eccacc6b5847ac96f791378ecf7bb25d03d0
Author: Igal Shilman <ig...@data-artisans.com>
AuthorDate: Thu Feb 21 13:02:04 2019 +0100

    [FLINK-11323] [tests] Add more test cases to TypeSerializerSnapshotMigrationTestBase
    
    This commit adds two more tests to TypeSerializerSnapshotMigrationTestBase
    that validates that the new serializer are able to read previously stored
    data (perhaps after reconfiguration).
---
 .../TypeSerializerSnapshotMigrationTestBase.java   | 51 +++++++++++++++++++---
 1 file changed, 46 insertions(+), 5 deletions(-)

diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerSnapshotMigrationTestBase.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerSnapshotMigrationTestBase.java
index f52e411..539e108 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerSnapshotMigrationTestBase.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerSnapshotMigrationTestBase.java
@@ -87,13 +87,21 @@ public abstract class TypeSerializerSnapshotMigrationTestBase<ElementT> extends
 		TypeSerializerSnapshot<ElementT> snapshot = snapshotUnderTest();
 		TypeSerializer<ElementT> serializer = snapshot.restoreSerializer();
 
-		DataInputView input = dataUnderTest();
+		assertSerializerIsAbleToReadOldData(serializer);
+	}
 
-		final Matcher<ElementT> matcher = testSpecification.testDataElementMatcher;
-		for (int i = 0; i < testSpecification.testDataCount; i++) {
-			final ElementT result = serializer.deserialize(input);
-			assertThat(result, matcher);
+	@Test
+	public void reconfiguredSerializerIsAbleToDeserializePreviousData() throws IOException {
+		TypeSerializerSnapshot<ElementT> snapshot = snapshotUnderTest();
+		TypeSerializerSchemaCompatibility<ElementT> compatibility = snapshot.resolveSchemaCompatibility(testSpecification.createSerializer());
+
+		if (!compatibility.isCompatibleWithReconfiguredSerializer()) {
+			// this test only applies for reconfigured serializers.
+			return;
 		}
+
+		TypeSerializer<ElementT> serializer = compatibility.getReconfiguredSerializer();
+		assertSerializerIsAbleToReadOldData(serializer);
 	}
 
 	@SuppressWarnings("deprecation")
@@ -113,6 +121,28 @@ public abstract class TypeSerializerSnapshotMigrationTestBase<ElementT> extends
 		));
 	}
 
+	@Test
+	public void restoreSerializerFromNewSerializerSnapshotIsAbleToDeserializePreviousData() throws IOException {
+		TypeSerializer<ElementT> newSerializer = testSpecification.createSerializer();
+
+		TypeSerializerSchemaCompatibility<ElementT> compatibility =
+			snapshotUnderTest().resolveSchemaCompatibility(newSerializer);
+
+		final TypeSerializer<ElementT> nextSerializer;
+		if (compatibility.isCompatibleWithReconfiguredSerializer()) {
+			nextSerializer = compatibility.getReconfiguredSerializer();
+		} else if (compatibility.isCompatibleAsIs()) {
+			nextSerializer = newSerializer;
+		} else {
+			// this test does not apply.
+			return;
+		}
+
+		TypeSerializerSnapshot<ElementT> nextSnapshot = nextSerializer.snapshotConfiguration();
+
+		assertSerializerIsAbleToReadOldData(nextSnapshot.restoreSerializer());
+	}
+
 	// --------------------------------------------------------------------------------------------------------------
 	// Test Helpers
 	// --------------------------------------------------------------------------------------------------------------
@@ -161,6 +191,16 @@ public abstract class TypeSerializerSnapshotMigrationTestBase<ElementT> extends
 		return contentsOf(testSpecification.getTestDataLocation());
 	}
 
+	private void assertSerializerIsAbleToReadOldData(TypeSerializer<ElementT> serializer) throws IOException {
+		DataInputView input = dataUnderTest();
+
+		final Matcher<ElementT> matcher = testSpecification.testDataElementMatcher;
+		for (int i = 0; i < testSpecification.testDataCount; i++) {
+			final ElementT result = serializer.deserialize(input);
+			assertThat(result, matcher);
+		}
+	}
+
 	// --------------------------------------------------------------------------------------------------------------
 	// Static Helpers
 	// --------------------------------------------------------------------------------------------------------------
@@ -197,6 +237,7 @@ public abstract class TypeSerializerSnapshotMigrationTestBase<ElementT> extends
 	/**
 	 * Test Specification.
 	 */
+	@SuppressWarnings("WeakerAccess")
 	public static final class TestSpecification<T> {
 		private final Class<? extends TypeSerializer<T>> serializerType;
 		private final Class<? extends TypeSerializerSnapshot<T>> snapshotClass;


[flink] 05/17: [FLINK-11323] Migrate WritableSerializer to use new serialization compatibility abstractions

Posted by tz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tzulitai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 9d8a2e2607f949f755d1fb5934d55c164f536d27
Author: Igal Shilman <ig...@data-artisans.com>
AuthorDate: Sat Feb 9 21:11:06 2019 +0100

    [FLINK-11323] Migrate WritableSerializer to use new serialization compatibility abstractions
---
 .../java/typeutils/runtime/WritableSerializer.java |  46 ++++++++++-
 .../runtime/WritableSerializerMigrationTest.java   |  88 +++++++++++++++++++++
 .../resources/flink-1.6-writeable-serializer-data  | Bin 0 -> 150 bytes
 .../flink-1.6-writeable-serializer-snapshot        | Bin 0 -> 506 bytes
 .../resources/flink-1.7-writeable-serializer-data  | Bin 0 -> 150 bytes
 .../flink-1.7-writeable-serializer-snapshot        | Bin 0 -> 494 bytes
 6 files changed, 132 insertions(+), 2 deletions(-)

diff --git a/flink-connectors/flink-hadoop-compatibility/src/main/java/org/apache/flink/api/java/typeutils/runtime/WritableSerializer.java b/flink-connectors/flink-hadoop-compatibility/src/main/java/org/apache/flink/api/java/typeutils/runtime/WritableSerializer.java
index d6d8e27..4f7b13e 100644
--- a/flink-connectors/flink-hadoop-compatibility/src/main/java/org/apache/flink/api/java/typeutils/runtime/WritableSerializer.java
+++ b/flink-connectors/flink-hadoop-compatibility/src/main/java/org/apache/flink/api/java/typeutils/runtime/WritableSerializer.java
@@ -21,8 +21,11 @@ package org.apache.flink.api.java.typeutils.runtime;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.typeutils.CompatibilityResult;
 import org.apache.flink.api.common.typeutils.GenericTypeSerializerConfigSnapshot;
+import org.apache.flink.api.common.typeutils.GenericTypeSerializerSnapshot;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
+import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
+import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 import org.apache.flink.util.InstantiationUtil;
@@ -157,8 +160,8 @@ public final class WritableSerializer<T extends Writable> extends TypeSerializer
 	// --------------------------------------------------------------------------------------------
 
 	@Override
-	public WritableSerializerConfigSnapshot<T> snapshotConfiguration() {
-		return new WritableSerializerConfigSnapshot<>(typeClass);
+	public TypeSerializerSnapshot<T> snapshotConfiguration() {
+		return new WritableSerializerSnapshot<>(typeClass);
 	}
 
 	@Override
@@ -176,6 +179,7 @@ public final class WritableSerializer<T extends Writable> extends TypeSerializer
 	 * The config snapshot for this serializer.
 	 * @param <T>
 	 */
+	@Deprecated
 	public static final class WritableSerializerConfigSnapshot<T extends Writable>
 			extends GenericTypeSerializerConfigSnapshot<T> {
 
@@ -192,5 +196,43 @@ public final class WritableSerializer<T extends Writable> extends TypeSerializer
 		public int getVersion() {
 			return VERSION;
 		}
+
+		@Override
+		public TypeSerializerSchemaCompatibility<T> resolveSchemaCompatibility(TypeSerializer<T> newSerializer) {
+			return new WritableSerializerSnapshot<>(getTypeClass())
+				.resolveSchemaCompatibility(newSerializer);
+		}
 	}
+
+	/**
+	 * {@link WritableSerializer} snapshot class.
+	 */
+	public static final class WritableSerializerSnapshot<T extends Writable>
+		extends GenericTypeSerializerSnapshot<T, WritableSerializer> {
+
+		@SuppressWarnings("unused")
+		public WritableSerializerSnapshot() {
+		}
+
+		WritableSerializerSnapshot(Class<T> typeClass) {
+			super(typeClass);
+		}
+
+		@Override
+		protected TypeSerializer<T> createSerializer(Class<T> typeClass) {
+			return new WritableSerializer<>(typeClass);
+		}
+
+		@SuppressWarnings("unchecked")
+		@Override
+		protected Class<T> getTypeClass(WritableSerializer serializer) {
+			return serializer.typeClass;
+		}
+
+		@Override
+		protected Class<?> serializerClass() {
+			return WritableSerializer.class;
+		}
+	}
+
 }
diff --git a/flink-connectors/flink-hadoop-compatibility/src/test/java/org/apache/flink/api/java/typeutils/runtime/WritableSerializerMigrationTest.java b/flink-connectors/flink-hadoop-compatibility/src/test/java/org/apache/flink/api/java/typeutils/runtime/WritableSerializerMigrationTest.java
new file mode 100644
index 0000000..8dc7e89
--- /dev/null
+++ b/flink-connectors/flink-hadoop-compatibility/src/test/java/org/apache/flink/api/java/typeutils/runtime/WritableSerializerMigrationTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.typeutils.runtime;
+
+import org.apache.flink.api.common.typeutils.TypeSerializerSnapshotMigrationTestBase;
+import org.apache.flink.api.java.typeutils.runtime.WritableSerializer.WritableSerializerSnapshot;
+import org.apache.flink.api.java.typeutils.runtime.WritableSerializerMigrationTest.WritableName;
+import org.apache.flink.testutils.migration.MigrationVersion;
+
+import org.apache.hadoop.io.Writable;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Collection;
+
+/**
+ * State migration test for {@link RowSerializer}.
+ */
+@RunWith(Parameterized.class)
+public class WritableSerializerMigrationTest extends TypeSerializerSnapshotMigrationTestBase<WritableName> {
+
+	public WritableSerializerMigrationTest(TestSpecification<WritableName> testSpecification) {
+		super(testSpecification);
+	}
+
+	@SuppressWarnings("unchecked")
+	@Parameterized.Parameters(name = "Test Specification = {0}")
+	public static Collection<TestSpecification<?>> testSpecifications() {
+
+		TestSpecifications testSpecifications = new TestSpecifications(MigrationVersion.v1_6, MigrationVersion.v1_7);
+
+		testSpecifications.add(
+			"writeable-serializer",
+			WritableSerializer.class,
+			WritableSerializerSnapshot.class,
+			() -> new WritableSerializer<>(WritableName.class));
+
+		return testSpecifications.get();
+	}
+
+	/**
+	 * A dummy class that is used in this test.
+	 */
+	public static final class WritableName implements Writable {
+
+		public static final long serialVersionUID = 1L;
+
+		private String name;
+
+		public String getName() {
+			return name;
+		}
+
+		public void setName(String name) {
+			this.name = name;
+		}
+
+		@Override
+		public void write(DataOutput out) throws IOException {
+			out.writeUTF(name);
+		}
+
+		@Override
+		public void readFields(DataInput in) throws IOException {
+			name = in.readUTF();
+		}
+	}
+
+}
diff --git a/flink-connectors/flink-hadoop-compatibility/src/test/resources/flink-1.6-writeable-serializer-data b/flink-connectors/flink-hadoop-compatibility/src/test/resources/flink-1.6-writeable-serializer-data
new file mode 100644
index 0000000..c6bd30c
Binary files /dev/null and b/flink-connectors/flink-hadoop-compatibility/src/test/resources/flink-1.6-writeable-serializer-data differ
diff --git a/flink-connectors/flink-hadoop-compatibility/src/test/resources/flink-1.6-writeable-serializer-snapshot b/flink-connectors/flink-hadoop-compatibility/src/test/resources/flink-1.6-writeable-serializer-snapshot
new file mode 100644
index 0000000..268534f
Binary files /dev/null and b/flink-connectors/flink-hadoop-compatibility/src/test/resources/flink-1.6-writeable-serializer-snapshot differ
diff --git a/flink-connectors/flink-hadoop-compatibility/src/test/resources/flink-1.7-writeable-serializer-data b/flink-connectors/flink-hadoop-compatibility/src/test/resources/flink-1.7-writeable-serializer-data
new file mode 100644
index 0000000..e857bed
Binary files /dev/null and b/flink-connectors/flink-hadoop-compatibility/src/test/resources/flink-1.7-writeable-serializer-data differ
diff --git a/flink-connectors/flink-hadoop-compatibility/src/test/resources/flink-1.7-writeable-serializer-snapshot b/flink-connectors/flink-hadoop-compatibility/src/test/resources/flink-1.7-writeable-serializer-snapshot
new file mode 100644
index 0000000..b54029b
Binary files /dev/null and b/flink-connectors/flink-hadoop-compatibility/src/test/resources/flink-1.7-writeable-serializer-snapshot differ


[flink] 03/17: [FLINK-11323] [core] Migrate KryoSerializer to use new serialization compatibility abstractions

Posted by tz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tzulitai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit f436fbfe070dcea31be3560441b6f77c2ce54165
Author: Igal Shilman <ig...@data-artisans.com>
AuthorDate: Tue Jan 15 14:16:19 2019 +0100

    [FLINK-11323] [core] Migrate KryoSerializer to use new serialization compatibility abstractions
    
    This commit adds a new KryoSerializerSnapshot class and uses that
    instead of the now deprecated KryoSerializerConfigSnapshot.
    
    It also adds migration tests for the new KryoSerializerSnapshot,
    extending the TypeSerializerSnapshotMigrationTestBase.
---
 .../typeutils/runtime/kryo/KryoSerializer.java     |  93 ++---
 .../runtime/kryo/KryoSerializerSnapshot.java       | 166 +++++++++
 .../runtime/kryo/KryoSerializerSnapshotData.java   | 377 +++++++++++++++++++++
 .../TypeSerializerSnapshotMigrationTestBase.java   |   6 +
 .../runtime/kryo/KryoPojosForMigrationTests.java   | 154 +++++++++
 .../runtime/kryo/KryoSnapshotMigrationTest.java    | 146 ++++++++
 .../flink-1.6-kryo-type-serializer-custom-data     |   1 +
 .../flink-1.6-kryo-type-serializer-custom-snapshot | Bin 0 -> 4808 bytes
 .../resources/flink-1.6-kryo-type-serializer-data  |   1 +
 ...link-1.6-kryo-type-serializer-empty-config-data | Bin 0 -> 180 bytes
 ...-1.6-kryo-type-serializer-empty-config-snapshot | Bin 0 -> 2759 bytes
 .../flink-1.6-kryo-type-serializer-snapshot        | Bin 0 -> 3854 bytes
 .../flink-1.7-kryo-type-serializer-custom-data     |   1 +
 .../flink-1.7-kryo-type-serializer-custom-snapshot | Bin 0 -> 4796 bytes
 .../resources/flink-1.7-kryo-type-serializer-data  |   1 +
 ...link-1.7-kryo-type-serializer-empty-config-data | Bin 0 -> 892 bytes
 ...-1.7-kryo-type-serializer-empty-config-snapshot | Bin 0 -> 2747 bytes
 .../flink-1.7-kryo-type-serializer-snapshot        | Bin 0 -> 3842 bytes
 18 files changed, 906 insertions(+), 40 deletions(-)

diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java
index 98b07e1..f1d3022 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java
@@ -20,9 +20,10 @@ package org.apache.flink.api.java.typeutils.runtime.kryo;
 
 import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.typeutils.CompatibilityResult;
+import org.apache.flink.api.common.ExecutionConfig.SerializableSerializer;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
+import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
+import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
 import org.apache.flink.api.java.typeutils.AvroUtils;
 import org.apache.flink.api.java.typeutils.runtime.DataInputViewStream;
 import org.apache.flink.api.java.typeutils.runtime.DataOutputViewStream;
@@ -174,6 +175,36 @@ public class KryoSerializer<T> extends TypeSerializer<T> {
 		}
 	}
 
+	// for KryoSerializerSnapshot
+	// ------------------------------------------------------------------------
+
+	KryoSerializer(Class<T> type,
+				   LinkedHashMap<Class<?>, SerializableSerializer<?>> defaultSerializers,
+				   LinkedHashMap<Class<?>, Class<? extends Serializer<?>>> defaultSerializerClasses,
+				   LinkedHashMap<String, KryoRegistration> kryoRegistrations) {
+
+		this.type = checkNotNull(type, "Type class cannot be null.");
+		this.defaultSerializerClasses = checkNotNull(defaultSerializerClasses, "Default serializer classes cannot be null.");
+		this.defaultSerializers = checkNotNull(defaultSerializers, "Default serializers cannot be null.");
+		this.kryoRegistrations = checkNotNull(kryoRegistrations, "Kryo registrations cannot be null.");
+	}
+
+	Class<T> getType() {
+		return type;
+	}
+
+	LinkedHashMap<Class<?>, SerializableSerializer<?>> getDefaultKryoSerializers() {
+		return defaultSerializers;
+	}
+
+	LinkedHashMap<Class<?>, Class<? extends Serializer<?>>> getDefaultKryoSerializerClasses() {
+		return defaultSerializerClasses;
+	}
+
+	LinkedHashMap<String, KryoRegistration> getKryoRegistrations() {
+		return kryoRegistrations;
+	}
+
 	// ------------------------------------------------------------------------
 
 	@Override
@@ -452,46 +483,15 @@ public class KryoSerializer<T> extends TypeSerializer<T> {
 	// --------------------------------------------------------------------------------------------
 
 	@Override
-	public KryoSerializerConfigSnapshot<T> snapshotConfiguration() {
-		return new KryoSerializerConfigSnapshot<>(type, kryoRegistrations);
-	}
-
-	@SuppressWarnings("unchecked")
-	@Override
-	public CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot<?> configSnapshot) {
-		if (configSnapshot instanceof KryoSerializerConfigSnapshot) {
-			final KryoSerializerConfigSnapshot<T> config = (KryoSerializerConfigSnapshot<T>) configSnapshot;
-
-			if (type.equals(config.getTypeClass())) {
-				LinkedHashMap<String, KryoRegistration> reconfiguredRegistrations = config.getKryoRegistrations();
-
-				// reconfigure by assuring that classes which were previously registered are registered
-				// again in the exact same order; new class registrations will be appended.
-				// this also overwrites any dummy placeholders that the restored old configuration has
-				reconfiguredRegistrations.putAll(kryoRegistrations);
-
-				// check if there is still any dummy placeholders even after reconfiguration;
-				// if so, then this new Kryo serializer cannot read old data and is therefore incompatible
-				for (Map.Entry<String, KryoRegistration> reconfiguredRegistrationEntry : reconfiguredRegistrations.entrySet()) {
-					if (reconfiguredRegistrationEntry.getValue().isDummy()) {
-						LOG.warn("The Kryo registration for a previously registered class {} does not have a " +
-							"proper serializer, because its previous serializer cannot be loaded or is no " +
-							"longer valid but a new serializer is not available", reconfiguredRegistrationEntry.getKey());
-
-						return CompatibilityResult.requiresMigration();
-					}
-				}
-
-				// there's actually no way to tell if new Kryo serializers are compatible with
-				// the previous ones they overwrite; we can only signal compatibility and hope for the best
-				this.kryoRegistrations = reconfiguredRegistrations;
-				return CompatibilityResult.compatible();
-			}
-		}
-
-		return CompatibilityResult.requiresMigration();
+	public TypeSerializerSnapshot<T> snapshotConfiguration() {
+		return new KryoSerializerSnapshot<>(
+			type,
+			defaultSerializers,
+			defaultSerializerClasses,
+			kryoRegistrations);
 	}
 
+	@Deprecated
 	public static final class KryoSerializerConfigSnapshot<T> extends KryoRegistrationSerializerConfigSnapshot<T> {
 
 		private static final int VERSION = 1;
@@ -510,6 +510,19 @@ public class KryoSerializer<T> extends TypeSerializer<T> {
 		public int getVersion() {
 			return VERSION;
 		}
+
+		@Override
+		public TypeSerializerSchemaCompatibility<T> resolveSchemaCompatibility(TypeSerializer<T> newSerializer) {
+			KryoSerializer<T> javaSerializedKryoSerializer = (KryoSerializer<T>) super.restoreSerializer();
+
+			KryoSerializerSnapshot<T> snapshot = new KryoSerializerSnapshot<>(
+				javaSerializedKryoSerializer.getType(),
+				javaSerializedKryoSerializer.getDefaultKryoSerializers(),
+				javaSerializedKryoSerializer.getDefaultKryoSerializerClasses(),
+				javaSerializedKryoSerializer.getKryoRegistrations());
+
+			return snapshot.resolveSchemaCompatibility(newSerializer);
+		}
 	}
 
 	// --------------------------------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshot.java
new file mode 100644
index 0000000..37a0104
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshot.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.typeutils.runtime.kryo;
+
+import org.apache.flink.api.common.ExecutionConfig.SerializableSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
+import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
+import org.apache.flink.api.java.typeutils.runtime.KryoRegistration;
+import org.apache.flink.api.java.typeutils.runtime.kryo.OptionalMap.MergeResult;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+
+import com.esotericsoftware.kryo.Serializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.LinkedHashMap;
+import java.util.function.Function;
+
+import static org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializerSnapshotData.createFrom;
+import static org.apache.flink.api.java.typeutils.runtime.kryo.OptionalMap.mergeRightIntoLeft;
+import static org.apache.flink.api.java.typeutils.runtime.kryo.OptionalMap.optionalMapOf;
+
+/**
+ * {@link TypeSerializerSnapshot} for {@link KryoSerializer}.
+ */
+public class KryoSerializerSnapshot<T> implements TypeSerializerSnapshot<T> {
+
+	private static final Logger LOG = LoggerFactory.getLogger(KryoSerializerSnapshot.class);
+
+	private static final int VERSION = 2;
+
+	private KryoSerializerSnapshotData<T> snapshotData;
+
+	@SuppressWarnings("unused")
+	public KryoSerializerSnapshot() {
+	}
+
+	KryoSerializerSnapshot(Class<T> typeClass,
+			LinkedHashMap<Class<?>, SerializableSerializer<?>> defaultKryoSerializers,
+			LinkedHashMap<Class<?>, Class<? extends Serializer<?>>> defaultKryoSerializerClasses,
+			LinkedHashMap<String, KryoRegistration> kryoRegistrations) {
+
+		this.snapshotData = createFrom(typeClass, defaultKryoSerializers, defaultKryoSerializerClasses, kryoRegistrations);
+	}
+
+	@Override
+	public int getCurrentVersion() {
+		return VERSION;
+	}
+
+	@Override
+	public void writeSnapshot(DataOutputView out) throws IOException {
+		snapshotData.writeSnapshotData(out);
+	}
+
+	@Override
+	public void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException {
+		this.snapshotData = createFrom(in, userCodeClassLoader);
+	}
+
+	@Override
+	public TypeSerializer<T> restoreSerializer() {
+		return new KryoSerializer<>(
+			snapshotData.getTypeClass(),
+			snapshotData.getDefaultKryoSerializers().unwrapOptionals(),
+			snapshotData.getDefaultKryoSerializerClasses().unwrapOptionals(),
+			snapshotData.getKryoRegistrations().unwrapOptionals());
+	}
+
+	@Override
+	public TypeSerializerSchemaCompatibility<T> resolveSchemaCompatibility(TypeSerializer<T> newSerializer) {
+		if (!(newSerializer instanceof KryoSerializer)) {
+			return TypeSerializerSchemaCompatibility.incompatible();
+		}
+		KryoSerializer<T> kryoSerializer = (KryoSerializer<T>) newSerializer;
+		if (kryoSerializer.getType() != snapshotData.getTypeClass()) {
+			return TypeSerializerSchemaCompatibility.incompatible();
+		}
+		return resolveSchemaCompatibility(kryoSerializer);
+	}
+
+	private TypeSerializerSchemaCompatibility<T> resolveSchemaCompatibility(KryoSerializer<T> newSerializer) {
+		// merge the default serializers
+		final MergeResult<Class<?>, SerializableSerializer<?>> reconfiguredDefaultKryoSerializers = mergeRightIntoLeft(
+			snapshotData.getDefaultKryoSerializers(),
+			optionalMapOf(newSerializer.getDefaultKryoSerializers(), Class::getName));
+
+		if (reconfiguredDefaultKryoSerializers.hasMissingKeys()) {
+			logMissingKeys(reconfiguredDefaultKryoSerializers);
+			return TypeSerializerSchemaCompatibility.incompatible();
+		}
+
+		// merge default serializer classes
+		final MergeResult<Class<?>, Class<? extends Serializer<?>>> reconfiguredDefaultKryoSerializerClasses = mergeRightIntoLeft(
+			snapshotData.getDefaultKryoSerializerClasses(),
+			optionalMapOf(newSerializer.getDefaultKryoSerializerClasses(), Class::getName));
+
+		if (reconfiguredDefaultKryoSerializerClasses.hasMissingKeys()) {
+			logMissingKeys(reconfiguredDefaultKryoSerializerClasses);
+			return TypeSerializerSchemaCompatibility.incompatible();
+		}
+
+		// merge registration
+		final MergeResult<String, KryoRegistration> reconfiguredRegistrations = mergeRightIntoLeft(
+			snapshotData.getKryoRegistrations(),
+			optionalMapOf(newSerializer.getKryoRegistrations(), Function.identity()));
+
+		if (reconfiguredRegistrations.hasMissingKeys()) {
+			logMissingKeys(reconfiguredRegistrations);
+			return TypeSerializerSchemaCompatibility.incompatible();
+		}
+
+		// there are no missing keys, now we have to decide whether we are compatible as-is or we require reconfiguration.
+		return resolveSchemaCompatibility(
+			reconfiguredDefaultKryoSerializers,
+			reconfiguredDefaultKryoSerializerClasses,
+			reconfiguredRegistrations);
+	}
+
+	private TypeSerializerSchemaCompatibility<T> resolveSchemaCompatibility(
+		MergeResult<Class<?>, SerializableSerializer<?>> reconfiguredDefaultKryoSerializers,
+		MergeResult<Class<?>, Class<? extends Serializer<?>>> reconfiguredDefaultKryoSerializerClasses,
+		MergeResult<String, KryoRegistration> reconfiguredRegistrations) {
+
+		if (reconfiguredDefaultKryoSerializers.isOrderedSubset() &&
+			reconfiguredDefaultKryoSerializerClasses.isOrderedSubset() &&
+			reconfiguredRegistrations.isOrderedSubset()) {
+
+			return TypeSerializerSchemaCompatibility.compatibleAsIs();
+		}
+
+		// reconfigure a new KryoSerializer
+		KryoSerializer<T> reconfiguredSerializer = new KryoSerializer<>(
+			snapshotData.getTypeClass(),
+			reconfiguredDefaultKryoSerializers.getMerged(),
+			reconfiguredDefaultKryoSerializerClasses.getMerged(),
+			reconfiguredRegistrations.getMerged());
+
+		return TypeSerializerSchemaCompatibility.compatibleWithReconfiguredSerializer(reconfiguredSerializer);
+	}
+
+	private void logMissingKeys(MergeResult<?, ?> mergeResult) {
+		mergeResult.missingKeys().forEach(key -> LOG.warn("The Kryo registration for a previously registered class {} does not have a " +
+			"proper serializer, because its previous serializer cannot be loaded or is no " +
+			"longer valid but a new serializer is not available", key));
+	}
+}
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshotData.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshotData.java
new file mode 100644
index 0000000..9587101
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshotData.java
@@ -0,0 +1,377 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.typeutils.runtime.kryo;
+
+import org.apache.flink.api.common.ExecutionConfig.SerializableSerializer;
+import org.apache.flink.api.java.typeutils.runtime.DataInputViewStream;
+import org.apache.flink.api.java.typeutils.runtime.DataOutputViewStream;
+import org.apache.flink.api.java.typeutils.runtime.KryoRegistration;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.util.InstantiationUtil;
+
+import com.esotericsoftware.kryo.Serializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InvalidClassException;
+import java.util.LinkedHashMap;
+import java.util.Map.Entry;
+import java.util.function.Function;
+
+import static org.apache.flink.api.java.typeutils.runtime.kryo.OptionalMap.optionalMapOf;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+final class KryoSerializerSnapshotData<T> {
+
+	private static final Logger LOG = LoggerFactory.getLogger(KryoSerializerSnapshotData.class);
+
+	// --------------------------------------------------------------------------------------------
+	// Factories
+	// --------------------------------------------------------------------------------------------
+
+	static <T> KryoSerializerSnapshotData<T> createFrom(
+		Class<T> typeClass,
+		LinkedHashMap<Class<?>, SerializableSerializer<?>> defaultKryoSerializers,
+		LinkedHashMap<Class<?>, Class<? extends Serializer<?>>> defaultKryoSerializerClasses,
+		LinkedHashMap<String, KryoRegistration> kryoRegistrations) {
+
+		return new KryoSerializerSnapshotData<>(
+			typeClass,
+			optionalMapOf(defaultKryoSerializers, Class::getName),
+			optionalMapOf(defaultKryoSerializerClasses, Class::getName),
+			optionalMapOf(kryoRegistrations, Function.identity()));
+	}
+
+	static <T> KryoSerializerSnapshotData<T> createFrom(DataInputView in, ClassLoader cl) throws IOException {
+		Class<T> typeClass = readTypeClass(in, cl);
+		OptionalMap<String, KryoRegistration> kryoRegistrations = readKryoRegistrations(in, cl);
+		OptionalMap<Class<?>, SerializableSerializer<?>> defaultSerializer = readDefaultKryoSerializers(in, cl);
+		OptionalMap<Class<?>, Class<? extends Serializer<?>>> defaultSerializerClasses =
+			readDefaultKryoSerializerClasses(in, cl);
+
+		return new KryoSerializerSnapshotData<>(
+			typeClass,
+			defaultSerializer,
+			defaultSerializerClasses,
+			kryoRegistrations);
+	}
+
+	// --------------------------------------------------------------------------------------------
+	// Fields
+	// --------------------------------------------------------------------------------------------
+
+	private final Class<T> typeClass;
+	private final OptionalMap<Class<?>, SerializableSerializer<?>> defaultKryoSerializers;
+	private final OptionalMap<Class<?>, Class<? extends Serializer<?>>> defaultKryoSerializerClasses;
+	private final OptionalMap<String, KryoRegistration> kryoRegistrations;
+
+	private KryoSerializerSnapshotData(Class<T> typeClass,
+			OptionalMap<Class<?>, SerializableSerializer<?>> defaultKryoSerializers,
+			OptionalMap<Class<?>, Class<? extends Serializer<?>>> defaultKryoSerializerClasses,
+			OptionalMap<String, KryoRegistration> kryoRegistrations) {
+
+		this.typeClass = typeClass;
+		this.defaultKryoSerializers = defaultKryoSerializers;
+		this.defaultKryoSerializerClasses = defaultKryoSerializerClasses;
+		this.kryoRegistrations = kryoRegistrations;
+	}
+
+	// --------------------------------------------------------------------------------------------
+	// Getters
+	// --------------------------------------------------------------------------------------------
+
+	Class<T> getTypeClass() {
+		return typeClass;
+	}
+
+	OptionalMap<Class<?>, SerializableSerializer<?>> getDefaultKryoSerializers() {
+		return defaultKryoSerializers;
+	}
+
+	OptionalMap<Class<?>, Class<? extends Serializer<?>>> getDefaultKryoSerializerClasses() {
+		return defaultKryoSerializerClasses;
+	}
+
+	OptionalMap<String, KryoRegistration> getKryoRegistrations() {
+		return kryoRegistrations;
+	}
+
+	// --------------------------------------------------------------------------------------------
+	// Write
+	// --------------------------------------------------------------------------------------------
+
+	void writeSnapshotData(DataOutputView out) throws IOException {
+		writeTypeClass(out);
+		writeKryoRegistrations(out, kryoRegistrations);
+		writeDefaultKryoSerializers(out, defaultKryoSerializers);
+		writeDefaultKryoSerializerClasses(out, defaultKryoSerializerClasses);
+	}
+
+	private void writeTypeClass(DataOutputView out) throws IOException {
+		out.writeUTF(typeClass.getName());
+	}
+
+	private static void writeKryoRegistrations(
+		DataOutputView out,
+		OptionalMap<String, KryoRegistration> kryoRegistrations) throws IOException {
+
+		out.writeInt(kryoRegistrations.size());
+		for (Entry<String, KryoRegistration> entry : kryoRegistrations.unwrapOptionals().entrySet()) {
+			out.writeUTF(entry.getKey());
+			KryoRegistrationUtil.writeKryoRegistration(entry.getValue(), out);
+		}
+	}
+
+	private void writeDefaultKryoSerializers(
+		DataOutputView out,
+		OptionalMap<Class<?>,
+			SerializableSerializer<?>> defaultKryoSerializers) throws IOException {
+
+		out.writeInt(defaultKryoSerializers.size());
+		for (Entry<Class<?>, SerializableSerializer<?>> entry : defaultKryoSerializers.unwrapOptionals().entrySet()) {
+			Class<?> javaClass = entry.getKey();
+			SerializableSerializer<?> serializerInstance = entry.getValue();
+
+			out.writeUTF(javaClass.getName());
+			try (final DataOutputViewStream outViewWrapper = new DataOutputViewStream(out)) {
+				InstantiationUtil.serializeObject(outViewWrapper, serializerInstance);
+			}
+		}
+	}
+
+	private static void writeDefaultKryoSerializerClasses(
+		DataOutputView out,
+		OptionalMap<Class<?>, Class<? extends Serializer<?>>> defaultKryoSerializerClasses)
+		throws IOException {
+
+		out.writeInt(defaultKryoSerializerClasses.size());
+
+		for (Entry<Class<?>, Class<? extends Serializer<?>>> entry : defaultKryoSerializerClasses.unwrapOptionals().entrySet()) {
+			Class<?> javaClass = entry.getKey();
+			Class<? extends Serializer<?>> serializerClass = entry.getValue();
+			out.writeUTF(javaClass.getName());
+			out.writeUTF(serializerClass.getName());
+		}
+	}
+
+	// --------------------------------------------------------------------------------------------
+	// Read
+	// --------------------------------------------------------------------------------------------
+
+	private static <T> Class<T> readTypeClass(DataInputView in, ClassLoader userCodeClassLoader) throws IOException {
+		return InstantiationUtil.resolveClassByName(in, userCodeClassLoader);
+	}
+
+	private static OptionalMap<String, KryoRegistration> readKryoRegistrations(
+		DataInputView in,
+		ClassLoader userCodeClassLoader) throws IOException {
+
+		OptionalMap<String, KryoRegistration> registrations = new OptionalMap<>();
+		final int size = in.readInt();
+		for (int i = 0; i < size; i++) {
+			final String name = in.readUTF();
+			KryoRegistration kryoRegistration = KryoRegistrationUtil.tryReadKryoRegistration(
+					in,
+					userCodeClassLoader);
+			registrations.put(name, name, kryoRegistration);
+		}
+
+		return registrations;
+	}
+
+	private static OptionalMap<Class<?>, SerializableSerializer<?>> readDefaultKryoSerializers(DataInputView in, ClassLoader cl) throws IOException {
+		OptionalMap<Class<?>, SerializableSerializer<?>> kryoSerializers = new OptionalMap<>();
+		final int size = in.readInt();
+		for (int i = 0; i < size; i++) {
+			final String className = in.readUTF();
+			Class<?> javaClass = null;
+			try {
+				javaClass = Class.forName(className, false, cl);
+			}
+			catch (ClassNotFoundException e) {
+				LOG.warn("Cannot find registered class " + className + " for Kryo serialization in classpath.", e);
+			}
+			SerializableSerializer<?> kryoSerializer = null;
+			try {
+				try (final DataInputViewStream inViewWrapper = new DataInputViewStream(in)) {
+					kryoSerializer = InstantiationUtil.deserializeObject(inViewWrapper, cl);
+				}
+			}
+			catch (Throwable e) {
+				LOG.warn("Cannot deserialize a previously serialized kryo serializer for the type " + className, e);
+			}
+			kryoSerializers.put(className, javaClass, kryoSerializer);
+		}
+		return kryoSerializers;
+	}
+
+	@SuppressWarnings("unchecked")
+	private static OptionalMap<Class<?>, Class<? extends Serializer<?>>> readDefaultKryoSerializerClasses(
+		DataInputView in,
+		ClassLoader cl) throws IOException {
+
+		OptionalMap<Class<?>, Class<? extends Serializer<?>>> kryoSerializerClasses = new OptionalMap<>();
+		final int size = in.readInt();
+		for (int i = 0; i < size; i++) {
+			final String className = in.readUTF();
+			Class<?> typeClass = null;
+			try {
+				typeClass = Class.forName(className, false, cl);
+			}
+			catch (ClassNotFoundException e) {
+				LOG.warn("Cannot find registered class " + className + " for Kryo serialization in classpath.", e);
+			}
+			final String kryoSerializerClassName = in.readUTF();
+			Class<? extends Serializer<?>> kryoSerializerClass = null;
+			try {
+				kryoSerializerClass = (Class<? extends Serializer<?>>) Class.forName(kryoSerializerClassName, false, cl);
+			}
+			catch (Throwable e) {
+				LOG.warn("Cannot find registered class " + className + " for Kryo serialization in classpath.", e);
+			}
+			kryoSerializerClasses.put(className, typeClass, kryoSerializerClass);
+		}
+		return kryoSerializerClasses;
+	}
+
+	// --------------------------------------------------------------------------------------------
+	// Helpers
+	// --------------------------------------------------------------------------------------------
+
+	private static final class KryoRegistrationUtil {
+
+		static void writeKryoRegistration(
+				KryoRegistration kryoRegistration,
+				DataOutputView out) throws IOException {
+
+			checkNotNull(kryoRegistration);
+			out.writeUTF(kryoRegistration.getRegisteredClass().getName());
+
+			final KryoRegistration.SerializerDefinitionType serializerDefinitionType =
+					kryoRegistration.getSerializerDefinitionType();
+
+			out.writeInt(serializerDefinitionType.ordinal());
+			switch (serializerDefinitionType) {
+				case UNSPECIFIED: {
+					// nothing else to write
+					break;
+				}
+				case CLASS: {
+					Class<? extends Serializer<?>> serializerClass = kryoRegistration.getSerializerClass();
+					assert serializerClass != null;
+					out.writeUTF(serializerClass.getName());
+					break;
+				}
+				case INSTANCE: {
+					try (final DataOutputViewStream outViewWrapper = new DataOutputViewStream(out)) {
+						InstantiationUtil.serializeObject(outViewWrapper, kryoRegistration.getSerializableSerializerInstance());
+					}
+					break;
+				}
+				default: {
+					throw new IllegalStateException(
+							"Unrecognized Kryo registration serializer definition type: " + serializerDefinitionType);
+				}
+			}
+		}
+
+		static KryoRegistration tryReadKryoRegistration(
+				DataInputView in,
+				ClassLoader userCodeClassLoader) throws IOException {
+
+			String registeredClassname = in.readUTF();
+			Class<?> registeredClass;
+			try {
+				registeredClass = Class.forName(registeredClassname, true, userCodeClassLoader);
+			}
+			catch (ClassNotFoundException e) {
+				LOG.warn("Cannot find registered class " + registeredClassname + " for Kryo serialization in classpath;" +
+						" using a dummy class as a placeholder.", e);
+				return null;
+			}
+
+			final KryoRegistration.SerializerDefinitionType serializerDefinitionType =
+					KryoRegistration.SerializerDefinitionType.values()[in.readInt()];
+
+			switch (serializerDefinitionType) {
+				case UNSPECIFIED: {
+					return new KryoRegistration(registeredClass);
+				}
+				case CLASS: {
+					return tryReadWithSerializerClass(in, userCodeClassLoader, registeredClassname, registeredClass);
+				}
+				case INSTANCE: {
+					return tryReadWithSerializerInstance(in, userCodeClassLoader, registeredClassname, registeredClass);
+				}
+				default: {
+					throw new IllegalStateException(
+							"Unrecognized Kryo registration serializer definition type: " + serializerDefinitionType);
+				}
+			}
+		}
+
+		@SuppressWarnings("unchecked")
+		private static KryoRegistration tryReadWithSerializerClass(
+				DataInputView in,
+				ClassLoader userCodeClassLoader,
+				String registeredClassname,
+				Class<?> registeredClass) throws IOException {
+			String serializerClassname = in.readUTF();
+			Class serializerClass;
+			try {
+				serializerClass = Class.forName(serializerClassname, true, userCodeClassLoader);
+				return new KryoRegistration(registeredClass, serializerClass);
+			}
+			catch (ClassNotFoundException e) {
+				LOG.warn("Cannot find registered Kryo serializer class for class " + registeredClassname +
+						" in classpath; using a dummy Kryo serializer that should be replaced as soon as" +
+						" a new Kryo serializer for the class is present", e);
+			}
+			return null;
+		}
+
+		private static KryoRegistration tryReadWithSerializerInstance(
+				DataInputView in,
+				ClassLoader userCodeClassLoader,
+				String registeredClassname,
+				Class<?> registeredClass) throws IOException {
+			SerializableSerializer<? extends Serializer<?>> serializerInstance;
+
+			try (final DataInputViewStream inViewWrapper = new DataInputViewStream(in)) {
+				serializerInstance = InstantiationUtil.deserializeObject(inViewWrapper, userCodeClassLoader);
+				return new KryoRegistration(registeredClass, serializerInstance);
+			}
+			catch (ClassNotFoundException e) {
+				LOG.warn("Cannot find registered Kryo serializer class for class " + registeredClassname +
+						" in classpath; using a dummy Kryo serializer that should be replaced as soon as" +
+						" a new Kryo serializer for the class is present", e);
+			}
+			catch (InvalidClassException e) {
+				LOG.warn("The registered Kryo serializer class for class " + registeredClassname +
+						" has changed and is no longer valid; using a dummy Kryo serializer that should be replaced" +
+						" as soon as a new Kryo serializer for the class is present.", e);
+
+			}
+			return null;
+		}
+
+	}
+}
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerSnapshotMigrationTestBase.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerSnapshotMigrationTestBase.java
index 31fc5ea..f52e411 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerSnapshotMigrationTestBase.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerSnapshotMigrationTestBase.java
@@ -271,6 +271,11 @@ public abstract class TypeSerializerSnapshotMigrationTestBase<ElementT> extends
 			return this;
 		}
 
+		public TestSpecification<T> withTestDataCount(int expectedDataItmes) {
+			this.testDataCount = expectedDataItmes;
+			return this;
+		}
+
 		private TypeSerializer<T> createSerializer() {
 			try {
 				return (serializerProvider == null) ? serializerType.newInstance() : serializerProvider.get();
@@ -296,6 +301,7 @@ public abstract class TypeSerializerSnapshotMigrationTestBase<ElementT> extends
 		public String toString() {
 			return String.format("%s , %s, %s", name, serializerType.getSimpleName(), snapshotClass.getSimpleName());
 		}
+
 	}
 
 	/**
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoPojosForMigrationTests.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoPojosForMigrationTests.java
new file mode 100644
index 0000000..c7b6cb0
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoPojosForMigrationTests.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.typeutils.runtime.kryo;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.Serializer;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+
+import java.io.Serializable;
+
+/**
+ * POJOS needed for {@link KryoPojosForMigrationTests}.
+ */
+@SuppressWarnings("WeakerAccess")
+public class KryoPojosForMigrationTests {
+
+	public static abstract class Animal {
+	}
+
+	public static class Dog extends Animal {
+		private final String name;
+
+		public Dog(String name) {
+			this.name = name;
+		}
+
+		public String getName() {
+			return name;
+		}
+	}
+
+	public static class Cat extends Animal {
+		private final String name;
+		private final int age;
+
+		public Cat(String name, int age) {
+			this.name = name;
+			this.age = age;
+		}
+
+		public String getName() {
+			return name;
+		}
+
+		public int getAge() {
+			return age;
+		}
+	}
+
+	public static class Parrot extends Animal {
+		private final String accent;
+
+		public Parrot(String accent) {
+			this.accent = accent;
+		}
+
+		public String getAccent() {
+			return accent;
+		}
+	}
+
+	// HousePets is registered explicitly in flink-1.6-kryo-type-serializer-*-* test resources.
+	@SuppressWarnings("unused")
+	public static class HousePets {
+		private final Dog dog;
+		private final Cat cat;
+
+		public HousePets(Dog dog, Cat cat) {
+			this.dog = dog;
+			this.cat = cat;
+		}
+
+		public Dog getDog() {
+			return dog;
+		}
+
+		public Cat getCat() {
+			return cat;
+		}
+	}
+
+	/**
+	 * A Serializer that was registered during test data generation.
+	 */
+	public static class ParrotKryoSerializer extends Serializer<Parrot> implements Serializable {
+
+		private static final long serialVersionUID = 1L;
+
+		@Override
+		public void write(Kryo kryo, Output output, Parrot object) {
+			output.writeString(object.getAccent());
+		}
+
+		@Override
+		public Parrot read(Kryo kryo, Input input, Class<Parrot> type) {
+			return new Parrot(input.readString());
+		}
+	}
+
+	/**
+	 * A Serializer that was registered during test data generation.
+	 */
+	public static class DogKryoSerializer extends Serializer<Dog> implements Serializable {
+
+		private static final long serialVersionUID = 1L;
+
+		@Override
+		public void write(Kryo kryo, Output output, Dog object) {
+			output.writeString(object.getName());
+		}
+
+		@Override
+		public Dog read(Kryo kryo, Input input, Class<Dog> type) {
+			return new Dog(input.readString());
+		}
+	}
+
+	/**
+	 * A Serializer that is registered in migration tests.
+	 */
+	public static class DogV2KryoSerializer extends Serializer<Dog> implements Serializable {
+
+		private static final long serialVersionUID = 1L;
+
+		@Override
+		public void write(Kryo kryo, Output output, Dog object) {
+			output.writeString(object.getName());
+		}
+
+		@Override
+		public Dog read(Kryo kryo, Input input, Class<Dog> type) {
+			return new Dog(input.readString());
+		}
+	}
+
+
+}
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSnapshotMigrationTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSnapshotMigrationTest.java
new file mode 100644
index 0000000..a982eca
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSnapshotMigrationTest.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.typeutils.runtime.kryo;
+
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
+import org.apache.flink.api.common.typeutils.TypeSerializerSnapshotMigrationTestBase;
+import org.apache.flink.api.java.typeutils.runtime.kryo.KryoPojosForMigrationTests.Animal;
+import org.apache.flink.api.java.typeutils.runtime.kryo.KryoPojosForMigrationTests.Cat;
+import org.apache.flink.api.java.typeutils.runtime.kryo.KryoPojosForMigrationTests.Dog;
+import org.apache.flink.api.java.typeutils.runtime.kryo.KryoPojosForMigrationTests.DogV2KryoSerializer;
+import org.apache.flink.api.java.typeutils.runtime.kryo.KryoPojosForMigrationTests.Parrot;
+import org.apache.flink.api.java.typeutils.runtime.kryo.KryoPojosForMigrationTests.ParrotKryoSerializer;
+import org.apache.flink.testutils.migration.MigrationVersion;
+
+import com.esotericsoftware.kryo.serializers.DefaultSerializers.StringSerializer;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.function.Supplier;
+
+import static org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility.compatibleAsIs;
+import static org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility.compatibleWithReconfiguredSerializer;
+
+/**
+ * Tests migrations for {@link KryoSerializerSnapshot}.
+ */
+@SuppressWarnings("WeakerAccess")
+@RunWith(Parameterized.class)
+public class KryoSnapshotMigrationTest extends TypeSerializerSnapshotMigrationTestBase<KryoPojosForMigrationTests> {
+
+	public KryoSnapshotMigrationTest(TestSpecification<KryoPojosForMigrationTests> testSpecification) {
+		super(testSpecification);
+	}
+
+	@SuppressWarnings("unchecked")
+	@Parameterized.Parameters(name = "Test Specification = {0}")
+	public static Collection<Object[]> testSpecifications() {
+
+		List<Object[]> specs = new ArrayList<>();
+
+		add(specs, "kryo-type-serializer-empty-config",
+			() -> new KryoSerializer<>(Animal.class, new ExecutionConfig()));
+
+		add(specs, "kryo-type-serializer-empty-config", () -> {
+
+			ExecutionConfig executionConfig = new ExecutionConfig();
+			executionConfig.registerKryoType(DummyClassOne.class);
+			executionConfig.registerTypeWithKryoSerializer(DummyClassTwo.class, StringSerializer.class);
+
+			return new KryoSerializer<>(Animal.class, executionConfig);
+		}, COMPATIBLE_WITH_RECONFIGURED);
+
+		add(specs, "kryo-type-serializer", () -> {
+
+			ExecutionConfig executionConfig = new ExecutionConfig();
+			executionConfig.registerKryoType(DummyClassOne.class);
+			executionConfig.registerKryoType(Dog.class);
+			executionConfig.registerKryoType(DummyClassTwo.class);
+			executionConfig.registerKryoType(Cat.class);
+			executionConfig.registerKryoType(Parrot.class);
+
+			return new KryoSerializer<>(Animal.class, executionConfig);
+		}, COMPATIBLE_WITH_RECONFIGURED);
+
+		add(specs, "kryo-type-serializer-custom", () -> {
+			ExecutionConfig executionConfig = new ExecutionConfig();
+			executionConfig.registerKryoType(DummyClassOne.class);
+			executionConfig.registerTypeWithKryoSerializer(Dog.class, DogV2KryoSerializer.class);
+			executionConfig.registerKryoType(DummyClassTwo.class);
+			executionConfig.registerKryoType(Cat.class);
+			executionConfig.registerTypeWithKryoSerializer(Parrot.class, ParrotKryoSerializer.class);
+
+			return new KryoSerializer<>(Animal.class, executionConfig);
+		}, COMPATIBLE_WITH_RECONFIGURED);
+
+		return specs;
+	}
+
+	private static void add(List<Object[]> all, String name, Supplier<TypeSerializer<Animal>> supplier) {
+		add(all, name, supplier, compatibleAsIs());
+	}
+
+	private static void add(List<Object[]> all,
+							String name, Supplier<TypeSerializer<Animal>> supplier,
+							TypeSerializerSchemaCompatibility<Animal> expected) {
+
+		TestSpecification<Animal> flink16 = TestSpecification.<Animal>builder(
+			MigrationVersion.v1_6 + " " + name,
+			KryoSerializer.class,
+			KryoSerializerSnapshot.class,
+			MigrationVersion.v1_6)
+			.withNewSerializerProvider(supplier, expected)
+			.withSnapshotDataLocation("flink-1.6-" + name + "-snapshot")
+			.withTestData("flink-1.6-" + name + "-data", 2);
+
+		TestSpecification<Animal> flink17 = TestSpecification.<Animal>builder(
+			MigrationVersion.v1_7 + " " + name,
+			KryoSerializer.class,
+			KryoSerializerSnapshot.class,
+			MigrationVersion.v1_7)
+			.withNewSerializerProvider(supplier, expected)
+			.withSnapshotDataLocation("flink-1.7-" + name + "-snapshot")
+			.withTestData("flink-1.7-" + name + "-data", 2);
+
+		all.add(new Object[]{flink16});
+		all.add(new Object[]{flink17});
+	}
+
+	/**
+	 * Dummy class to be registered in the tests.
+	 */
+	public static final class DummyClassOne {
+
+	}
+
+	/**
+	 * Dummy class to be registered in the tests.
+	 */
+	public static final class DummyClassTwo {
+
+	}
+
+	private static final TypeSerializerSchemaCompatibility<Animal> COMPATIBLE_WITH_RECONFIGURED =
+		compatibleWithReconfiguredSerializer(new KryoSerializer<>(Animal.class, new ExecutionConfig()));
+}
diff --git a/flink-core/src/test/resources/flink-1.6-kryo-type-serializer-custom-data b/flink-core/src/test/resources/flink-1.6-kryo-type-serializer-custom-data
new file mode 100644
index 0000000..b273ce4
--- /dev/null
+++ b/flink-core/src/test/resources/flink-1.6-kryo-type-serializer-custom-data
@@ -0,0 +1 @@
+Scottis�Bobb�Scottis�Scottis�Scottis�Scottis�Scottis�Scottis�Scottis�Elvi�
\ No newline at end of file
diff --git a/flink-core/src/test/resources/flink-1.6-kryo-type-serializer-custom-snapshot b/flink-core/src/test/resources/flink-1.6-kryo-type-serializer-custom-snapshot
new file mode 100644
index 0000000..de61118
Binary files /dev/null and b/flink-core/src/test/resources/flink-1.6-kryo-type-serializer-custom-snapshot differ
diff --git a/flink-core/src/test/resources/flink-1.6-kryo-type-serializer-data b/flink-core/src/test/resources/flink-1.6-kryo-type-serializer-data
new file mode 100644
index 0000000..43cdb64
--- /dev/null
+++ b/flink-core/src/test/resources/flink-1.6-kryo-type-serializer-data
@@ -0,0 +1 @@
+PHome�Bob�
\ No newline at end of file
diff --git a/flink-core/src/test/resources/flink-1.6-kryo-type-serializer-empty-config-data b/flink-core/src/test/resources/flink-1.6-kryo-type-serializer-empty-config-data
new file mode 100644
index 0000000..0c66b7d
Binary files /dev/null and b/flink-core/src/test/resources/flink-1.6-kryo-type-serializer-empty-config-data differ
diff --git a/flink-core/src/test/resources/flink-1.6-kryo-type-serializer-empty-config-snapshot b/flink-core/src/test/resources/flink-1.6-kryo-type-serializer-empty-config-snapshot
new file mode 100644
index 0000000..eff068b
Binary files /dev/null and b/flink-core/src/test/resources/flink-1.6-kryo-type-serializer-empty-config-snapshot differ
diff --git a/flink-core/src/test/resources/flink-1.6-kryo-type-serializer-snapshot b/flink-core/src/test/resources/flink-1.6-kryo-type-serializer-snapshot
new file mode 100644
index 0000000..2af53a7
Binary files /dev/null and b/flink-core/src/test/resources/flink-1.6-kryo-type-serializer-snapshot differ
diff --git a/flink-core/src/test/resources/flink-1.7-kryo-type-serializer-custom-data b/flink-core/src/test/resources/flink-1.7-kryo-type-serializer-custom-data
new file mode 100644
index 0000000..b273ce4
--- /dev/null
+++ b/flink-core/src/test/resources/flink-1.7-kryo-type-serializer-custom-data
@@ -0,0 +1 @@
+Scottis�Bobb�Scottis�Scottis�Scottis�Scottis�Scottis�Scottis�Scottis�Elvi�
\ No newline at end of file
diff --git a/flink-core/src/test/resources/flink-1.7-kryo-type-serializer-custom-snapshot b/flink-core/src/test/resources/flink-1.7-kryo-type-serializer-custom-snapshot
new file mode 100644
index 0000000..4098c43
Binary files /dev/null and b/flink-core/src/test/resources/flink-1.7-kryo-type-serializer-custom-snapshot differ
diff --git a/flink-core/src/test/resources/flink-1.7-kryo-type-serializer-data b/flink-core/src/test/resources/flink-1.7-kryo-type-serializer-data
new file mode 100644
index 0000000..d56bc15
--- /dev/null
+++ b/flink-core/src/test/resources/flink-1.7-kryo-type-serializer-data
@@ -0,0 +1 @@
+PHome�Bob�Bob�Bob�Bob�Bob�Bob�Bob�Bob�Bob�
\ No newline at end of file
diff --git a/flink-core/src/test/resources/flink-1.7-kryo-type-serializer-empty-config-data b/flink-core/src/test/resources/flink-1.7-kryo-type-serializer-empty-config-data
new file mode 100644
index 0000000..62aa733
Binary files /dev/null and b/flink-core/src/test/resources/flink-1.7-kryo-type-serializer-empty-config-data differ
diff --git a/flink-core/src/test/resources/flink-1.7-kryo-type-serializer-empty-config-snapshot b/flink-core/src/test/resources/flink-1.7-kryo-type-serializer-empty-config-snapshot
new file mode 100644
index 0000000..87f7ba9
Binary files /dev/null and b/flink-core/src/test/resources/flink-1.7-kryo-type-serializer-empty-config-snapshot differ
diff --git a/flink-core/src/test/resources/flink-1.7-kryo-type-serializer-snapshot b/flink-core/src/test/resources/flink-1.7-kryo-type-serializer-snapshot
new file mode 100644
index 0000000..8117ac9
Binary files /dev/null and b/flink-core/src/test/resources/flink-1.7-kryo-type-serializer-snapshot differ


[flink] 10/17: [FLINK-11485] [core] Refactor multiple nested serializer compatibility resolution logic to CompositeTypeSerializerUtil

Posted by tz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tzulitai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit d5d45c8cc5924a6f387009eb6d0d053106055c3b
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
AuthorDate: Tue Feb 19 14:08:03 2019 +0800

    [FLINK-11485] [core] Refactor multiple nested serializer compatibility resolution logic to CompositeTypeSerializerUtil
    
    This commit refactors the logic of resolving overall compatibility
    results across multiple nested serializers of a composite serializer out
    of the CompositeTypeSerializerSnapshot class.
    
    This allows us to reuse this functionality when implementing the
    compatibility check for the PojoSerializerSnapshot
---
 .../typeutils/CompositeTypeSerializerSnapshot.java |  54 +---
 .../typeutils/CompositeTypeSerializerUtil.java     | 146 ++++++++++
 .../typeutils/CompositeTypeSerializerUtilTest.java | 149 +++++++++++
 .../SchemaCompatibilityTestingSerializer.java      | 296 +++++++++++++++++++++
 4 files changed, 597 insertions(+), 48 deletions(-)

diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerSnapshot.java
index d14a80e..0e8b2f3 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerSnapshot.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerSnapshot.java
@@ -26,10 +26,10 @@ import org.apache.flink.api.common.typeutils.base.MapSerializer;
 import org.apache.flink.api.java.typeutils.runtime.EitherSerializer;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
-import org.apache.flink.util.Preconditions;
 
 import java.io.IOException;
 
+import static org.apache.flink.api.common.typeutils.CompositeTypeSerializerUtil.IntermediateCompatibilityResult;
 import static org.apache.flink.util.Preconditions.checkNotNull;
 
 /**
@@ -304,57 +304,15 @@ public abstract class CompositeTypeSerializerSnapshot<T, S extends TypeSerialize
 			TypeSerializer<?>[] newNestedSerializers,
 			TypeSerializerSnapshot<?>[] nestedSerializerSnapshots) {
 
-		Preconditions.checkArgument(newNestedSerializers.length == nestedSerializerSnapshots.length,
-			"Different number of new serializers and existing serializer snapshots.");
-
-		TypeSerializer<?>[] reconfiguredNestedSerializers = new TypeSerializer[newNestedSerializers.length];
-
-		// check nested serializers for compatibility
-		boolean nestedSerializerRequiresMigration = false;
-		boolean hasReconfiguredNestedSerializers = false;
-		for (int i = 0; i < nestedSerializerSnapshots.length; i++) {
-			TypeSerializerSchemaCompatibility<?> compatibility =
-				resolveCompatibility(newNestedSerializers[i], nestedSerializerSnapshots[i]);
-
-			// if any one of the new nested serializers is incompatible, we can just short circuit the result
-			if (compatibility.isIncompatible()) {
-				return TypeSerializerSchemaCompatibility.incompatible();
-			}
-
-			if (compatibility.isCompatibleAfterMigration()) {
-				nestedSerializerRequiresMigration = true;
-			} else if (compatibility.isCompatibleWithReconfiguredSerializer()) {
-				hasReconfiguredNestedSerializers = true;
-				reconfiguredNestedSerializers[i] = compatibility.getReconfiguredSerializer();
-			} else if (compatibility.isCompatibleAsIs()) {
-				reconfiguredNestedSerializers[i] = newNestedSerializers[i];
-			} else {
-				throw new IllegalStateException("Undefined compatibility type.");
-			}
-		}
-
-		if (nestedSerializerRequiresMigration) {
-			return TypeSerializerSchemaCompatibility.compatibleAfterMigration();
-		}
+		IntermediateCompatibilityResult<T> intermediateResult =
+			CompositeTypeSerializerUtil.constructIntermediateCompatibilityResult(newNestedSerializers, nestedSerializerSnapshots);
 
-		if (hasReconfiguredNestedSerializers) {
+		if (intermediateResult.isCompatibleWithReconfiguredSerializer()) {
 			@SuppressWarnings("unchecked")
-			TypeSerializer<T> reconfiguredCompositeSerializer = createOuterSerializerWithNestedSerializers(reconfiguredNestedSerializers);
+			TypeSerializer<T> reconfiguredCompositeSerializer = createOuterSerializerWithNestedSerializers(intermediateResult.getNestedSerializers());
 			return TypeSerializerSchemaCompatibility.compatibleWithReconfiguredSerializer(reconfiguredCompositeSerializer);
 		}
 
-		// ends up here if everything is compatible as is
-		return TypeSerializerSchemaCompatibility.compatibleAsIs();
-	}
-
-	@SuppressWarnings("unchecked")
-	private static <E> TypeSerializerSchemaCompatibility<E> resolveCompatibility(
-		TypeSerializer<?> serializer,
-		TypeSerializerSnapshot<?> snapshot) {
-
-		TypeSerializer<E> typedSerializer = (TypeSerializer<E>) serializer;
-		TypeSerializerSnapshot<E> typedSnapshot = (TypeSerializerSnapshot<E>) snapshot;
-
-		return typedSnapshot.resolveSchemaCompatibility(typedSerializer);
+		return intermediateResult.getFinalResult();
 	}
 }
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerUtil.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerUtil.java
index f98bc72..8e1a65a 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerUtil.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerUtil.java
@@ -19,8 +19,11 @@
 package org.apache.flink.api.common.typeutils;
 
 import org.apache.flink.annotation.Internal;
+import org.apache.flink.util.Preconditions;
 
 import static org.apache.flink.util.Preconditions.checkArgument;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.apache.flink.util.Preconditions.checkState;
 
 /**
  * Utilities for the {@link CompositeTypeSerializerSnapshot}.
@@ -62,4 +65,147 @@ public class CompositeTypeSerializerUtil {
 		NestedSerializersSnapshotDelegate delegate = new NestedSerializersSnapshotDelegate(nestedSnapshots);
 		compositeSnapshot.setNestedSerializersSnapshotDelegate(delegate);
 	}
+
+	/**
+	 * Constructs an {@link IntermediateCompatibilityResult} with the given array of nested serializers and their
+	 * corresponding serializer snapshots.
+	 *
+	 * <p>This result is considered "intermediate", because the actual final result is not yet built if it isn't
+	 * defined. This is the case if the final result is supposed to be
+	 * {@link TypeSerializerSchemaCompatibility#compatibleWithReconfiguredSerializer(TypeSerializer)}, where
+	 * construction of the reconfigured serializer instance should be done by the caller.
+	 *
+	 * <p>For other cases, i.e. {@link TypeSerializerSchemaCompatibility#compatibleAsIs()},
+	 * {@link TypeSerializerSchemaCompatibility#compatibleAfterMigration()}, and
+	 * {@link TypeSerializerSchemaCompatibility#incompatible()}, these results are considered final.
+	 *
+	 * @param newNestedSerializers the new nested serializers to check for compatibility.
+	 * @param nestedSerializerSnapshots the associated nested serializers' snapshots.
+	 *
+	 * @return the intermediate compatibility result of the new nested serializers.
+	 */
+	public static <T> IntermediateCompatibilityResult<T> constructIntermediateCompatibilityResult(
+		TypeSerializer<?>[] newNestedSerializers,
+		TypeSerializerSnapshot<?>[] nestedSerializerSnapshots) {
+
+		Preconditions.checkArgument(newNestedSerializers.length == nestedSerializerSnapshots.length,
+			"Different number of new serializers and existing serializer snapshots.");
+
+		TypeSerializer<?>[] nestedSerializers = new TypeSerializer[newNestedSerializers.length];
+
+		// check nested serializers for compatibility
+		boolean nestedSerializerRequiresMigration = false;
+		boolean hasReconfiguredNestedSerializers = false;
+		for (int i = 0; i < nestedSerializerSnapshots.length; i++) {
+			TypeSerializerSchemaCompatibility<?> compatibility =
+				resolveCompatibility(newNestedSerializers[i], nestedSerializerSnapshots[i]);
+
+			// if any one of the new nested serializers is incompatible, we can just short circuit the result
+			if (compatibility.isIncompatible()) {
+				return IntermediateCompatibilityResult.definedIncompatibleResult();
+			}
+
+			if (compatibility.isCompatibleAfterMigration()) {
+				nestedSerializerRequiresMigration = true;
+			} else if (compatibility.isCompatibleWithReconfiguredSerializer()) {
+				hasReconfiguredNestedSerializers = true;
+				nestedSerializers[i] = compatibility.getReconfiguredSerializer();
+			} else if (compatibility.isCompatibleAsIs()) {
+				nestedSerializers[i] = newNestedSerializers[i];
+			} else {
+				throw new IllegalStateException("Undefined compatibility type.");
+			}
+		}
+
+		if (nestedSerializerRequiresMigration) {
+			return IntermediateCompatibilityResult.definedCompatibleAfterMigrationResult();
+		}
+
+		if (hasReconfiguredNestedSerializers) {
+			return IntermediateCompatibilityResult.undefinedReconfigureResult(nestedSerializers);
+		}
+
+		// ends up here if everything is compatible as is
+		return IntermediateCompatibilityResult.definedCompatibleAsIsResult(nestedSerializers);
+	}
+
+	public static class IntermediateCompatibilityResult<T> {
+
+		private final TypeSerializerSchemaCompatibility.Type compatibilityType;
+		private final TypeSerializer<?>[] nestedSerializers;
+
+		static <T> IntermediateCompatibilityResult<T> definedCompatibleAsIsResult(TypeSerializer<?>[] originalSerializers) {
+			return new IntermediateCompatibilityResult<>(TypeSerializerSchemaCompatibility.Type.COMPATIBLE_AS_IS, originalSerializers);
+		}
+
+		static <T> IntermediateCompatibilityResult<T> definedIncompatibleResult() {
+			return new IntermediateCompatibilityResult<>(TypeSerializerSchemaCompatibility.Type.INCOMPATIBLE, null);
+		}
+
+		static <T> IntermediateCompatibilityResult<T> definedCompatibleAfterMigrationResult() {
+			return new IntermediateCompatibilityResult<>(TypeSerializerSchemaCompatibility.Type.COMPATIBLE_AFTER_MIGRATION, null);
+		}
+
+		static <T> IntermediateCompatibilityResult<T> undefinedReconfigureResult(TypeSerializer<?>[] reconfiguredNestedSerializers) {
+			return new IntermediateCompatibilityResult<>(TypeSerializerSchemaCompatibility.Type.COMPATIBLE_WITH_RECONFIGURED_SERIALIZER, reconfiguredNestedSerializers);
+		}
+
+		private IntermediateCompatibilityResult(
+				TypeSerializerSchemaCompatibility.Type compatibilityType,
+				TypeSerializer<?>[] nestedSerializers) {
+			this.compatibilityType = checkNotNull(compatibilityType);
+			this.nestedSerializers = nestedSerializers;
+		}
+
+		public boolean isCompatibleWithReconfiguredSerializer() {
+			return compatibilityType == TypeSerializerSchemaCompatibility.Type.COMPATIBLE_WITH_RECONFIGURED_SERIALIZER;
+		}
+
+		public boolean isCompatibleAsIs() {
+			return compatibilityType == TypeSerializerSchemaCompatibility.Type.COMPATIBLE_AS_IS;
+		}
+
+		public boolean isCompatibleAfterMigration() {
+			return compatibilityType == TypeSerializerSchemaCompatibility.Type.COMPATIBLE_AFTER_MIGRATION;
+		}
+
+		public boolean isIncompatible() {
+			return compatibilityType == TypeSerializerSchemaCompatibility.Type.INCOMPATIBLE;
+		}
+
+		public TypeSerializerSchemaCompatibility<T> getFinalResult() {
+			checkState(
+				compatibilityType != TypeSerializerSchemaCompatibility.Type.COMPATIBLE_WITH_RECONFIGURED_SERIALIZER,
+				"unable to build final result if intermediate compatibility type is COMPATIBLE_WITH_RECONFIGURED_SERIALIZER.");
+			switch (compatibilityType) {
+				case COMPATIBLE_AS_IS:
+					return TypeSerializerSchemaCompatibility.compatibleAsIs();
+				case COMPATIBLE_AFTER_MIGRATION:
+					return TypeSerializerSchemaCompatibility.compatibleAfterMigration();
+				case INCOMPATIBLE:
+					return TypeSerializerSchemaCompatibility.incompatible();
+				default:
+					throw new IllegalStateException("unrecognized compatibility type.");
+			}
+		}
+
+		public TypeSerializer<?>[] getNestedSerializers() {
+			checkState(
+				compatibilityType == TypeSerializerSchemaCompatibility.Type.COMPATIBLE_AS_IS
+					|| compatibilityType == TypeSerializerSchemaCompatibility.Type.COMPATIBLE_WITH_RECONFIGURED_SERIALIZER,
+				"only intermediate compatibility types COMPATIBLE_AS_IS and COMPATIBLE_WITH_RECONFIGURED_SERIALIZER have nested serializers.");
+			return nestedSerializers;
+		}
+	}
+
+	@SuppressWarnings("unchecked")
+	private static <E> TypeSerializerSchemaCompatibility<E> resolveCompatibility(
+		TypeSerializer<?> serializer,
+		TypeSerializerSnapshot<?> snapshot) {
+
+		TypeSerializer<E> typedSerializer = (TypeSerializer<E>) serializer;
+		TypeSerializerSnapshot<E> typedSnapshot = (TypeSerializerSnapshot<E>) snapshot;
+
+		return typedSnapshot.resolveSchemaCompatibility(typedSerializer);
+	}
 }
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerUtilTest.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerUtilTest.java
new file mode 100644
index 0000000..37edcd7
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerUtilTest.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.typeutils;
+
+import org.apache.flink.api.common.typeutils.CompositeTypeSerializerUtil.IntermediateCompatibilityResult;
+import org.apache.flink.api.common.typeutils.base.IntSerializer;
+import org.apache.flink.testutils.migration.SchemaCompatibilityTestingSerializer;
+import org.junit.Test;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests for the {@link CompositeTypeSerializerUtil}.
+ */
+public class CompositeTypeSerializerUtilTest {
+
+	// ------------------------------------------------------------------------------------------------
+	//  Tests for CompositeTypeSerializerUtil#constructIntermediateCompatibilityResult
+	// ------------------------------------------------------------------------------------------------
+
+	@Test
+	public void testCompatibleAsIsIntermediateCompatibilityResult() {
+		final TypeSerializerSnapshot<?>[] testSerializerSnapshots = new TypeSerializerSnapshot<?>[] {
+			new SchemaCompatibilityTestingSerializer.InitialSerializer().snapshotConfiguration(),
+			new SchemaCompatibilityTestingSerializer.InitialSerializer().snapshotConfiguration(),
+		};
+
+		final TypeSerializer<?>[] testNewSerializers = new TypeSerializer<?>[] {
+			new SchemaCompatibilityTestingSerializer.InitialSerializer(),
+			new SchemaCompatibilityTestingSerializer.InitialSerializer(),
+		};
+
+		IntermediateCompatibilityResult<?> intermediateCompatibilityResult =
+			CompositeTypeSerializerUtil.constructIntermediateCompatibilityResult(testNewSerializers, testSerializerSnapshots);
+
+		assertTrue(intermediateCompatibilityResult.isCompatibleAsIs());
+		assertTrue(intermediateCompatibilityResult.getFinalResult().isCompatibleAsIs());
+		assertArrayEquals(testNewSerializers, intermediateCompatibilityResult.getNestedSerializers());
+	}
+
+	@Test
+	public void testCompatibleWithReconfiguredSerializerIntermediateCompatibilityResult() {
+		final TypeSerializerSnapshot<?>[] testSerializerSnapshots = new TypeSerializerSnapshot<?>[] {
+			new SchemaCompatibilityTestingSerializer.InitialSerializer().snapshotConfiguration(),
+			new SchemaCompatibilityTestingSerializer.InitialSerializer().snapshotConfiguration(),
+		};
+
+		final TypeSerializer<?>[] testNewSerializers = new TypeSerializer<?>[] {
+			new SchemaCompatibilityTestingSerializer.InitialSerializer(),
+			new SchemaCompatibilityTestingSerializer.ReconfigurationRequiringSerializer<>(),
+		};
+
+		IntermediateCompatibilityResult<?> intermediateCompatibilityResult =
+			CompositeTypeSerializerUtil.constructIntermediateCompatibilityResult(testNewSerializers, testSerializerSnapshots);
+
+		assertTrue(intermediateCompatibilityResult.isCompatibleWithReconfiguredSerializer());
+
+		final TypeSerializer<?>[] expectedReconfiguredNestedSerializers = new TypeSerializer<?>[] {
+			new SchemaCompatibilityTestingSerializer.InitialSerializer(),
+			new SchemaCompatibilityTestingSerializer.InitialSerializer(),
+		};
+		assertArrayEquals(expectedReconfiguredNestedSerializers, intermediateCompatibilityResult.getNestedSerializers());
+	}
+
+	@Test
+	public void testCompatibleAfterMigrationIntermediateCompatibilityResult() {
+		final TypeSerializerSnapshot<?>[] testSerializerSnapshots = new TypeSerializerSnapshot<?>[] {
+			new SchemaCompatibilityTestingSerializer.InitialSerializer().snapshotConfiguration(),
+			new SchemaCompatibilityTestingSerializer.InitialSerializer().snapshotConfiguration(),
+			new SchemaCompatibilityTestingSerializer.InitialSerializer().snapshotConfiguration(),
+		};
+
+		final TypeSerializer<?>[] testNewSerializers = new TypeSerializer<?>[] {
+			new SchemaCompatibilityTestingSerializer.ReconfigurationRequiringSerializer<>(),
+			new SchemaCompatibilityTestingSerializer.UpgradedSchemaSerializer<>(),
+			new SchemaCompatibilityTestingSerializer.InitialSerializer()
+		};
+
+		IntermediateCompatibilityResult<?> intermediateCompatibilityResult =
+			CompositeTypeSerializerUtil.constructIntermediateCompatibilityResult(testNewSerializers, testSerializerSnapshots);
+
+		assertTrue(intermediateCompatibilityResult.isCompatibleAfterMigration());
+		assertTrue(intermediateCompatibilityResult.getFinalResult().isCompatibleAfterMigration());
+	}
+
+	@Test
+	public void testIncompatibleIntermediateCompatibilityResult() {
+		final TypeSerializerSnapshot<?>[] testSerializerSnapshots = new TypeSerializerSnapshot<?>[] {
+			new SchemaCompatibilityTestingSerializer.InitialSerializer().snapshotConfiguration(),
+			new SchemaCompatibilityTestingSerializer.InitialSerializer().snapshotConfiguration(),
+			new SchemaCompatibilityTestingSerializer.InitialSerializer().snapshotConfiguration(),
+			new SchemaCompatibilityTestingSerializer.InitialSerializer().snapshotConfiguration(),
+		};
+
+		final TypeSerializer<?>[] testNewSerializers = new TypeSerializer<?>[] {
+			new SchemaCompatibilityTestingSerializer.InitialSerializer(),
+			new SchemaCompatibilityTestingSerializer.IncompatibleSerializer<>(),
+			new SchemaCompatibilityTestingSerializer.ReconfigurationRequiringSerializer<>(),
+			new SchemaCompatibilityTestingSerializer.UpgradedSchemaSerializer()
+		};
+
+		IntermediateCompatibilityResult<?> intermediateCompatibilityResult =
+			CompositeTypeSerializerUtil.constructIntermediateCompatibilityResult(testNewSerializers, testSerializerSnapshots);
+
+		assertTrue(intermediateCompatibilityResult.isIncompatible());
+		assertTrue(intermediateCompatibilityResult.getFinalResult().isIncompatible());
+	}
+
+	@Test(expected = IllegalStateException.class)
+	public void testGetFinalResultOnUndefinedReconfigureIntermediateCompatibilityResultFails() {
+		IntermediateCompatibilityResult<Integer> intermediateCompatibilityResult =
+			IntermediateCompatibilityResult.undefinedReconfigureResult(new TypeSerializer[]{ IntSerializer.INSTANCE });
+
+		intermediateCompatibilityResult.getFinalResult();
+	}
+
+	@Test(expected = IllegalStateException.class)
+	public void testGetNestedSerializersOnCompatibleAfterMigrationIntermediateCompatibilityResultFails() {
+		IntermediateCompatibilityResult<Integer> intermediateCompatibilityResult =
+			IntermediateCompatibilityResult.definedCompatibleAfterMigrationResult();
+
+		intermediateCompatibilityResult.getNestedSerializers();
+	}
+
+	@Test(expected = IllegalStateException.class)
+	public void testGetNestedSerializersOnIncompatibleIntermediateCompatibilityResultFails() {
+		IntermediateCompatibilityResult<Integer> intermediateCompatibilityResult =
+			IntermediateCompatibilityResult.definedIncompatibleResult();
+
+		intermediateCompatibilityResult.getNestedSerializers();
+	}
+}
diff --git a/flink-core/src/test/java/org/apache/flink/testutils/migration/SchemaCompatibilityTestingSerializer.java b/flink-core/src/test/java/org/apache/flink/testutils/migration/SchemaCompatibilityTestingSerializer.java
new file mode 100644
index 0000000..253eed7
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/testutils/migration/SchemaCompatibilityTestingSerializer.java
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.testutils.migration;
+
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
+import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+
+import java.io.IOException;
+
+/**
+ * The {@link SchemaCompatibilityTestingSerializer} is a mock serializer that can be used
+ * for schema compatibility and serializer upgrade related tests.
+ *
+ * <p>To use this, tests should always first instantiate an {@link InitialSerializer} serializer
+ * instance and then take a snapshot of it. Next, depending on what the new serializer is provided
+ * to the taken snapshot, tests can expect the following compatibility results:
+ * <ul>
+ *     <li>{@link InitialSerializer} as the new serializer: the compatibility result will be
+ *         {@link TypeSerializerSchemaCompatibility#compatibleAsIs()}.</li>
+ *     <li>{@link UpgradedSchemaSerializer} as the new serializer: the compatibility result will be
+ *         {@link TypeSerializerSchemaCompatibility#compatibleAfterMigration()}.</li>
+ *     <li>{@link ReconfigurationRequiringSerializer} as the new serializer: the compatibility result
+ *         will be {@link TypeSerializerSchemaCompatibility#compatibleWithReconfiguredSerializer(TypeSerializer)},
+ *         with a new instance of the {@link InitialSerializer} as the wrapped reconfigured serializer.</li>
+ *     <li>{@link IncompatibleSerializer} as the new serializer: the compatibility result will be
+ *         {@link TypeSerializerSchemaCompatibility#incompatible()}.</li>
+ * </ul>
+ */
+public abstract class SchemaCompatibilityTestingSerializer<T> extends TypeSerializer<T> {
+
+	private static final long serialVersionUID = 1L;
+
+	// ------------------------------------------------------------------------------------------------
+	//  Irrelevant serializer methods
+	// ------------------------------------------------------------------------------------------------
+
+	@Override
+	public TypeSerializer<T> duplicate() {
+		throw new UnsupportedOperationException(
+			"this is a SchemaCompatibilityTestingSerializer; should have only been used for snapshotting / compatibility test purposes.");
+	}
+
+	@Override
+	public void serialize(T record, DataOutputView target) throws IOException {
+		throw new UnsupportedOperationException(
+			"this is a SchemaCompatibilityTestingSerializer; should have only been used for snapshotting / compatibility test purposes.");
+
+	}
+
+	@Override
+	public T deserialize(T reuse, DataInputView source) throws IOException {
+		throw new UnsupportedOperationException(
+			"this is a SchemaCompatibilityTestingSerializer; should have only been used for snapshotting / compatibility test purposes.");
+
+	}
+
+	@Override
+	public T deserialize(DataInputView source) throws IOException {
+		throw new UnsupportedOperationException(
+			"this is a SchemaCompatibilityTestingSerializer; should have only been used for snapshotting / compatibility test purposes.");
+
+	}
+
+	@Override
+	public void copy(DataInputView source, DataOutputView target) throws IOException {
+		throw new UnsupportedOperationException(
+			"this is a SchemaCompatibilityTestingSerializer; should have only been used for snapshotting / compatibility test purposes.");
+
+	}
+
+	@Override
+	public T copy(T from) {
+		throw new UnsupportedOperationException(
+			"this is a SchemaCompatibilityTestingSerializer; should have only been used for snapshotting / compatibility test purposes.");
+
+	}
+
+	@Override
+	public T copy(T from, T reuse) {
+		throw new UnsupportedOperationException(
+			"this is a SchemaCompatibilityTestingSerializer; should have only been used for snapshotting / compatibility test purposes.");
+
+	}
+
+	@Override
+	public T createInstance() {
+		throw new UnsupportedOperationException(
+			"this is a SchemaCompatibilityTestingSerializer; should have only been used for snapshotting / compatibility test purposes.");
+
+	}
+
+	@Override
+	public boolean isImmutableType() {
+		throw new UnsupportedOperationException(
+			"this is a SchemaCompatibilityTestingSerializer; should have only been used for snapshotting / compatibility test purposes.");
+
+	}
+
+	@Override
+	public int getLength() {
+		throw new UnsupportedOperationException(
+			"this is a SchemaCompatibilityTestingSerializer; should have only been used for snapshotting / compatibility test purposes.");
+
+	}
+
+	// ------------------------------------------------------------------------------------------------
+	//  Test serializer variants
+	// ------------------------------------------------------------------------------------------------
+
+	public static class InitialSerializer<T> extends SchemaCompatibilityTestingSerializer<T> {
+
+		private static final long serialVersionUID = 1L;
+
+		@Override
+		public TypeSerializerSnapshot<T> snapshotConfiguration() {
+			return new InitialSerializerSnapshot();
+		}
+
+		@Override
+		public boolean equals(Object obj) {
+			return obj != null && obj.getClass() == InitialSerializer.class;
+		}
+
+		@Override
+		public int hashCode() {
+			return getClass().hashCode();
+		}
+	}
+
+	public static class UpgradedSchemaSerializer<T> extends SchemaCompatibilityTestingSerializer<T> {
+
+		private static final long serialVersionUID = 1L;
+
+		@Override
+		public TypeSerializerSnapshot<T> snapshotConfiguration() {
+			return new UpgradedSchemaSerializerSnapshot();
+		}
+
+		@Override
+		public boolean equals(Object obj) {
+			return obj != null && obj.getClass() == UpgradedSchemaSerializer.class;
+		}
+
+		@Override
+		public int hashCode() {
+			return getClass().hashCode();
+		}
+	}
+
+	public static class ReconfigurationRequiringSerializer<T> extends SchemaCompatibilityTestingSerializer<T> {
+
+		private static final long serialVersionUID = 1L;
+
+		@Override
+		public TypeSerializerSnapshot<T> snapshotConfiguration() {
+			throw new UnsupportedOperationException(
+				"this is a ReconfigurationRequiringSerializer; should not have been snapshotted.");
+		}
+
+		@Override
+		public boolean equals(Object obj) {
+			return obj != null && obj.getClass() == ReconfigurationRequiringSerializer.class;
+		}
+
+		@Override
+		public int hashCode() {
+			return getClass().hashCode();
+		}
+	}
+
+	public static class IncompatibleSerializer<T> extends SchemaCompatibilityTestingSerializer<T> {
+
+		private static final long serialVersionUID = 1L;
+
+		@Override
+		public TypeSerializerSnapshot<T> snapshotConfiguration() {
+			throw new UnsupportedOperationException(
+				"this is a IncompatibleSerializer; should not have been snapshotted.");
+		}
+
+		@Override
+		public boolean equals(Object obj) {
+			return obj != null && obj.getClass() == IncompatibleSerializer.class;
+		}
+
+		@Override
+		public int hashCode() {
+			return getClass().hashCode();
+		}
+	}
+
+	// ------------------------------------------------------------------------------------------------
+	//  Test serializer snapshots
+	// ------------------------------------------------------------------------------------------------
+
+	public class InitialSerializerSnapshot implements TypeSerializerSnapshot<T> {
+
+		@Override
+		public int getCurrentVersion() {
+			return 1;
+		}
+
+		@Override
+		public void writeSnapshot(DataOutputView out) throws IOException {}
+
+		@Override
+		public void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException {}
+
+		@Override
+		public TypeSerializer<T> restoreSerializer() {
+			return new SchemaCompatibilityTestingSerializer.InitialSerializer<>();
+		}
+
+		@Override
+		public TypeSerializerSchemaCompatibility<T> resolveSchemaCompatibility(TypeSerializer<T> newSerializer) {
+			if (newSerializer instanceof SchemaCompatibilityTestingSerializer.InitialSerializer) {
+				return TypeSerializerSchemaCompatibility.compatibleAsIs();
+			}
+
+			if (newSerializer instanceof SchemaCompatibilityTestingSerializer.UpgradedSchemaSerializer) {
+				return TypeSerializerSchemaCompatibility.compatibleAfterMigration();
+			}
+
+			if (newSerializer instanceof SchemaCompatibilityTestingSerializer.ReconfigurationRequiringSerializer) {
+				return TypeSerializerSchemaCompatibility.compatibleWithReconfiguredSerializer(new SchemaCompatibilityTestingSerializer.InitialSerializer<T>());
+			}
+
+			if (newSerializer instanceof SchemaCompatibilityTestingSerializer.IncompatibleSerializer) {
+				return TypeSerializerSchemaCompatibility.incompatible();
+			}
+
+			// reaches here if newSerializer isn't a SchemaCompatibilityTestingSerializer
+			return TypeSerializerSchemaCompatibility.incompatible();
+		}
+	}
+
+	public class UpgradedSchemaSerializerSnapshot implements TypeSerializerSnapshot<T> {
+
+		@Override
+		public int getCurrentVersion() {
+			return 1;
+		}
+
+		@Override
+		public void writeSnapshot(DataOutputView out) throws IOException {}
+
+		@Override
+		public void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException {}
+
+		@Override
+		public TypeSerializer<T> restoreSerializer() {
+			return new SchemaCompatibilityTestingSerializer.UpgradedSchemaSerializer<>();
+		}
+
+		@Override
+		public TypeSerializerSchemaCompatibility<T> resolveSchemaCompatibility(TypeSerializer<T> newSerializer) {
+			if (newSerializer instanceof SchemaCompatibilityTestingSerializer.InitialSerializer) {
+				return TypeSerializerSchemaCompatibility.incompatible();
+			}
+
+			if (newSerializer instanceof SchemaCompatibilityTestingSerializer.UpgradedSchemaSerializer) {
+				return TypeSerializerSchemaCompatibility.compatibleAsIs();
+			}
+
+			if (newSerializer instanceof SchemaCompatibilityTestingSerializer.ReconfigurationRequiringSerializer) {
+				return TypeSerializerSchemaCompatibility.compatibleWithReconfiguredSerializer(new SchemaCompatibilityTestingSerializer.UpgradedSchemaSerializer<T>());
+			}
+
+			if (newSerializer instanceof SchemaCompatibilityTestingSerializer.IncompatibleSerializer) {
+				return TypeSerializerSchemaCompatibility.incompatible();
+			}
+
+			// reaches here if newSerializer isn't a SchemaCompatibilityTestingSerializer
+			return TypeSerializerSchemaCompatibility.incompatible();
+		}
+	}
+}


[flink] 07/17: [FLINK-11323] Migrate TrySerializer to use new serialization compatibility abstractions

Posted by tz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tzulitai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 186b8df4155a4c171d71f1c806290bd94374416c
Author: Igal Shilman <ig...@data-artisans.com>
AuthorDate: Sat Feb 9 21:16:25 2019 +0100

    [FLINK-11323] Migrate TrySerializer to use new serialization compatibility abstractions
    
    This closes #7496.
---
 .../ScalaTrySerializerConfigSnapshot.java          |  13 ++++
 .../typeutils/ScalaTrySerializerSnapshot.java      |  71 ++++++++++++++++++++
 .../flink/api/scala/typeutils/TrySerializer.scala  |  73 ++++++++-------------
 .../ScalaTrySerializerSnapshotMigrationTest.java   |  59 +++++++++++++++++
 .../resources/flink-1.6-scala-try-serializer-data  | Bin 0 -> 1658 bytes
 .../flink-1.6-scala-try-serializer-snapshot        | Bin 0 -> 7054 bytes
 .../resources/flink-1.7-scala-try-serializer-data  | Bin 0 -> 1658 bytes
 .../flink-1.7-scala-try-serializer-snapshot        | Bin 0 -> 8932 bytes
 8 files changed, 171 insertions(+), 45 deletions(-)

diff --git a/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaTrySerializerConfigSnapshot.java b/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaTrySerializerConfigSnapshot.java
index 72baca4..bcd31b9 100644
--- a/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaTrySerializerConfigSnapshot.java
+++ b/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaTrySerializerConfigSnapshot.java
@@ -19,8 +19,10 @@
 package org.apache.flink.api.scala.typeutils;
 
 import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot;
+import org.apache.flink.api.common.typeutils.CompositeTypeSerializerUtil;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
+import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
 
 import scala.util.Try;
 
@@ -31,6 +33,7 @@ import scala.util.Try;
  * allow calling different base class constructors from subclasses, while we need that
  * for the default empty constructor.
  */
+@Deprecated
 public class ScalaTrySerializerConfigSnapshot<E> extends CompositeTypeSerializerConfigSnapshot<Try<E>> {
 
 	private static final int VERSION = 1;
@@ -49,4 +52,14 @@ public class ScalaTrySerializerConfigSnapshot<E> extends CompositeTypeSerializer
 	public int getVersion() {
 		return VERSION;
 	}
+
+	@Override
+	public TypeSerializerSchemaCompatibility<Try<E>> resolveSchemaCompatibility(TypeSerializer<Try<E>> newSerializer) {
+
+		return CompositeTypeSerializerUtil.delegateCompatibilityCheckToNewSnapshot(
+			newSerializer,
+			new ScalaTrySerializerSnapshot<>(),
+			getNestedSerializersAndConfigs().get(0).f1,
+			getNestedSerializersAndConfigs().get(1).f1);
+	}
 }
diff --git a/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaTrySerializerSnapshot.java b/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaTrySerializerSnapshot.java
new file mode 100644
index 0000000..e104838
--- /dev/null
+++ b/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaTrySerializerSnapshot.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.scala.typeutils;
+
+import org.apache.flink.api.common.typeutils.CompositeTypeSerializerSnapshot;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
+
+import scala.util.Try;
+
+/**
+ * A {@link TypeSerializerSnapshot} for the Scala {@link TrySerializer}.
+ *
+ * <p>This configuration snapshot class is implemented in Java because Scala does not
+ * allow calling different base class constructors from subclasses, while we need that
+ * for the default empty constructor.
+ */
+public class ScalaTrySerializerSnapshot<E> extends CompositeTypeSerializerSnapshot<Try<E>, TrySerializer<E>> {
+
+	private static final int VERSION = 2;
+
+	/** This empty nullary constructor is required for deserializing the configuration. */
+	@SuppressWarnings("unused")
+	public ScalaTrySerializerSnapshot() {
+		super(correspondingSerializerClass());
+	}
+
+	public ScalaTrySerializerSnapshot(TrySerializer<E> trySerializer) {
+		super(trySerializer);
+	}
+
+	@Override
+	protected int getCurrentOuterSnapshotVersion() {
+		return VERSION;
+	}
+
+	@Override
+	protected TypeSerializer<?>[] getNestedSerializers(TrySerializer<E> outerSerializer) {
+		return new TypeSerializer[]{outerSerializer.getElementSerializer(), outerSerializer.getThrowableSerializer()};
+	}
+
+	@Override
+	@SuppressWarnings("unchecked")
+	protected TrySerializer<E> createOuterSerializerWithNestedSerializers(TypeSerializer<?>[] nestedSerializers) {
+		TypeSerializer<E> elementSerializer = (TypeSerializer<E>) nestedSerializers[0];
+		TypeSerializer<Throwable> valueSerializer = (TypeSerializer<Throwable>) nestedSerializers[1];
+
+		return new TrySerializer<>(elementSerializer, valueSerializer);
+	}
+
+	@SuppressWarnings("unchecked")
+	private static <E> Class<TrySerializer<E>> correspondingSerializerClass() {
+		return (Class<TrySerializer<E>>) (Class<?>) TrySerializer.class;
+	}
+}
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TrySerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TrySerializer.scala
index 8c92f77..cf39529 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TrySerializer.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TrySerializer.scala
@@ -32,12 +32,21 @@ import scala.util.{Failure, Success, Try}
 @SerialVersionUID(-3052182891252564491L)
 class TrySerializer[A](
     private val elemSerializer: TypeSerializer[A],
-    private val executionConfig: ExecutionConfig)
+    private val throwableSerializer: TypeSerializer[Throwable])
   extends TypeSerializer[Try[A]] {
 
-  override def duplicate: TrySerializer[A] = this
+  private[typeutils] def this(elemSerializer: TypeSerializer[A],
+                              executionConfig: ExecutionConfig) = {
+    this(
+      elemSerializer,
+      new KryoSerializer[Throwable](classOf[Throwable], executionConfig)
+    )
+  }
 
-  val throwableSerializer = new KryoSerializer[Throwable](classOf[Throwable], executionConfig)
+  override def duplicate: TrySerializer[A] = new TrySerializer[A](
+    elemSerializer.duplicate(),
+    throwableSerializer.duplicate()
+  )
 
   override def createInstance: Try[A] = {
     Failure(new RuntimeException("Empty Failure"))
@@ -93,55 +102,18 @@ class TrySerializer[A](
   }
 
   override def hashCode(): Int = {
-    31 * elemSerializer.hashCode() + executionConfig.hashCode()
+    31 * elemSerializer.hashCode() + throwableSerializer.hashCode()
   }
 
   // --------------------------------------------------------------------------------------------
   // Serializer configuration snapshotting & compatibility
   // --------------------------------------------------------------------------------------------
 
-  override def snapshotConfiguration(): ScalaTrySerializerConfigSnapshot[A] = {
-    new ScalaTrySerializerConfigSnapshot[A](elemSerializer, throwableSerializer)
-  }
+  private[typeutils] def getElementSerializer: TypeSerializer[A] = elemSerializer
+  private[typeutils] def getThrowableSerializer: TypeSerializer[Throwable] = throwableSerializer
 
-  override def ensureCompatibility(
-      configSnapshot: TypeSerializerConfigSnapshot[_]): CompatibilityResult[Try[A]] = {
-
-    configSnapshot match {
-      case trySerializerConfigSnapshot
-          : ScalaTrySerializerConfigSnapshot[A] =>
-        ensureCompatibilityInternal(trySerializerConfigSnapshot)
-      case legacyTrySerializerConfigSnapshot
-          : TrySerializer.TrySerializerConfigSnapshot[A] =>
-        ensureCompatibilityInternal(legacyTrySerializerConfigSnapshot)
-      case _ => CompatibilityResult.requiresMigration()
-    }
-  }
-
-  private def ensureCompatibilityInternal(
-      compositeConfigSnapshot: CompositeTypeSerializerConfigSnapshot[Try[A]])
-        : CompatibilityResult[Try[A]] = {
-
-    val previousSerializersAndConfigs =
-      compositeConfigSnapshot.getNestedSerializersAndConfigs
-
-    val elemCompatRes = CompatibilityUtil.resolveCompatibilityResult(
-      previousSerializersAndConfigs.get(0).f0,
-      classOf[UnloadableDummyTypeSerializer[_]],
-      previousSerializersAndConfigs.get(0).f1,
-      elemSerializer)
-
-    val throwableCompatRes = CompatibilityUtil.resolveCompatibilityResult(
-      previousSerializersAndConfigs.get(1).f0,
-      classOf[UnloadableDummyTypeSerializer[_]],
-      previousSerializersAndConfigs.get(1).f1,
-      throwableSerializer)
-
-    if (elemCompatRes.isRequiresMigration || throwableCompatRes.isRequiresMigration) {
-      CompatibilityResult.requiresMigration()
-    } else {
-      CompatibilityResult.compatible()
-    }
+  override def snapshotConfiguration(): ScalaTrySerializerSnapshot[A] = {
+    new ScalaTrySerializerSnapshot[A](this)
   }
 }
 
@@ -151,10 +123,21 @@ object TrySerializer {
     * We need to keep this to be compatible with snapshots taken in Flink 1.3.0.
     * Once Flink 1.3.x is no longer supported, this can be removed.
     */
+  @Deprecated
   class TrySerializerConfigSnapshot[A]()
       extends CompositeTypeSerializerConfigSnapshot[Try[A]]() {
 
     override def getVersion: Int = TrySerializerConfigSnapshot.VERSION
+
+    override def resolveSchemaCompatibility(
+      newSerializer: TypeSerializer[Try[A]]
+    ): TypeSerializerSchemaCompatibility[Try[A]] =
+
+      CompositeTypeSerializerUtil.delegateCompatibilityCheckToNewSnapshot(
+        newSerializer,
+        new ScalaTrySerializerSnapshot[A](),
+        getNestedSerializersAndConfigs.get(0).f1,
+        getNestedSerializersAndConfigs.get(1).f1)
   }
 
   object TrySerializerConfigSnapshot {
diff --git a/flink-scala/src/test/java/org/apache/flink/api/scala/typeutils/ScalaTrySerializerSnapshotMigrationTest.java b/flink-scala/src/test/java/org/apache/flink/api/scala/typeutils/ScalaTrySerializerSnapshotMigrationTest.java
new file mode 100644
index 0000000..6fd66b1
--- /dev/null
+++ b/flink-scala/src/test/java/org/apache/flink/api/scala/typeutils/ScalaTrySerializerSnapshotMigrationTest.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.scala.typeutils;
+
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.typeutils.TypeSerializerSnapshotMigrationTestBase;
+import org.apache.flink.api.common.typeutils.base.StringSerializer;
+import org.apache.flink.testutils.migration.MigrationVersion;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.Collection;
+
+import scala.util.Try;
+
+/**
+ * Migration test for the {@link TrySerializer}.
+ */
+@RunWith(Parameterized.class)
+public class ScalaTrySerializerSnapshotMigrationTest extends TypeSerializerSnapshotMigrationTestBase<Try<String>> {
+
+	private static final String SPEC_NAME = "scala-try-serializer";
+
+	public ScalaTrySerializerSnapshotMigrationTest(TestSpecification<Try<String>> testSpecification) {
+		super(testSpecification);
+	}
+
+	@SuppressWarnings("unchecked")
+	@Parameterized.Parameters(name = "Test Specification = {0}")
+	public static Collection<TestSpecification<?>> testSpecifications() {
+
+		final TestSpecifications testSpecifications = new TestSpecifications(MigrationVersion.v1_6, MigrationVersion.v1_7);
+
+		testSpecifications.add(
+			SPEC_NAME,
+			TrySerializer.class,
+			ScalaTrySerializerSnapshot.class,
+			() -> new TrySerializer<>(StringSerializer.INSTANCE, new ExecutionConfig()));
+
+		return testSpecifications.get();
+	}
+}
diff --git a/flink-scala/src/test/resources/flink-1.6-scala-try-serializer-data b/flink-scala/src/test/resources/flink-1.6-scala-try-serializer-data
new file mode 100644
index 0000000..b5739a2
Binary files /dev/null and b/flink-scala/src/test/resources/flink-1.6-scala-try-serializer-data differ
diff --git a/flink-scala/src/test/resources/flink-1.6-scala-try-serializer-snapshot b/flink-scala/src/test/resources/flink-1.6-scala-try-serializer-snapshot
new file mode 100644
index 0000000..43cc75e
Binary files /dev/null and b/flink-scala/src/test/resources/flink-1.6-scala-try-serializer-snapshot differ
diff --git a/flink-scala/src/test/resources/flink-1.7-scala-try-serializer-data b/flink-scala/src/test/resources/flink-1.7-scala-try-serializer-data
new file mode 100644
index 0000000..585c1a5
Binary files /dev/null and b/flink-scala/src/test/resources/flink-1.7-scala-try-serializer-data differ
diff --git a/flink-scala/src/test/resources/flink-1.7-scala-try-serializer-snapshot b/flink-scala/src/test/resources/flink-1.7-scala-try-serializer-snapshot
new file mode 100644
index 0000000..760da78
Binary files /dev/null and b/flink-scala/src/test/resources/flink-1.7-scala-try-serializer-snapshot differ


[flink] 13/17: [FLINK-11485] [tests] Adjust existing PojoSerializer upgrade tests

Posted by tz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tzulitai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 69bf1d6a2b8c5f922e5860ca341a0a40332e10ca
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
AuthorDate: Tue Feb 19 20:00:06 2019 +0800

    [FLINK-11485] [tests] Adjust existing PojoSerializer upgrade tests
    
    The following tests have been adjusted to new scope of functionality
    related to upgrading the PojoSerializer: PojoSerializerTest and
    PojoSerializerUpgradeTest.
    
    In PojoSerializerTest, assertions that were supposed to check for
    "isCompatibleWithReconfiguredSerializer" cases have been adjusted to
    correctly do so. Also, a test where changing field order was tested was
    removed, since the new PojoSerializerSnapshot always assumes that fields
    have been sorted already by the type extractor.
    
    In PojoSerializerUpgradeTest, tests which remove / add fields to POJO
    classes were updated to no longer expect failures. They were expecting
    failures because POJOs were not evolvable in the past.
    
    The updates to these tests also provide coverage for the POJO schema
    evolution feature.
    
    This closes #7759.
---
 .../java/typeutils/runtime/PojoSerializerTest.java | 157 +++------------------
 .../PojoSerializerUpgradeTest.java                 |  66 ++-------
 2 files changed, 33 insertions(+), 190 deletions(-)

diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java
index ef66403..a45508d 100644
--- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java
@@ -29,9 +29,7 @@ import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
 import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
 import org.apache.flink.api.common.typeutils.TypeSerializerSnapshotSerializationUtil;
-import org.apache.flink.api.common.typeutils.UnloadableDummyTypeSerializer;
 import org.apache.flink.api.java.tuple.Tuple1;
-import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.api.java.tuple.Tuple3;
 import org.apache.flink.api.java.typeutils.PojoTypeInfo;
 import org.apache.flink.api.java.typeutils.TupleTypeInfo;
@@ -43,13 +41,9 @@ import org.junit.Test;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
-import java.lang.reflect.Field;
 import java.util.ArrayList;
 import java.util.Date;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.Objects;
 import java.util.Random;
 
@@ -347,12 +341,14 @@ public class PojoSerializerTest extends SerializerTestBase<PojoSerializerTest.Te
 		@SuppressWarnings("unchecked")
 		TypeSerializerSchemaCompatibility<TestUserClass> compatResult =
 			pojoSerializerConfigSnapshot.resolveSchemaCompatibility(pojoSerializer);
-		assertTrue(compatResult.isCompatibleAsIs());
+		assertTrue(compatResult.isCompatibleWithReconfiguredSerializer());
+		assertTrue(compatResult.getReconfiguredSerializer() instanceof PojoSerializer);
 
 		// reconfigure - check reconfiguration result and that registration ids remains the same
 		//assertEquals(ReconfigureResult.COMPATIBLE, pojoSerializer.reconfigure(pojoSerializerConfigSnapshot));
-		assertEquals(subClassATag, pojoSerializer.getRegisteredClasses().get(SubTestUserClassA.class).intValue());
-		assertEquals(subClassBTag, pojoSerializer.getRegisteredClasses().get(SubTestUserClassB.class).intValue());
+		PojoSerializer<TestUserClass> reconfiguredPojoSerializer = (PojoSerializer<TestUserClass>) compatResult.getReconfiguredSerializer();
+		assertEquals(subClassATag, reconfiguredPojoSerializer.getRegisteredClasses().get(SubTestUserClassA.class).intValue());
+		assertEquals(subClassBTag, reconfiguredPojoSerializer.getRegisteredClasses().get(SubTestUserClassB.class).intValue());
 	}
 
 	/**
@@ -394,10 +390,13 @@ public class PojoSerializerTest extends SerializerTestBase<PojoSerializerTest.Te
 		@SuppressWarnings("unchecked")
 		TypeSerializerSchemaCompatibility<TestUserClass> compatResult =
 			pojoSerializerConfigSnapshot.resolveSchemaCompatibility(pojoSerializer);
-		assertTrue(compatResult.isCompatibleAsIs());
-		assertEquals(2, pojoSerializer.getSubclassSerializerCache().size());
-		assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class));
-		assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassB.class));
+		assertTrue(compatResult.isCompatibleWithReconfiguredSerializer());
+		assertTrue(compatResult.getReconfiguredSerializer() instanceof PojoSerializer);
+
+		PojoSerializer<TestUserClass> reconfiguredPojoSerializer = (PojoSerializer<TestUserClass>) compatResult.getReconfiguredSerializer();
+		assertEquals(2, reconfiguredPojoSerializer.getSubclassSerializerCache().size());
+		assertTrue(reconfiguredPojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class));
+		assertTrue(reconfiguredPojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassB.class));
 	}
 
 	/**
@@ -459,127 +458,15 @@ public class PojoSerializerTest extends SerializerTestBase<PojoSerializerTest.Te
 		@SuppressWarnings("unchecked")
 		TypeSerializerSchemaCompatibility<TestUserClass> compatResult =
 			pojoSerializerConfigSnapshot.resolveSchemaCompatibility(pojoSerializer);
-		assertTrue(compatResult.isCompatibleAsIs());
-		assertEquals(2, pojoSerializer.getSubclassSerializerCache().size());
-		assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class));
-		assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassB.class));
-		assertEquals(2, pojoSerializer.getRegisteredClasses().size());
-		assertTrue(pojoSerializer.getRegisteredClasses().containsKey(SubTestUserClassA.class));
-		assertTrue(pojoSerializer.getRegisteredClasses().containsKey(SubTestUserClassB.class));
-	}
-
-	/**
-	 * Verifies that reconfiguration reorders the fields of the new Pojo serializer to remain the same.
-	 */
-	@Test
-	public void testReconfigureWithDifferentFieldOrder() throws Exception {
-		Field[] mockOriginalFieldOrder = {
-			TestUserClass.class.getField("dumm4"),
-			TestUserClass.class.getField("dumm3"),
-			TestUserClass.class.getField("nestedClass"),
-			TestUserClass.class.getField("dumm1"),
-			TestUserClass.class.getField("dumm2"),
-			TestUserClass.class.getField("dumm5"),
-		};
-
-		// creating this serializer just for generating config snapshots of the field serializers
-		PojoSerializer<TestUserClass> ser = (PojoSerializer<TestUserClass>) type.createSerializer(new ExecutionConfig());
-
-		LinkedHashMap<String, Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> mockOriginalFieldToSerializerConfigSnapshot =
-			new LinkedHashMap<>(mockOriginalFieldOrder.length);
-		mockOriginalFieldToSerializerConfigSnapshot.put(
-			mockOriginalFieldOrder[0].getName(),
-			new Tuple2<>(
-				ser.getFieldSerializers()[3],
-				ser.getFieldSerializers()[3].snapshotConfiguration()));
-		mockOriginalFieldToSerializerConfigSnapshot.put(
-			mockOriginalFieldOrder[1].getName(),
-			new Tuple2<>(
-				ser.getFieldSerializers()[2],
-				ser.getFieldSerializers()[2].snapshotConfiguration()));
-		mockOriginalFieldToSerializerConfigSnapshot.put(
-			mockOriginalFieldOrder[2].getName(),
-			new Tuple2<>(
-				ser.getFieldSerializers()[5],
-				ser.getFieldSerializers()[5].snapshotConfiguration()));
-		mockOriginalFieldToSerializerConfigSnapshot.put(
-			mockOriginalFieldOrder[3].getName(),
-			new Tuple2<>(
-				ser.getFieldSerializers()[0],
-				ser.getFieldSerializers()[0].snapshotConfiguration()));
-		mockOriginalFieldToSerializerConfigSnapshot.put(
-			mockOriginalFieldOrder[4].getName(),
-			new Tuple2<>(
-				ser.getFieldSerializers()[1],
-				ser.getFieldSerializers()[1].snapshotConfiguration()));
-		mockOriginalFieldToSerializerConfigSnapshot.put(
-			mockOriginalFieldOrder[5].getName(),
-			new Tuple2<>(
-				ser.getFieldSerializers()[4],
-				ser.getFieldSerializers()[4].snapshotConfiguration()));
-
-		PojoSerializer<TestUserClass> pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(new ExecutionConfig());
-
-		assertEquals(TestUserClass.class.getField("dumm1"), pojoSerializer.getFields()[0]);
-		assertEquals(TestUserClass.class.getField("dumm2"), pojoSerializer.getFields()[1]);
-		assertEquals(TestUserClass.class.getField("dumm3"), pojoSerializer.getFields()[2]);
-		assertEquals(TestUserClass.class.getField("dumm4"), pojoSerializer.getFields()[3]);
-		assertEquals(TestUserClass.class.getField("dumm5"), pojoSerializer.getFields()[4]);
-		assertEquals(TestUserClass.class.getField("nestedClass"), pojoSerializer.getFields()[5]);
-
-		PojoSerializer.PojoSerializerConfigSnapshot<TestUserClass> mockPreviousConfigSnapshot =
-			new PojoSerializer.PojoSerializerConfigSnapshot<>(
-				TestUserClass.class,
-				mockOriginalFieldToSerializerConfigSnapshot, // this mocks the previous field order
-				new LinkedHashMap<>(), // empty; irrelevant for this test
-				new HashMap<>()); // empty; irrelevant for this test
-
-		// reconfigure - check reconfiguration result and that fields are reordered to the previous order
-		TypeSerializerSchemaCompatibility<TestUserClass> compatResult =
-			mockPreviousConfigSnapshot.resolveSchemaCompatibility(pojoSerializer);
-		assertTrue(compatResult.isCompatibleAsIs());
-		int i = 0;
-		for (Field field : mockOriginalFieldOrder) {
-			assertEquals(field, pojoSerializer.getFields()[i]);
-			i++;
-		}
-	}
-
-	private static void verifyPojoSerializerConfigSnapshotWithSerializerSerializationFailure(
-			PojoSerializer.PojoSerializerConfigSnapshot<?> original,
-			PojoSerializer.PojoSerializerConfigSnapshot<?> deserializedConfig) {
-
-		LinkedHashMap<String, Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> originalFieldSerializersAndConfs =
-				original.getFieldToSerializerConfigSnapshot();
-		for (Map.Entry<String, Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> entry
-				: deserializedConfig.getFieldToSerializerConfigSnapshot().entrySet()) {
-
-			Assert.assertTrue(entry.getValue().f0 instanceof UnloadableDummyTypeSerializer);
-
-			if (entry.getValue().f1 instanceof PojoSerializer.PojoSerializerConfigSnapshot) {
-				verifyPojoSerializerConfigSnapshotWithSerializerSerializationFailure(
-					(PojoSerializer.PojoSerializerConfigSnapshot<?>) originalFieldSerializersAndConfs.get(entry.getKey()).f1,
-					(PojoSerializer.PojoSerializerConfigSnapshot<?>) entry.getValue().f1);
-			} else {
-				Assert.assertEquals(originalFieldSerializersAndConfs.get(entry.getKey()).f1, entry.getValue().f1);
-			}
-		}
-
-		LinkedHashMap<Class<?>, Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> originalRegistrations =
-				original.getRegisteredSubclassesToSerializerConfigSnapshots();
-
-		for (Map.Entry<Class<?>, Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> entry
-				: deserializedConfig.getRegisteredSubclassesToSerializerConfigSnapshots().entrySet()) {
-
-			Assert.assertTrue(entry.getValue().f0 instanceof UnloadableDummyTypeSerializer);
-
-			if (entry.getValue().f1 instanceof PojoSerializer.PojoSerializerConfigSnapshot) {
-				verifyPojoSerializerConfigSnapshotWithSerializerSerializationFailure(
-					(PojoSerializer.PojoSerializerConfigSnapshot<?>) originalRegistrations.get(entry.getKey()).f1,
-					(PojoSerializer.PojoSerializerConfigSnapshot<?>) entry.getValue().f1);
-			} else {
-				Assert.assertEquals(originalRegistrations.get(entry.getKey()).f1, entry.getValue().f1);
-			}
-		}
+		assertTrue(compatResult.isCompatibleWithReconfiguredSerializer());
+		assertTrue(compatResult.getReconfiguredSerializer() instanceof PojoSerializer);
+
+		PojoSerializer<TestUserClass> reconfiguredPojoSerializer = (PojoSerializer<TestUserClass>) compatResult.getReconfiguredSerializer();
+		assertEquals(2, reconfiguredPojoSerializer.getSubclassSerializerCache().size());
+		assertTrue(reconfiguredPojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class));
+		assertTrue(reconfiguredPojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassB.class));
+		assertEquals(2, reconfiguredPojoSerializer.getRegisteredClasses().size());
+		assertTrue(reconfiguredPojoSerializer.getRegisteredClasses().containsKey(SubTestUserClassA.class));
+		assertTrue(reconfiguredPojoSerializer.getRegisteredClasses().containsKey(SubTestUserClassB.class));
 	}
 }
diff --git a/flink-tests/src/test/java/org/apache/flink/test/typeserializerupgrade/PojoSerializerUpgradeTest.java b/flink-tests/src/test/java/org/apache/flink/test/typeserializerupgrade/PojoSerializerUpgradeTest.java
index 3f6f09b..a02741e 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/typeserializerupgrade/PojoSerializerUpgradeTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/typeserializerupgrade/PojoSerializerUpgradeTest.java
@@ -23,8 +23,6 @@ import org.apache.flink.api.common.functions.ReduceFunction;
 import org.apache.flink.api.common.functions.RichMapFunction;
 import org.apache.flink.api.common.state.ListState;
 import org.apache.flink.api.common.state.ListStateDescriptor;
-import org.apache.flink.api.common.state.MapState;
-import org.apache.flink.api.common.state.MapStateDescriptor;
 import org.apache.flink.api.common.state.ReducingState;
 import org.apache.flink.api.common.state.ReducingStateDescriptor;
 import org.apache.flink.api.common.state.ValueState;
@@ -225,74 +223,38 @@ public class PojoSerializerUpgradeTest extends TestLogger {
 	}
 
 	/**
-	 * Adding fields to a POJO as keyed state should require a state migration.
+	 * Adding fields to a POJO as keyed state should succeed.
 	 */
 	@Test
 	public void testAdditionalFieldWithKeyedState() throws Exception {
-		try {
-			testPojoSerializerUpgrade(SOURCE_A, SOURCE_D, true, true);
-			fail("Expected a state migration exception.");
-		} catch (Exception e) {
-			if (CommonTestUtils.containsCause(e, StateMigrationException.class)) {
-				// StateMigrationException expected
-			} else {
-				throw e;
-			}
-		}
+		testPojoSerializerUpgrade(SOURCE_A, SOURCE_D, true, true);
 	}
 
 	/**
-	 * Adding fields to a POJO as operator state should require a state migration.
+	 * Adding fields to a POJO as operator state should succeed.
 	 */
 	@Test
 	public void testAdditionalFieldWithOperatorState() throws Exception {
-		try {
-			testPojoSerializerUpgrade(SOURCE_A, SOURCE_D, true, false);
-			fail("Expected a state migration exception.");
-		} catch (Exception e) {
-			if (CommonTestUtils.containsCause(e, StateMigrationException.class)) {
-				// StateMigrationException expected
-			} else {
-				throw e;
-			}
-		}
+		testPojoSerializerUpgrade(SOURCE_A, SOURCE_D, true, false);
 	}
 
 	/**
-	 * Removing fields from a POJO as keyed state should require a state migration.
+	 * Removing fields from a POJO as keyed state should succeed.
 	 */
 	@Test
 	public void testMissingFieldWithKeyedState() throws Exception {
-		try {
-			testPojoSerializerUpgrade(SOURCE_A, SOURCE_E, false, true);
-			fail("Expected a state migration exception.");
-		} catch (Exception e) {
-			if (CommonTestUtils.containsCause(e, StateMigrationException.class)) {
-				// StateMigrationException expected
-			} else {
-				throw e;
-			}
-		}
+		testPojoSerializerUpgrade(SOURCE_A, SOURCE_E, false, true);
 	}
 
 	/**
-	 * Removing fields from a POJO as operator state should require a state migration.
+	 * Removing fields from a POJO as operator state should succeed.
 	 */
 	@Test
 	public void testMissingFieldWithOperatorState() throws Exception {
-		try {
-			testPojoSerializerUpgrade(SOURCE_A, SOURCE_E, false, false);
-			fail("Expected a state migration exception.");
-		} catch (Exception e) {
-			if (CommonTestUtils.containsCause(e, StateMigrationException.class)) {
-				// StateMigrationException expected
-			} else {
-				throw e;
-			}
-		}
+		testPojoSerializerUpgrade(SOURCE_A, SOURCE_E, false, false);
 	}
 
-	public void testPojoSerializerUpgrade(String classSourceA, String classSourceB, boolean hasBField, boolean isKeyedState) throws Exception {
+	private void testPojoSerializerUpgrade(String classSourceA, String classSourceB, boolean hasBField, boolean isKeyedState) throws Exception {
 		final Configuration taskConfiguration = new Configuration();
 		final ExecutionConfig executionConfig = new ExecutionConfig();
 		final KeySelector<Long, Long> keySelector = new IdentityKeySelector<>();
@@ -424,7 +386,6 @@ public class PojoSerializerUpgradeTest extends TestLogger {
 
 		// keyed states
 		private transient ValueState<Object> keyedValueState;
-		private transient MapState<Object, Object> keyedMapState;
 		private transient ListState<Object> keyedListState;
 		private transient ReducingState<Object> keyedReducingState;
 
@@ -436,7 +397,7 @@ public class PojoSerializerUpgradeTest extends TestLogger {
 		private transient Field fieldA;
 		private transient Field fieldB;
 
-		public StatefulMapper(boolean keyed, boolean verify, boolean hasBField) {
+		StatefulMapper(boolean keyed, boolean verify, boolean hasBField) {
 			this.keyed = keyed;
 			this.verify = verify;
 			this.hasBField = hasBField;
@@ -456,9 +417,6 @@ public class PojoSerializerUpgradeTest extends TestLogger {
 				if (keyed) {
 					assertEquals(pojo, keyedValueState.value());
 
-					assertTrue(keyedMapState.contains(pojo));
-					assertEquals(pojo, keyedMapState.get(pojo));
-
 					Iterator<Object> listIterator = keyedListState.get().iterator();
 
 					boolean elementFound = false;
@@ -488,7 +446,6 @@ public class PojoSerializerUpgradeTest extends TestLogger {
 			} else {
 				if (keyed) {
 					keyedValueState.update(pojo);
-					keyedMapState.put(pojo, pojo);
 					keyedListState.add(pojo);
 					keyedReducingState.add(pojo);
 				} else {
@@ -505,6 +462,7 @@ public class PojoSerializerUpgradeTest extends TestLogger {
 
 		}
 
+		@SuppressWarnings("unchecked")
 		@Override
 		public void initializeState(FunctionInitializationContext context) throws Exception {
 			pojoClass = getRuntimeContext().getUserCodeClassLoader().loadClass(POJO_NAME);
@@ -520,8 +478,6 @@ public class PojoSerializerUpgradeTest extends TestLogger {
 			if (keyed) {
 				keyedValueState = context.getKeyedStateStore().getState(
 					new ValueStateDescriptor<>("keyedValueState", (Class<Object>) pojoClass));
-				keyedMapState = context.getKeyedStateStore().getMapState(
-					new MapStateDescriptor<>("keyedMapState", (Class<Object>) pojoClass, (Class<Object>) pojoClass));
 				keyedListState = context.getKeyedStateStore().getListState(
 					new ListStateDescriptor<>("keyedListState", (Class<Object>) pojoClass));
 


[flink] 08/17: [FLINK-11485] [core] Move OptionalMap to org.apache.flink.util

Posted by tz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tzulitai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit a3822f220fc724934b55b9a7994e86afa926a41f
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
AuthorDate: Thu Jan 31 17:29:26 2019 +0800

    [FLINK-11485] [core] Move OptionalMap to org.apache.flink.util
    
    This utility class may commonly be used by the KryoSerializer and the
    PojoSerializer. This commit moves the utility to a more suitable package
    and adjusts usage access.
---
 .../runtime/kryo/KryoSerializerSnapshot.java       |  6 ++--
 .../runtime/kryo/KryoSerializerSnapshotData.java   |  3 +-
 .../runtime/kryo => util}/OptionalMap.java         | 34 ++++++++++++----------
 .../runtime/kryo => util}/OptionalMapTest.java     |  4 +--
 4 files changed, 26 insertions(+), 21 deletions(-)

diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshot.java
index 37a0104..77242e6 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshot.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshot.java
@@ -23,9 +23,9 @@ import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
 import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
 import org.apache.flink.api.java.typeutils.runtime.KryoRegistration;
-import org.apache.flink.api.java.typeutils.runtime.kryo.OptionalMap.MergeResult;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.util.OptionalMap.MergeResult;
 
 import com.esotericsoftware.kryo.Serializer;
 import org.slf4j.Logger;
@@ -36,8 +36,8 @@ import java.util.LinkedHashMap;
 import java.util.function.Function;
 
 import static org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializerSnapshotData.createFrom;
-import static org.apache.flink.api.java.typeutils.runtime.kryo.OptionalMap.mergeRightIntoLeft;
-import static org.apache.flink.api.java.typeutils.runtime.kryo.OptionalMap.optionalMapOf;
+import static org.apache.flink.util.OptionalMap.mergeRightIntoLeft;
+import static org.apache.flink.util.OptionalMap.optionalMapOf;
 
 /**
  * {@link TypeSerializerSnapshot} for {@link KryoSerializer}.
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshotData.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshotData.java
index 9587101..40b12a4 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshotData.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshotData.java
@@ -25,6 +25,7 @@ import org.apache.flink.api.java.typeutils.runtime.KryoRegistration;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 import org.apache.flink.util.InstantiationUtil;
+import org.apache.flink.util.OptionalMap;
 
 import com.esotericsoftware.kryo.Serializer;
 import org.slf4j.Logger;
@@ -36,7 +37,7 @@ import java.util.LinkedHashMap;
 import java.util.Map.Entry;
 import java.util.function.Function;
 
-import static org.apache.flink.api.java.typeutils.runtime.kryo.OptionalMap.optionalMapOf;
+import static org.apache.flink.util.OptionalMap.optionalMapOf;
 import static org.apache.flink.util.Preconditions.checkNotNull;
 
 final class KryoSerializerSnapshotData<T> {
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/OptionalMap.java b/flink-core/src/main/java/org/apache/flink/util/OptionalMap.java
similarity index 89%
rename from flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/OptionalMap.java
rename to flink-core/src/main/java/org/apache/flink/util/OptionalMap.java
index afabb4b..4552102 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/OptionalMap.java
+++ b/flink-core/src/main/java/org/apache/flink/util/OptionalMap.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.flink.api.java.typeutils.runtime.kryo;
+package org.apache.flink.util;
 
 import org.apache.flink.annotation.VisibleForTesting;
 
@@ -36,7 +36,7 @@ import static org.apache.flink.util.Preconditions.checkNotNull;
  * An OptionalMap is an order preserving map (like {@link LinkedHashMap}) where keys have a unique string name, but are
  * optionally present, and the values are optional.
  */
-final class OptionalMap<K, V> {
+public final class OptionalMap<K, V> {
 
 
 	// --------------------------------------------------------------------------------------------------------
@@ -55,7 +55,7 @@ final class OptionalMap<K, V> {
 	 * @param <V>           value type
 	 * @return an {@code OptionalMap} with optional named keys, and optional values.
 	 */
-	static <K, V> OptionalMap<K, V> optionalMapOf(LinkedHashMap<K, V> sourceMap, Function<K, String> keyNameGetter) {
+	public static <K, V> OptionalMap<K, V> optionalMapOf(LinkedHashMap<K, V> sourceMap, Function<K, String> keyNameGetter) {
 
 		LinkedHashMap<String, KeyValue<K, V>> underlyingMap = new LinkedHashMap<>(sourceMap.size());
 
@@ -70,7 +70,7 @@ final class OptionalMap<K, V> {
 	/**
 	 * Tries to merges the keys and the values of @right into @left.
 	 */
-	static <K, V> MergeResult<K, V> mergeRightIntoLeft(OptionalMap<K, V> left, OptionalMap<K, V> right) {
+	public static <K, V> MergeResult<K, V> mergeRightIntoLeft(OptionalMap<K, V> left, OptionalMap<K, V> right) {
 		OptionalMap<K, V> merged = new OptionalMap<>(left);
 		merged.putAll(right);
 
@@ -83,7 +83,7 @@ final class OptionalMap<K, V> {
 
 	private final LinkedHashMap<String, KeyValue<K, V>> underlyingMap;
 
-	OptionalMap() {
+	public OptionalMap() {
 		this(new LinkedHashMap<>());
 	}
 
@@ -100,11 +100,11 @@ final class OptionalMap<K, V> {
 	// API
 	// --------------------------------------------------------------------------------------------------------
 
-	int size() {
+	public int size() {
 		return underlyingMap.size();
 	}
 
-	void put(String keyName, @Nullable K key, @Nullable V value) {
+	public void put(String keyName, @Nullable K key, @Nullable V value) {
 		checkNotNull(keyName);
 
 		underlyingMap.compute(keyName, (unused, kv) ->
@@ -121,7 +121,7 @@ final class OptionalMap<K, V> {
 	/**
 	 * Returns the key names of any keys or values that are absent.
 	 */
-	Set<String> absentKeysOrValues() {
+	public Set<String> absentKeysOrValues() {
 		return underlyingMap.entrySet()
 			.stream()
 			.filter(OptionalMap::keyOrValueIsAbsent)
@@ -134,7 +134,7 @@ final class OptionalMap<K, V> {
 	 * a map with these key and values, stripped from their Optional wrappers.
 	 * NOTE: please note that if any of the key or values are absent this method would throw an {@link IllegalStateException}.
 	 */
-	LinkedHashMap<K, V> unwrapOptionals() {
+	public LinkedHashMap<K, V> unwrapOptionals() {
 		final LinkedHashMap<K, V> unwrapped = new LinkedHashMap<>(underlyingMap.size());
 
 		for (Entry<String, KeyValue<K, V>> entry : underlyingMap.entrySet()) {
@@ -155,7 +155,7 @@ final class OptionalMap<K, V> {
 	/**
 	 * Returns the key names added to this map.
 	 */
-	Set<String> keyNames() {
+	public Set<String> keyNames() {
 		return underlyingMap.keySet();
 	}
 
@@ -215,7 +215,11 @@ final class OptionalMap<K, V> {
 	// Merge
 	// --------------------------------------------------------------------------------------------------------
 
-	static final class MergeResult<K, V> {
+	/**
+	 * The result of merging two {@link LinkedOptionalMap}s using
+	 * {@link #mergeRightIntoLeft(LinkedOptionalMap, LinkedOptionalMap)}.
+	 */
+	public static final class MergeResult<K, V> {
 		private final OptionalMap<K, V> merged;
 		private final Set<String> missingKeys;
 		private final boolean isOrderedSubset;
@@ -226,22 +230,22 @@ final class OptionalMap<K, V> {
 			this.isOrderedSubset = isOrderedSubset;
 		}
 
-		boolean hasMissingKeys() {
+		public boolean hasMissingKeys() {
 			return !missingKeys.isEmpty();
 		}
 
-		Set<String> missingKeys() {
+		public Set<String> missingKeys() {
 			return missingKeys;
 		}
 
-		LinkedHashMap<K, V> getMerged() {
+		public LinkedHashMap<K, V> getMerged() {
 			return merged.unwrapOptionals();
 		}
 
 		/**
 		 * Returns {@code true} if keyNames present at @left, appearing in prefix order at @right.
 		 */
-		boolean isOrderedSubset() {
+		public boolean isOrderedSubset() {
 			return isOrderedSubset;
 		}
 	}
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/OptionalMapTest.java b/flink-core/src/test/java/org/apache/flink/util/OptionalMapTest.java
similarity index 97%
rename from flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/OptionalMapTest.java
rename to flink-core/src/test/java/org/apache/flink/util/OptionalMapTest.java
index ff7f31f..050dba8 100644
--- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/OptionalMapTest.java
+++ b/flink-core/src/test/java/org/apache/flink/util/OptionalMapTest.java
@@ -16,9 +16,9 @@
  * limitations under the License.
  */
 
-package org.apache.flink.api.java.typeutils.runtime.kryo;
+package org.apache.flink.util;
 
-import org.apache.flink.api.java.typeutils.runtime.kryo.OptionalMap.MergeResult;
+import org.apache.flink.util.OptionalMap.MergeResult;
 
 import org.junit.Test;
 


[flink] 15/17: [FLINK-10897] [docs] Document POJO schema evolution in /dev/stream/state/schema_evolution.md

Posted by tz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tzulitai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 9c025932a0a28539168228c53540501443d593e8
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
AuthorDate: Wed Feb 20 13:42:06 2019 +0800

    [FLINK-10897] [docs] Document POJO schema evolution in /dev/stream/state/schema_evolution.md
---
 docs/dev/stream/state/schema_evolution.md | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/docs/dev/stream/state/schema_evolution.md b/docs/dev/stream/state/schema_evolution.md
index fd0a73c..aa8d41e 100644
--- a/docs/dev/stream/state/schema_evolution.md
+++ b/docs/dev/stream/state/schema_evolution.md
@@ -78,11 +78,25 @@ Further details about the migration process is out of the scope of this document
 
 ## Supported data types for schema evolution
 
-Currently, schema evolution is supported only for Avro. Therefore, if you care about schema evolution for
-state, it is currently recommended to always use Avro for state data types.
+Currently, schema evolution is supported only for POJO and Avro types. Therefore, if you care about schema evolution for
+state, it is currently recommended to always use either Pojo or Avro for state data types.
 
-There are plans to extend the support for more composite types, such as POJOs; for more details,
-please refer to [FLINK-10897](https://issues.apache.org/jira/browse/FLINK-10897).
+There are plans to extend the support for more composite types; for more details,
+please refer to [FLINK-10896](https://issues.apache.org/jira/browse/FLINK-10896).
+
+### POJO types
+
+Flink supports evolving schema of [POJO types]({{ site.baseurl }}/dev/types_serialization.html#rules-for-pojo-types), 
+based on the following set of rules:
+
+ 1. Fields can be removed. Once removed, the previous value for the removed field will be dropped in future checkpoints and savepoints.
+ 2. New fields can be added. The new field will be initialized to the default value for its type, as
+    [defined by Java](https://docs.oracle.com/javase/tutorial/java/nutsandbolts/datatypes.html).
+ 3. Declared fields types cannot change.
+ 4. Class name of the POJO type cannot change, including the namespace of the class.
+
+Note that the schema of POJO type state can only be evolved when restoring from a previous savepoint with Flink versions
+newer than 1.8.0. When restoring with Flink versions older than 1.8.0, the schema cannot be changed.
 
 ### Avro types
 


[flink] 12/17: [FLINK-11485] [core] Migrate PojoSerializer to use new serialization compatibility abstractions

Posted by tz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tzulitai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 1850193b956b899aefd3ea84fe726e10b9359b55
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
AuthorDate: Tue Feb 19 16:51:41 2019 +0800

    [FLINK-11485] [core] Migrate PojoSerializer to use new serialization compatibility abstractions
---
 .../api/java/typeutils/runtime/PojoSerializer.java | 275 ++++++---------------
 .../TypeSerializerSnapshotMigrationTestBase.java   |   2 +-
 .../PojoSerializerSnapshotMigrationTest.java       | 127 ++++++++++
 ...1.6-pojo-new-and-removed-fields-serializer-data | Bin 0 -> 320 bytes
 ...pojo-new-and-removed-fields-serializer-snapshot | Bin 0 -> 4696 bytes
 .../test/resources/flink-1.6-pojo-serializer-data  | Bin 0 -> 230 bytes
 .../resources/flink-1.6-pojo-serializer-snapshot   | Bin 0 -> 4133 bytes
 ...1.7-pojo-new-and-removed-fields-serializer-data | Bin 0 -> 320 bytes
 ...pojo-new-and-removed-fields-serializer-snapshot | Bin 0 -> 4724 bytes
 .../test/resources/flink-1.7-pojo-serializer-data  | Bin 0 -> 230 bytes
 .../resources/flink-1.7-pojo-serializer-snapshot   | Bin 0 -> 4148 bytes
 11 files changed, 197 insertions(+), 207 deletions(-)

diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java
index 8abe0d9..fbaedc0 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java
@@ -20,15 +20,14 @@ package org.apache.flink.api.java.typeutils.runtime;
 
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.typeutils.CompatibilityResult;
-import org.apache.flink.api.common.typeutils.CompatibilityUtil;
 import org.apache.flink.api.common.typeutils.GenericTypeSerializerConfigSnapshot;
+import org.apache.flink.api.common.typeutils.LegacySerializerSnapshotTransformer;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
+import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
 import org.apache.flink.api.common.typeutils.TypeSerializerSerializationUtil;
 import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
 import org.apache.flink.api.common.typeutils.TypeSerializerSnapshotSerializationUtil;
-import org.apache.flink.api.common.typeutils.UnloadableDummyTypeSerializer;
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.api.java.typeutils.TypeExtractor;
 import org.apache.flink.core.memory.ByteArrayInputStreamWithPos;
@@ -44,6 +43,7 @@ import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
 import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
@@ -77,27 +77,20 @@ public final class PojoSerializer<T> extends TypeSerializer<T> {
 	 * <p>The fields are kept as a separate transient member, with their serialization
 	 * handled with the {@link #readObject(ObjectInputStream)} and {@link #writeObject(ObjectOutputStream)}
 	 * methods.
-	 *
-	 * <p>These may be reconfigured in {@link #ensureCompatibility(TypeSerializerConfigSnapshot)}.
 	 */
 	private transient Field[] fields;
-	private TypeSerializer<Object>[] fieldSerializers;
+	private final TypeSerializer<Object>[] fieldSerializers;
 	private final int numFields;
 
 	/**
 	 * Registered subclasses and their serializers.
 	 * Each subclass to their registered class tag is maintained as a separate map ordered by the class tag.
-	 *
-	 * <p>These may be reconfigured in {@link #ensureCompatibility(TypeSerializerConfigSnapshot)}.
 	 */
-	private LinkedHashMap<Class<?>, Integer> registeredClasses;
-	private TypeSerializer<?>[] registeredSerializers;
+	private final LinkedHashMap<Class<?>, Integer> registeredClasses;
+	private final TypeSerializer<?>[] registeredSerializers;
 
 	/**
 	 * Cache of non-registered subclasses to their serializers, created on-the-fly.
-	 *
-	 * <p>This cache is persisted and will be repopulated with reconfigured serializers
-	 * in {@link #ensureCompatibility(TypeSerializerConfigSnapshot)}.
 	 */
 	private transient HashMap<Class<?>, TypeSerializer<?>> subclassSerializerCache;
 
@@ -105,10 +98,6 @@ public final class PojoSerializer<T> extends TypeSerializer<T> {
 
 	/**
 	 * Configuration of the current execution.
-	 *
-	 * <p>Nested serializers created using this will have the most up-to-date configuration,
-	 * and can be resolved for backwards compatibility with previous configuration
-	 * snapshots in {@link #ensureCompatibility(TypeSerializerConfigSnapshot)}.
 	 */
 	private final ExecutionConfig executionConfig;
 
@@ -586,8 +575,8 @@ public final class PojoSerializer<T> extends TypeSerializer<T> {
 	// --------------------------------------------------------------------------------------------
 
 	@Override
-	public PojoSerializerConfigSnapshot<T> snapshotConfiguration() {
-		return buildConfigSnapshot(
+	public PojoSerializerSnapshot<T> snapshotConfiguration() {
+		return buildSnapshot(
 				clazz,
 				registeredClasses,
 				registeredSerializers,
@@ -596,163 +585,11 @@ public final class PojoSerializer<T> extends TypeSerializer<T> {
 				subclassSerializerCache);
 	}
 
-	@SuppressWarnings("unchecked")
-	@Override
-	public CompatibilityResult<T> ensureCompatibility(TypeSerializerConfigSnapshot<?> configSnapshot) {
-		if (configSnapshot instanceof PojoSerializerConfigSnapshot) {
-			final PojoSerializerConfigSnapshot<T> config = (PojoSerializerConfigSnapshot<T>) configSnapshot;
-
-			boolean requiresMigration = false;
-
-			if (clazz.equals(config.getTypeClass())) {
-				if (this.numFields == config.getFieldToSerializerConfigSnapshot().size()) {
-
-					CompatibilityResult<?> compatResult;
-
-					// ----------- check field order and compatibility of field serializers -----------
-
-					// reordered fields and their serializers;
-					// this won't be applied to this serializer until all compatibility checks have been completed
-					final Field[] reorderedFields = new Field[this.numFields];
-					final TypeSerializer<Object>[] reorderedFieldSerializers =
-						(TypeSerializer<Object>[]) new TypeSerializer<?>[this.numFields];
-
-					int i = 0;
-					for (Map.Entry<String, Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> fieldToConfigSnapshotEntry
-							: config.getFieldToSerializerConfigSnapshot().entrySet()) {
-
-						int fieldIndex = findField(fieldToConfigSnapshotEntry.getKey());
-						if (fieldIndex != -1) {
-							reorderedFields[i] = fields[fieldIndex];
-
-							compatResult = CompatibilityUtil.resolveCompatibilityResult(
-								fieldToConfigSnapshotEntry.getValue().f0,
-								UnloadableDummyTypeSerializer.class,
-								fieldToConfigSnapshotEntry.getValue().f1,
-								fieldSerializers[fieldIndex]);
-
-							if (compatResult.isRequiresMigration()) {
-								requiresMigration = true;
-
-								if (compatResult.getConvertDeserializer() != null) {
-									reorderedFieldSerializers[i] = (TypeSerializer<Object>) compatResult.getConvertDeserializer();
-								} else {
-									return CompatibilityResult.requiresMigration();
-								}
-							} else {
-								reorderedFieldSerializers[i] = fieldSerializers[fieldIndex];
-							}
-						} else {
-							return CompatibilityResult.requiresMigration();
-						}
-
-						i++;
-					}
-
-					// ---- check subclass registration order and compatibility of registered serializers ----
-
-					// reordered subclass registrations and their serializers;
-					// this won't be applied to this serializer until all compatibility checks have been completed
-					final LinkedHashMap<Class<?>, Integer> reorderedRegisteredSubclassesToClasstags;
-					final TypeSerializer<?>[] reorderedRegisteredSubclassSerializers;
-
-					final LinkedHashMap<Class<?>, Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> previousRegistrations =
-						config.getRegisteredSubclassesToSerializerConfigSnapshots();
-
-					// the reconfigured list of registered subclasses will be the previous registered
-					// subclasses in the original order with new subclasses appended at the end
-					LinkedHashSet<Class<?>> reorderedRegisteredSubclasses = new LinkedHashSet<>();
-					reorderedRegisteredSubclasses.addAll(previousRegistrations.keySet());
-					reorderedRegisteredSubclasses.addAll(
-						getRegisteredSubclassesFromExecutionConfig(clazz, executionConfig));
-
-					// re-establish the registered class tags and serializers
-					reorderedRegisteredSubclassesToClasstags = createRegisteredSubclassTags(reorderedRegisteredSubclasses);
-					reorderedRegisteredSubclassSerializers = createRegisteredSubclassSerializers(
-						reorderedRegisteredSubclasses, executionConfig);
-
-					i = 0;
-					for (Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>> previousRegisteredSerializerConfig : previousRegistrations.values()) {
-						// check compatibility of subclass serializer
-						compatResult = CompatibilityUtil.resolveCompatibilityResult(
-								previousRegisteredSerializerConfig.f0,
-								UnloadableDummyTypeSerializer.class,
-								previousRegisteredSerializerConfig.f1,
-								reorderedRegisteredSubclassSerializers[i]);
-
-						if (compatResult.isRequiresMigration()) {
-							requiresMigration = true;
-
-							if (compatResult.getConvertDeserializer() == null) {
-								return CompatibilityResult.requiresMigration();
-							}
-						}
-
-						i++;
-					}
-
-					// ------------------ ensure compatibility of non-registered subclass serializers ------------------
-
-					// the rebuilt cache for non-registered subclass serializers;
-					// this won't be applied to this serializer until all compatibility checks have been completed
-					HashMap<Class<?>, TypeSerializer<?>> rebuiltCache = new HashMap<>();
-
-					for (Map.Entry<Class<?>, Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> previousCachedEntry
-							: config.getNonRegisteredSubclassesToSerializerConfigSnapshots().entrySet()) {
-
-						TypeSerializer<?> cachedSerializer = createSubclassSerializer(previousCachedEntry.getKey());
-
-						// check compatibility of cached subclass serializer
-						compatResult = CompatibilityUtil.resolveCompatibilityResult(
-								previousCachedEntry.getValue().f0,
-								UnloadableDummyTypeSerializer.class,
-								previousCachedEntry.getValue().f1,
-								cachedSerializer);
-
-						if (compatResult.isRequiresMigration()) {
-							requiresMigration = true;
-
-							if (compatResult.getConvertDeserializer() != null) {
-								rebuiltCache.put(previousCachedEntry.getKey(), cachedSerializer);
-							} else {
-								return CompatibilityResult.requiresMigration();
-							}
-						} else {
-							rebuiltCache.put(previousCachedEntry.getKey(), cachedSerializer);
-						}
-					}
-
-					// completed compatibility checks; up to this point, we can just reconfigure
-					// the serializer so that it is compatible and migration is not required
-
-					if (!requiresMigration) {
-						this.fields = reorderedFields;
-						this.fieldSerializers = reorderedFieldSerializers;
-
-						this.registeredClasses = reorderedRegisteredSubclassesToClasstags;
-						this.registeredSerializers = reorderedRegisteredSubclassSerializers;
-
-						this.subclassSerializerCache = rebuiltCache;
-
-						return CompatibilityResult.compatible();
-					} else {
-						return CompatibilityResult.requiresMigration(
-							new PojoSerializer<>(
-								clazz,
-								reorderedFields,
-								reorderedFieldSerializers,
-								reorderedRegisteredSubclassesToClasstags,
-								reorderedRegisteredSubclassSerializers,
-								rebuiltCache,
-								null));
-					}
-				}
-			}
-		}
-
-		return CompatibilityResult.requiresMigration();
-	}
-
+	/**
+	 * @deprecated This snapshot class is no longer being used.
+	 *             It has been fully replaced by {@link PojoSerializerSnapshot}.
+	 */
+	@Deprecated
 	public static final class PojoSerializerConfigSnapshot<T> extends GenericTypeSerializerConfigSnapshot<T> {
 
 		private static final int VERSION = 1;
@@ -821,6 +658,54 @@ public final class PojoSerializer<T> extends TypeSerializer<T> {
 			this.ignoreTypeSerializerSerialization = ignoreTypeSerializerSerialization;
 		}
 
+		/**
+		 * This legacy snapshot delegates compatibility checks to the {@link PojoSerializerSnapshot}.
+		 */
+		@Override
+		public TypeSerializerSchemaCompatibility<T> resolveSchemaCompatibility(TypeSerializer<T> newSerializer) {
+			LinkedHashMap<String, TypeSerializerSnapshot<?>> legacyFieldSerializerSnapshots =
+				preprocessLegacySerializerSnapshotTuples(fieldToSerializerConfigSnapshot);
+
+			int numFields = legacyFieldSerializerSnapshots.size();
+			ArrayList<Field> fields = new ArrayList<>(numFields);
+			ArrayList<TypeSerializerSnapshot<?>> fieldSerializerSnapshots = new ArrayList<>(numFields);
+			legacyFieldSerializerSnapshots.forEach((fieldName, fieldSerializerSnapshot) -> {
+				fields.add(PojoFieldUtils.getField(fieldName, getTypeClass()));
+				fieldSerializerSnapshots.add(fieldSerializerSnapshot);
+			});
+
+			PojoSerializerSnapshot<T> newSnapshot = new PojoSerializerSnapshot<>(
+				getTypeClass(),
+				fields.toArray(new Field[numFields]),
+				fieldSerializerSnapshots.toArray(new TypeSerializerSnapshot[numFields]),
+				preprocessLegacySerializerSnapshotTuples(registeredSubclassesToSerializerConfigSnapshots),
+				preprocessLegacySerializerSnapshotTuples(nonRegisteredSubclassesToSerializerConfigSnapshots));
+
+			return newSnapshot.resolveSchemaCompatibility(newSerializer);
+		}
+
+		@SuppressWarnings("unchecked")
+		private static <K> LinkedHashMap<K, TypeSerializerSnapshot<?>> preprocessLegacySerializerSnapshotTuples(Map<K, Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> originalMap) {
+			LinkedHashMap<K, TypeSerializerSnapshot<?>> converted = new LinkedHashMap<>(originalMap.size());
+
+			originalMap.forEach((key, serializerSnapshotTuple) -> {
+				TypeSerializer<?> serializer = serializerSnapshotTuple.f0;
+				TypeSerializerSnapshot<?> snapshot = serializerSnapshotTuple.f1;
+
+				if (snapshot instanceof TypeSerializerConfigSnapshot) {
+					((TypeSerializerConfigSnapshot) snapshot).setPriorSerializer(serializer);
+				}
+
+				if (serializer instanceof LegacySerializerSnapshotTransformer) {
+					snapshot = ((LegacySerializerSnapshotTransformer) serializer).transformLegacySerializerSnapshot(snapshot);
+				}
+
+				converted.put(key, snapshot);
+			});
+
+			return converted;
+		}
+
 		@Override
 		public void write(DataOutputView out) throws IOException {
 			super.write(out);
@@ -1219,7 +1104,7 @@ public final class PojoSerializer<T> extends TypeSerializer<T> {
 	/**
 	 * Build and return a snapshot of the serializer's parameters and currently cached serializers.
 	 */
-	private static <T> PojoSerializerConfigSnapshot<T> buildConfigSnapshot(
+	private static <T> PojoSerializerSnapshot<T> buildSnapshot(
 			Class<T> pojoType,
 			LinkedHashMap<Class<?>, Integer> registeredSubclassesToTags,
 			TypeSerializer<?>[] registeredSubclassSerializers,
@@ -1227,39 +1112,17 @@ public final class PojoSerializer<T> extends TypeSerializer<T> {
 			TypeSerializer<?>[] fieldSerializers,
 			HashMap<Class<?>, TypeSerializer<?>> nonRegisteredSubclassSerializerCache) {
 
-		final LinkedHashMap<String, Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> fieldToSerializerConfigSnapshots =
-			new LinkedHashMap<>(fields.length);
-
-		for (int i = 0; i < fields.length; i++) {
-			fieldToSerializerConfigSnapshots.put(
-				fields[i].getName(),
-				new Tuple2<>(fieldSerializers[i], fieldSerializers[i].snapshotConfiguration()));
-		}
-
-		final LinkedHashMap<Class<?>, Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> registeredSubclassesToSerializerConfigSnapshots =
-				new LinkedHashMap<>(registeredSubclassesToTags.size());
+		final LinkedHashMap<Class<?>, TypeSerializer<?>> subclassRegistry = new LinkedHashMap<>(registeredSubclassesToTags.size());
 
 		for (Map.Entry<Class<?>, Integer> entry : registeredSubclassesToTags.entrySet()) {
-			registeredSubclassesToSerializerConfigSnapshots.put(
-					entry.getKey(),
-					new Tuple2<>(
-						registeredSubclassSerializers[entry.getValue()],
-						registeredSubclassSerializers[entry.getValue()].snapshotConfiguration()));
+			subclassRegistry.put(entry.getKey(), registeredSubclassSerializers[entry.getValue()]);
 		}
 
-		final HashMap<Class<?>, Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> nonRegisteredSubclassesToSerializerConfigSnapshots =
-				new LinkedHashMap<>(nonRegisteredSubclassSerializerCache.size());
-
-		for (Map.Entry<Class<?>, TypeSerializer<?>> entry : nonRegisteredSubclassSerializerCache.entrySet()) {
-			nonRegisteredSubclassesToSerializerConfigSnapshots.put(
-				entry.getKey(),
-				new Tuple2<>(entry.getValue(), entry.getValue().snapshotConfiguration()));
-		}
-
-		return new PojoSerializerConfigSnapshot<>(
+		return new PojoSerializerSnapshot<>(
 				pojoType,
-				fieldToSerializerConfigSnapshots,
-				registeredSubclassesToSerializerConfigSnapshots,
-				nonRegisteredSubclassesToSerializerConfigSnapshots);
+				fields,
+				fieldSerializers,
+				subclassRegistry,
+				nonRegisteredSubclassSerializerCache);
 	}
 }
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerSnapshotMigrationTestBase.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerSnapshotMigrationTestBase.java
index 539e108..94081d4 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerSnapshotMigrationTestBase.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerSnapshotMigrationTestBase.java
@@ -529,7 +529,7 @@ public abstract class TypeSerializerSnapshotMigrationTestBase<ElementT> extends
 	// Utilities
 	// --------------------------------------------------------------------------------------------------------------
 
-	private static <T> Matcher<TypeSerializerSchemaCompatibility<T>> hasSameCompatibilityType(TypeSerializerSchemaCompatibility<T> expectedCompatibilty) {
+	public static <T> Matcher<TypeSerializerSchemaCompatibility<T>> hasSameCompatibilityType(TypeSerializerSchemaCompatibility<T> expectedCompatibilty) {
 		return new TypeSafeMatcher<TypeSerializerSchemaCompatibility<T>>() {
 
 			@Override
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerSnapshotMigrationTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerSnapshotMigrationTest.java
new file mode 100644
index 0000000..5cbee5a
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerSnapshotMigrationTest.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.typeutils.runtime;
+
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
+import org.apache.flink.api.common.typeutils.TypeSerializerSnapshotMigrationTestBase;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.testutils.migration.MigrationVersion;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.Collection;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Migration tests for the {@link PojoSerializerSnapshot}.
+ */
+@RunWith(Parameterized.class)
+public class PojoSerializerSnapshotMigrationTest extends TypeSerializerSnapshotMigrationTestBase<Object> {
+
+	public static class TestPojo {
+		public int id;
+		public String name;
+		public int age;
+
+		public TestPojo() {}
+
+		public TestPojo(int id, String name, int age) {
+			this.id = id;
+			this.name = name;
+			this.age = age;
+		}
+	}
+
+	/**
+	 * Data test files of test specification that used this type
+	 * had 2 fields {@code idRemoved} and {@code heightRemoved} that
+	 * are no longer present in this current POJO.
+	 */
+	public static class TestPojoWithNewAndRemovedFields {
+
+		public enum Color {
+			RED,
+			BLUE,
+			GREEN,
+			YELLOW
+		}
+
+		public String name;
+		public int age;
+		public String githubId;
+		public Color favoriteColor;
+		public boolean married;
+
+		public TestPojoWithNewAndRemovedFields() {}
+
+		public TestPojoWithNewAndRemovedFields(String name, int age, String githubId, Color favoriteColor, boolean married) {
+			this.name = name;
+			this.age = age;
+			this.githubId = githubId;
+			this.favoriteColor = favoriteColor;
+			this.married = married;
+		}
+	}
+
+	public PojoSerializerSnapshotMigrationTest(TestSpecification<Object> testSpecification) {
+		super(testSpecification);
+	}
+
+	@SuppressWarnings("unchecked")
+	@Parameterized.Parameters(name = "Test Specification = {0}")
+	public static Collection<TestSpecification<?>> testSpecifications() {
+
+		final TestSpecifications testSpecifications = new TestSpecifications(MigrationVersion.v1_6, MigrationVersion.v1_7);
+
+		testSpecifications.add(
+			"pojo-serializer",
+			PojoSerializer.class,
+			PojoSerializerSnapshot.class,
+			PojoSerializerSnapshotMigrationTest::testPojoSerializerSupplier);
+
+		testSpecifications.addWithCompatibilityMatcher(
+			"pojo-new-and-removed-fields-serializer",
+			PojoSerializer.class,
+			PojoSerializerSnapshot.class,
+			PojoSerializerSnapshotMigrationTest::testPojoWithNewAndRemovedFieldsSerializerSupplier,
+			hasSameCompatibilityType(TypeSerializerSchemaCompatibility.compatibleAfterMigration()));
+
+		return testSpecifications.get();
+	}
+
+	private static TypeSerializer<TestPojo> testPojoSerializerSupplier() {
+		TypeSerializer<TestPojo> serializer =
+			TypeExtractor.createTypeInfo(TestPojo.class).createSerializer(new ExecutionConfig());
+
+		assertTrue(serializer instanceof PojoSerializer);
+		return serializer;
+	}
+
+	private static TypeSerializer<TestPojoWithNewAndRemovedFields> testPojoWithNewAndRemovedFieldsSerializerSupplier() {
+		TypeSerializer<TestPojoWithNewAndRemovedFields> serializer =
+			TypeExtractor.createTypeInfo(TestPojoWithNewAndRemovedFields.class).createSerializer(new ExecutionConfig());
+
+		assertTrue(serializer instanceof PojoSerializer);
+		return serializer;
+	}
+}
diff --git a/flink-core/src/test/resources/flink-1.6-pojo-new-and-removed-fields-serializer-data b/flink-core/src/test/resources/flink-1.6-pojo-new-and-removed-fields-serializer-data
new file mode 100644
index 0000000..8d8e619
Binary files /dev/null and b/flink-core/src/test/resources/flink-1.6-pojo-new-and-removed-fields-serializer-data differ
diff --git a/flink-core/src/test/resources/flink-1.6-pojo-new-and-removed-fields-serializer-snapshot b/flink-core/src/test/resources/flink-1.6-pojo-new-and-removed-fields-serializer-snapshot
new file mode 100644
index 0000000..9ebb10c
Binary files /dev/null and b/flink-core/src/test/resources/flink-1.6-pojo-new-and-removed-fields-serializer-snapshot differ
diff --git a/flink-core/src/test/resources/flink-1.6-pojo-serializer-data b/flink-core/src/test/resources/flink-1.6-pojo-serializer-data
new file mode 100644
index 0000000..153528f
Binary files /dev/null and b/flink-core/src/test/resources/flink-1.6-pojo-serializer-data differ
diff --git a/flink-core/src/test/resources/flink-1.6-pojo-serializer-snapshot b/flink-core/src/test/resources/flink-1.6-pojo-serializer-snapshot
new file mode 100644
index 0000000..4d53f6c
Binary files /dev/null and b/flink-core/src/test/resources/flink-1.6-pojo-serializer-snapshot differ
diff --git a/flink-core/src/test/resources/flink-1.7-pojo-new-and-removed-fields-serializer-data b/flink-core/src/test/resources/flink-1.7-pojo-new-and-removed-fields-serializer-data
new file mode 100644
index 0000000..8d8e619
Binary files /dev/null and b/flink-core/src/test/resources/flink-1.7-pojo-new-and-removed-fields-serializer-data differ
diff --git a/flink-core/src/test/resources/flink-1.7-pojo-new-and-removed-fields-serializer-snapshot b/flink-core/src/test/resources/flink-1.7-pojo-new-and-removed-fields-serializer-snapshot
new file mode 100644
index 0000000..7de3813
Binary files /dev/null and b/flink-core/src/test/resources/flink-1.7-pojo-new-and-removed-fields-serializer-snapshot differ
diff --git a/flink-core/src/test/resources/flink-1.7-pojo-serializer-data b/flink-core/src/test/resources/flink-1.7-pojo-serializer-data
new file mode 100644
index 0000000..153528f
Binary files /dev/null and b/flink-core/src/test/resources/flink-1.7-pojo-serializer-data differ
diff --git a/flink-core/src/test/resources/flink-1.7-pojo-serializer-snapshot b/flink-core/src/test/resources/flink-1.7-pojo-serializer-snapshot
new file mode 100644
index 0000000..4f5c928
Binary files /dev/null and b/flink-core/src/test/resources/flink-1.7-pojo-serializer-snapshot differ


[flink] 16/17: [FLINK-6768] [core] More efficient field serializer lookups in PojoSerializerSnapshot

Posted by tz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tzulitai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit cf65720aef60fe771833a96de4d0e2d6a9d11725
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
AuthorDate: Thu Feb 21 11:13:09 2019 +0800

    [FLINK-6768] [core] More efficient field serializer lookups in PojoSerializerSnapshot
    
    In the PojoSerializerSnapshot#resolveSchemaCompatibility method we call
    for each field the findField(...) method in the PojoSerializer, which
    performs a linear scan of the array of fields. This effectively has
    O(n^2) complexity.
    
    This is improved by this commit by building an index in the
    resolveSchemaCompatibility method for faster field serializer lookups.
---
 .../typeutils/runtime/PojoSerializerSnapshot.java  | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerSnapshot.java
index 5843168..7130134 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerSnapshot.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerSnapshot.java
@@ -275,8 +275,9 @@ public class PojoSerializerSnapshot<T> implements TypeSerializerSnapshot<T> {
 		final ArrayList<TypeSerializerSnapshot<?>> associatedFieldSerializerSnapshots = new ArrayList<>(presentFieldSnapshots.size());
 		final ArrayList<TypeSerializer<?>> associatedNewFieldSerializers = new ArrayList<>(presentFieldSnapshots.size());
 
+		final Map<Field, TypeSerializer<?>> newFieldSerializersIndex = buildNewFieldSerializersIndex(newPojoSerializer);
 		for (LinkedOptionalMap.KeyValue<Field, TypeSerializerSnapshot<?>> presentFieldEntry : presentFieldSnapshots) {
-			TypeSerializer<?> associatedNewFieldSerializer = newPojoSerializer.getFieldSerializer(presentFieldEntry.getKey());
+			TypeSerializer<?> associatedNewFieldSerializer = newFieldSerializersIndex.get(presentFieldEntry.getKey());
 			checkState(
 				associatedNewFieldSerializer != null,
 				"a present field should have its associated new field serializer available.");
@@ -291,6 +292,25 @@ public class PojoSerializerSnapshot<T> implements TypeSerializerSnapshot<T> {
 	}
 
 	/**
+	 * Builds an index of fields to their corresponding serializers for the
+	 * new {@link PojoSerializer} for faster field serializer lookups.
+	 */
+	private static <T> Map<Field, TypeSerializer<?>> buildNewFieldSerializersIndex(PojoSerializer<T> newPojoSerializer) {
+		final Field[] newFields = newPojoSerializer.getFields();
+		final TypeSerializer<?>[] newFieldSerializers = newPojoSerializer.getFieldSerializers();
+
+		checkState(newFields.length == newFieldSerializers.length);
+
+		int numFields = newFields.length;
+		final Map<Field, TypeSerializer<?>> index = new HashMap<>(numFields);
+		for (int i = 0; i < numFields; i++) {
+			index.put(newFields[i], newFieldSerializers[i]);
+		}
+
+		return index;
+	}
+
+	/**
 	 * Finds which registered subclasses exists both in the new {@link PojoSerializer} as well as in the previous one
 	 * (represented by this snapshot), and returns an {@link IntermediateCompatibilityResult}
 	 * of the serializers of this preexisting registered subclasses.