You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tz...@apache.org on 2018/02/06 19:03:25 UTC
[25/28] flink git commit: [FLINK-8421] [DataStream] Make timer
serializers reconfigurable on restore
[FLINK-8421] [DataStream] Make timer serializers reconfigurable on restore
Previously, the key and namespace serializers for the
HeapInternalTimerService were not reconfigured on restore to be compatible
with previously written serializers.
This caused an immediate error to restore savepoints in Flink 1.4.0,
since in Flink 1.4.0 we changed the base registrations in the Kryo
serializer. That change requires serializer reconfiguration.
This commit fixes this by writing also the serializer configuration
snapshots of the key and namespace serializer into savepoints, and use
them to reconfigure the new serializers on rrestore. This improvement also
comes along with making the written data for timer service snapshots
versioned. Backwards compatibility with previous non-versioned formats
is not broken.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/71d682eb
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/71d682eb
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/71d682eb
Branch: refs/heads/release-1.4
Commit: 71d682eb7fef37481669f2ad07f891411223c0f0
Parents: 93d514e
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Wed Jan 24 18:09:44 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 18:30:31 2018 +0100
----------------------------------------------------------------------
.../api/operators/AbstractStreamOperator.java | 3 +-
.../api/operators/HeapInternalTimerService.java | 122 ++++----
.../operators/InternalTimeServiceManager.java | 51 ++--
.../InternalTimerServiceSerializationProxy.java | 132 +++++++++
.../api/operators/InternalTimersSnapshot.java | 120 ++++++++
.../InternalTimersSnapshotReaderWriters.java | 291 +++++++++++++++++++
.../operators/HeapInternalTimerServiceTest.java | 124 +++++---
7 files changed, 696 insertions(+), 147 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/71d682eb/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
index 58022a9..a8d9555 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
@@ -28,7 +28,6 @@ import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.MetricOptions;
-import org.apache.flink.core.memory.DataInputViewStreamWrapper;
import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
import org.apache.flink.metrics.Counter;
import org.apache.flink.metrics.Gauge;
@@ -437,7 +436,7 @@ public abstract class AbstractStreamOperator<OUT>
"Key Group " + keyGroupIdx + " does not belong to the local range.");
timeServiceManager.restoreStateForKeyGroup(
- new DataInputViewStreamWrapper(streamProvider.getStream()),
+ streamProvider.getStream(),
keyGroupIdx, getUserCodeClassloader());
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/71d682eb/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/HeapInternalTimerService.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/HeapInternalTimerService.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/HeapInternalTimerService.java
index 1b531aa..f6c4d28 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/HeapInternalTimerService.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/HeapInternalTimerService.java
@@ -19,14 +19,13 @@
package org.apache.flink.streaming.api.operators;
import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
+import org.apache.flink.api.common.typeutils.CompatibilityUtil;
import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.core.memory.DataInputViewStreamWrapper;
-import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
import org.apache.flink.runtime.state.KeyGroupRangeAssignment;
import org.apache.flink.runtime.state.KeyGroupsList;
import org.apache.flink.streaming.runtime.tasks.ProcessingTimeCallback;
import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService;
-import org.apache.flink.util.InstantiationUtil;
import org.apache.flink.util.Preconditions;
import java.io.IOException;
@@ -84,8 +83,6 @@ public class HeapInternalTimerService<K, N> implements InternalTimerService<N>,
private TypeSerializer<N> namespaceSerializer;
- private InternalTimer.TimerSerializer<K, N> timerSerializer;
-
private Triggerable<K, N> triggerTarget;
private volatile boolean isInitialized;
@@ -94,6 +91,9 @@ public class HeapInternalTimerService<K, N> implements InternalTimerService<N>,
private TypeSerializer<N> namespaceDeserializer;
+ /** The restored timers snapshot, if any. */
+ private InternalTimersSnapshot<K, N> restoredTimersSnapshot;
+
public HeapInternalTimerService(
int totalKeyGroups,
KeyGroupsList localKeyGroupRange,
@@ -148,10 +148,23 @@ public class HeapInternalTimerService<K, N> implements InternalTimerService<N>,
}
// the following is the case where we restore
- if ((this.keyDeserializer != null && !this.keyDeserializer.equals(keySerializer)) ||
- (this.namespaceDeserializer != null && !this.namespaceDeserializer.equals(namespaceSerializer))) {
- throw new IllegalStateException("Tried to initialize restored TimerService " +
- "with different serializers than those used to snapshot its state.");
+ if (restoredTimersSnapshot != null) {
+ CompatibilityResult<K> keySerializerCompatibility = CompatibilityUtil.resolveCompatibilityResult(
+ this.keyDeserializer,
+ null,
+ restoredTimersSnapshot.getKeySerializerConfigSnapshot(),
+ keySerializer);
+
+ CompatibilityResult<N> namespaceSerializerCompatibility = CompatibilityUtil.resolveCompatibilityResult(
+ this.namespaceDeserializer,
+ null,
+ restoredTimersSnapshot.getNamespaceSerializerConfigSnapshot(),
+ namespaceSerializer);
+
+ if (keySerializerCompatibility.isRequiresMigration() || namespaceSerializerCompatibility.isRequiresMigration()) {
+ throw new IllegalStateException("Tried to initialize restored TimerService " +
+ "with incompatible serializers than those used to snapshot its state.");
+ }
}
this.keySerializer = keySerializer;
@@ -161,8 +174,6 @@ public class HeapInternalTimerService<K, N> implements InternalTimerService<N>,
this.triggerTarget = Preconditions.checkNotNull(triggerTarget);
- this.timerSerializer = new InternalTimer.TimerSerializer<>(this.keySerializer, this.namespaceSerializer);
-
// re-register the restored timers (if any)
if (processingTimeTimersQueue.size() > 0) {
nextTimer = processingTimeService.registerTimer(processingTimeTimersQueue.peek().getTimestamp(), this);
@@ -280,86 +291,53 @@ public class HeapInternalTimerService<K, N> implements InternalTimerService<N>,
/**
* Snapshots the timers (both processing and event time ones) for a given {@code keyGroupIdx}.
- * @param stream the stream to write to.
+ *
* @param keyGroupIdx the id of the key-group to be put in the snapshot.
+ * @return a snapshot containing the timers for the given key-group, and the serializers for them
*/
- public void snapshotTimersForKeyGroup(DataOutputViewStreamWrapper stream, int keyGroupIdx) throws Exception {
- InstantiationUtil.serializeObject(stream, keySerializer);
- InstantiationUtil.serializeObject(stream, namespaceSerializer);
-
- // write the event time timers
- Set<InternalTimer<K, N>> eventTimers = getEventTimeTimerSetForKeyGroup(keyGroupIdx);
- if (eventTimers != null) {
- stream.writeInt(eventTimers.size());
- for (InternalTimer<K, N> timer : eventTimers) {
- this.timerSerializer.serialize(timer, stream);
- }
- } else {
- stream.writeInt(0);
- }
-
- // write the processing time timers
- Set<InternalTimer<K, N>> processingTimers = getProcessingTimeTimerSetForKeyGroup(keyGroupIdx);
- if (processingTimers != null) {
- stream.writeInt(processingTimers.size());
- for (InternalTimer<K, N> timer : processingTimers) {
- this.timerSerializer.serialize(timer, stream);
- }
- } else {
- stream.writeInt(0);
- }
+ public InternalTimersSnapshot<K, N> snapshotTimersForKeyGroup(int keyGroupIdx) {
+ return new InternalTimersSnapshot<>(
+ keySerializer,
+ keySerializer.snapshotConfiguration(),
+ namespaceSerializer,
+ namespaceSerializer.snapshotConfiguration(),
+ getEventTimeTimerSetForKeyGroup(keyGroupIdx),
+ getProcessingTimeTimerSetForKeyGroup(keyGroupIdx));
}
/**
* Restore the timers (both processing and event time ones) for a given {@code keyGroupIdx}.
- * @param stream the stream to read from.
+ *
+ * @param restoredTimersSnapshot the restored snapshot containing the key-group's timers,
+ * and the serializers that were used to write them
* @param keyGroupIdx the id of the key-group to be put in the snapshot.
- * @param userCodeClassLoader the class loader that will be used to deserialize
- * the local key and namespace serializers.
*/
- public void restoreTimersForKeyGroup(DataInputViewStreamWrapper stream, int keyGroupIdx,
- ClassLoader userCodeClassLoader) throws IOException, ClassNotFoundException {
-
- TypeSerializer<K> tmpKeyDeserializer = InstantiationUtil.deserializeObject(stream, userCodeClassLoader);
- TypeSerializer<N> tmpNamespaceDeserializer = InstantiationUtil.deserializeObject(stream, userCodeClassLoader);
+ @SuppressWarnings("unchecked")
+ public void restoreTimersForKeyGroup(InternalTimersSnapshot<?, ?> restoredTimersSnapshot, int keyGroupIdx) throws IOException {
+ this.restoredTimersSnapshot = (InternalTimersSnapshot<K, N>) restoredTimersSnapshot;
- if ((this.keyDeserializer != null && !this.keyDeserializer.equals(tmpKeyDeserializer)) ||
- (this.namespaceDeserializer != null && !this.namespaceDeserializer.equals(tmpNamespaceDeserializer))) {
+ if ((this.keyDeserializer != null && !this.keyDeserializer.equals(restoredTimersSnapshot.getKeySerializer())) ||
+ (this.namespaceDeserializer != null && !this.namespaceDeserializer.equals(restoredTimersSnapshot.getNamespaceSerializer()))) {
throw new IllegalArgumentException("Tried to restore timers " +
"for the same service with different serializers.");
}
- this.keyDeserializer = tmpKeyDeserializer;
- this.namespaceDeserializer = tmpNamespaceDeserializer;
-
- InternalTimer.TimerSerializer<K, N> timerSerializer =
- new InternalTimer.TimerSerializer<>(this.keyDeserializer, this.namespaceDeserializer);
+ this.keyDeserializer = this.restoredTimersSnapshot.getKeySerializer();
+ this.namespaceDeserializer = this.restoredTimersSnapshot.getNamespaceSerializer();
checkArgument(localKeyGroupRange.contains(keyGroupIdx),
"Key Group " + keyGroupIdx + " does not belong to the local range.");
- // read the event time timers
- int sizeOfEventTimeTimers = stream.readInt();
- if (sizeOfEventTimeTimers > 0) {
- Set<InternalTimer<K, N>> eventTimers = getEventTimeTimerSetForKeyGroup(keyGroupIdx);
- for (int i = 0; i < sizeOfEventTimeTimers; i++) {
- InternalTimer<K, N> timer = timerSerializer.deserialize(stream);
- eventTimers.add(timer);
- eventTimeTimersQueue.add(timer);
- }
- }
+ // restore the event time timers
+ Set<InternalTimer<K, N>> eventTimers = getEventTimeTimerSetForKeyGroup(keyGroupIdx);
+ eventTimers.addAll(this.restoredTimersSnapshot.getEventTimeTimers());
+ eventTimeTimersQueue.addAll(this.restoredTimersSnapshot.getEventTimeTimers());
- // read the processing time timers
- int sizeOfProcessingTimeTimers = stream.readInt();
- if (sizeOfProcessingTimeTimers > 0) {
- Set<InternalTimer<K, N>> processingTimers = getProcessingTimeTimerSetForKeyGroup(keyGroupIdx);
- for (int i = 0; i < sizeOfProcessingTimeTimers; i++) {
- InternalTimer<K, N> timer = timerSerializer.deserialize(stream);
- processingTimers.add(timer);
- processingTimeTimersQueue.add(timer);
- }
- }
+ // restore the processing time timers
+ Set<InternalTimer<K, N>> processingTimers = getProcessingTimeTimerSetForKeyGroup(keyGroupIdx);
+ processingTimers.addAll(this.restoredTimersSnapshot.getProcessingTimeTimers());
+ processingTimeTimersQueue.addAll(this.restoredTimersSnapshot.getProcessingTimeTimers());
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/71d682eb/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManager.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManager.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManager.java
index 7d5cb91..84c7bf1 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManager.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManager.java
@@ -21,8 +21,7 @@ package org.apache.flink.streaming.api.operators;
import org.apache.flink.annotation.Internal;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.core.memory.DataInputViewStreamWrapper;
-import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
+import org.apache.flink.core.memory.DataOutputView;
import org.apache.flink.runtime.state.KeyGroupsList;
import org.apache.flink.runtime.state.VoidNamespaceSerializer;
import org.apache.flink.streaming.api.watermark.Watermark;
@@ -30,6 +29,7 @@ import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService;
import org.apache.flink.util.Preconditions;
import java.io.IOException;
+import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;
@@ -111,36 +111,29 @@ public class InternalTimeServiceManager<K, N> {
////////////////// Fault Tolerance Methods ///////////////////
- public void snapshotStateForKeyGroup(DataOutputViewStreamWrapper stream, int keyGroupIdx) throws Exception {
- stream.writeInt(timerServices.size());
+ public void snapshotStateForKeyGroup(DataOutputView stream, int keyGroupIdx) throws IOException {
+ InternalTimerServiceSerializationProxy<K, N> serializationProxy =
+ new InternalTimerServiceSerializationProxy<>(timerServices, keyGroupIdx);
- for (Map.Entry<String, HeapInternalTimerService<K, N>> entry : timerServices.entrySet()) {
- String serviceName = entry.getKey();
- HeapInternalTimerService<?, ?> timerService = entry.getValue();
-
- stream.writeUTF(serviceName);
- timerService.snapshotTimersForKeyGroup(stream, keyGroupIdx);
- }
+ serializationProxy.write(stream);
}
- public void restoreStateForKeyGroup(DataInputViewStreamWrapper stream, int keyGroupIdx,
- ClassLoader userCodeClassLoader) throws IOException, ClassNotFoundException {
-
- int noOfTimerServices = stream.readInt();
- for (int i = 0; i < noOfTimerServices; i++) {
- String serviceName = stream.readUTF();
-
- HeapInternalTimerService<K, N> timerService = timerServices.get(serviceName);
- if (timerService == null) {
- timerService = new HeapInternalTimerService<>(
- totalKeyGroups,
- localKeyGroupRange,
- keyContext,
- processingTimeService);
- timerServices.put(serviceName, timerService);
- }
- timerService.restoreTimersForKeyGroup(stream, keyGroupIdx, userCodeClassLoader);
- }
+ public void restoreStateForKeyGroup(
+ InputStream stream,
+ int keyGroupIdx,
+ ClassLoader userCodeClassLoader) throws IOException {
+
+ InternalTimerServiceSerializationProxy<K, N> serializationProxy =
+ new InternalTimerServiceSerializationProxy<>(
+ timerServices,
+ userCodeClassLoader,
+ totalKeyGroups,
+ localKeyGroupRange,
+ keyContext,
+ processingTimeService,
+ keyGroupIdx);
+
+ serializationProxy.read(stream);
}
//////////////////// Methods used ONLY IN TESTS ////////////////////
http://git-wip-us.apache.org/repos/asf/flink/blob/71d682eb/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimerServiceSerializationProxy.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimerServiceSerializationProxy.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimerServiceSerializationProxy.java
new file mode 100644
index 0000000..53d1507
--- /dev/null
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimerServiceSerializationProxy.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.operators;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.core.io.PostVersionedIOReadableWritable;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.runtime.state.KeyGroupsList;
+import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * Serialization proxy for the timer services for a given key-group.
+ */
+@Internal
+public class InternalTimerServiceSerializationProxy<K, N> extends PostVersionedIOReadableWritable {
+
+ public static final int VERSION = 1;
+
+ /** The key-group timer services to write / read. */
+ private Map<String, HeapInternalTimerService<K, N>> timerServices;
+
+ /** The user classloader; only relevant if the proxy is used to restore timer services. */
+ private ClassLoader userCodeClassLoader;
+
+ /** Properties of restored timer services. */
+ private int keyGroupIdx;
+ private int totalKeyGroups;
+ private KeyGroupsList localKeyGroupRange;
+ private KeyContext keyContext;
+ private ProcessingTimeService processingTimeService;
+
+ /**
+ * Constructor to use when restoring timer services.
+ */
+ public InternalTimerServiceSerializationProxy(
+ Map<String, HeapInternalTimerService<K, N>> timerServicesMapToPopulate,
+ ClassLoader userCodeClassLoader,
+ int totalKeyGroups,
+ KeyGroupsList localKeyGroupRange,
+ KeyContext keyContext,
+ ProcessingTimeService processingTimeService,
+ int keyGroupIdx) {
+
+ this.timerServices = checkNotNull(timerServicesMapToPopulate);
+ this.userCodeClassLoader = checkNotNull(userCodeClassLoader);
+ this.totalKeyGroups = totalKeyGroups;
+ this.localKeyGroupRange = checkNotNull(localKeyGroupRange);
+ this.keyContext = checkNotNull(keyContext);
+ this.processingTimeService = checkNotNull(processingTimeService);
+ this.keyGroupIdx = keyGroupIdx;
+ }
+
+ /**
+ * Constructor to use when writing timer services.
+ */
+ public InternalTimerServiceSerializationProxy(
+ Map<String, HeapInternalTimerService<K, N>> timerServices,
+ int keyGroupIdx) {
+
+ this.timerServices = checkNotNull(timerServices);
+ this.keyGroupIdx = keyGroupIdx;
+ }
+
+ @Override
+ public int getVersion() {
+ return VERSION;
+ }
+
+ @Override
+ public void write(DataOutputView out) throws IOException {
+ super.write(out);
+
+ out.writeInt(timerServices.size());
+ for (Map.Entry<String, HeapInternalTimerService<K, N>> entry : timerServices.entrySet()) {
+ String serviceName = entry.getKey();
+ HeapInternalTimerService<K, N> timerService = entry.getValue();
+
+ out.writeUTF(serviceName);
+ InternalTimersSnapshotReaderWriters
+ .getWriterForVersion(VERSION, timerService.snapshotTimersForKeyGroup(keyGroupIdx))
+ .writeTimersSnapshot(out);
+ }
+ }
+
+ @Override
+ protected void read(DataInputView in, boolean wasVersioned) throws IOException {
+ int noOfTimerServices = in.readInt();
+
+ for (int i = 0; i < noOfTimerServices; i++) {
+ String serviceName = in.readUTF();
+
+ HeapInternalTimerService<K, N> timerService = timerServices.get(serviceName);
+ if (timerService == null) {
+ timerService = new HeapInternalTimerService<>(
+ totalKeyGroups,
+ localKeyGroupRange,
+ keyContext,
+ processingTimeService);
+ timerServices.put(serviceName, timerService);
+ }
+
+ int readerVersion = wasVersioned ? getReadVersion() : InternalTimersSnapshotReaderWriters.NO_VERSION;
+ InternalTimersSnapshot<?, ?> restoredTimersSnapshot = InternalTimersSnapshotReaderWriters
+ .getReaderForVersion(readerVersion, userCodeClassLoader)
+ .readTimersSnapshot(in);
+
+ timerService.restoreTimersForKeyGroup(restoredTimersSnapshot, keyGroupIdx);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/71d682eb/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimersSnapshot.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimersSnapshot.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimersSnapshot.java
new file mode 100644
index 0000000..ddaee6b
--- /dev/null
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimersSnapshot.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.operators;
+
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
+import org.apache.flink.util.Preconditions;
+
+import javax.annotation.Nullable;
+
+import java.util.Set;
+
+/**
+ * A snapshot of internal timers, containing event and processing timers and
+ * the serializers to use to write / read them.
+ */
+public class InternalTimersSnapshot<K, N> {
+
+ private TypeSerializer<K> keySerializer;
+ private TypeSerializerConfigSnapshot keySerializerConfigSnapshot;
+ private TypeSerializer<N> namespaceSerializer;
+ private TypeSerializerConfigSnapshot namespaceSerializerConfigSnapshot;
+
+ private Set<InternalTimer<K, N>> eventTimeTimers;
+ private Set<InternalTimer<K, N>> processingTimeTimers;
+
+ /** Empty constructor used when restoring the timers. */
+ public InternalTimersSnapshot() {}
+
+ /** Constructor to use when snapshotting the timers. */
+ public InternalTimersSnapshot(
+ TypeSerializer<K> keySerializer,
+ TypeSerializerConfigSnapshot keySerializerConfigSnapshot,
+ TypeSerializer<N> namespaceSerializer,
+ TypeSerializerConfigSnapshot namespaceSerializerConfigSnapshot,
+ @Nullable Set<InternalTimer<K, N>> eventTimeTimers,
+ @Nullable Set<InternalTimer<K, N>> processingTimeTimers) {
+
+ this.keySerializer = Preconditions.checkNotNull(keySerializer);
+ this.keySerializerConfigSnapshot = Preconditions.checkNotNull(keySerializerConfigSnapshot);
+ this.namespaceSerializer = Preconditions.checkNotNull(namespaceSerializer);
+ this.namespaceSerializerConfigSnapshot = Preconditions.checkNotNull(namespaceSerializerConfigSnapshot);
+ this.eventTimeTimers = eventTimeTimers;
+ this.processingTimeTimers = processingTimeTimers;
+ }
+
+ public TypeSerializer<K> getKeySerializer() {
+ return keySerializer;
+ }
+
+ public void setKeySerializer(TypeSerializer<K> keySerializer) {
+ this.keySerializer = keySerializer;
+ }
+
+ public TypeSerializerConfigSnapshot getKeySerializerConfigSnapshot() {
+ return keySerializerConfigSnapshot;
+ }
+
+ public void setKeySerializerConfigSnapshot(TypeSerializerConfigSnapshot keySerializerConfigSnapshot) {
+ this.keySerializerConfigSnapshot = keySerializerConfigSnapshot;
+ }
+
+ public TypeSerializer<N> getNamespaceSerializer() {
+ return namespaceSerializer;
+ }
+
+ public void setNamespaceSerializer(TypeSerializer<N> namespaceSerializer) {
+ this.namespaceSerializer = namespaceSerializer;
+ }
+
+ public TypeSerializerConfigSnapshot getNamespaceSerializerConfigSnapshot() {
+ return namespaceSerializerConfigSnapshot;
+ }
+
+ public void setNamespaceSerializerConfigSnapshot(TypeSerializerConfigSnapshot namespaceSerializerConfigSnapshot) {
+ this.namespaceSerializerConfigSnapshot = namespaceSerializerConfigSnapshot;
+ }
+
+ public Set<InternalTimer<K, N>> getEventTimeTimers() {
+ return eventTimeTimers;
+ }
+
+ public void setEventTimeTimers(Set<InternalTimer<K, N>> eventTimeTimers) {
+ this.eventTimeTimers = eventTimeTimers;
+ }
+
+ public Set<InternalTimer<K, N>> getProcessingTimeTimers() {
+ return processingTimeTimers;
+ }
+
+ public void setProcessingTimeTimers(Set<InternalTimer<K, N>> processingTimeTimers) {
+ this.processingTimeTimers = processingTimeTimers;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return super.equals(obj);
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/71d682eb/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimersSnapshotReaderWriters.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimersSnapshotReaderWriters.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimersSnapshotReaderWriters.java
new file mode 100644
index 0000000..3173625
--- /dev/null
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimersSnapshotReaderWriters.java
@@ -0,0 +1,291 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.operators;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
+import org.apache.flink.api.common.typeutils.TypeSerializerSerializationUtil;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.typeutils.runtime.DataInputViewStream;
+import org.apache.flink.core.memory.ByteArrayOutputStreamWithPos;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.util.InstantiationUtil;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * Readers and writers for different versions of the {@link InternalTimersSnapshot}.
+ * Outdated formats are also kept here for documentation of history backlog.
+ */
+@Internal
+public class InternalTimersSnapshotReaderWriters {
+
+ public static final int NO_VERSION = Integer.MIN_VALUE;
+
+ // -------------------------------------------------------------------------------
+ // Writers
+ // - pre-versioned: Flink 1.4.0
+ // - v1: Flink 1.4.1
+ // -------------------------------------------------------------------------------
+
+ public static <K, N> InternalTimersSnapshotWriter getWriterForVersion(int version, InternalTimersSnapshot<K, N> timersSnapshot) {
+
+ switch (version) {
+ case NO_VERSION:
+ return new InternalTimersSnapshotWriterPreVersioned<>(timersSnapshot);
+
+ case InternalTimerServiceSerializationProxy.VERSION:
+ return new InternalTimersSnapshotWriterV1<>(timersSnapshot);
+
+ default:
+ // guard for future
+ throw new IllegalStateException(
+ "Unrecognized internal timers snapshot writer version: " + version);
+ }
+ }
+
+ /**
+ * A writer for a {@link InternalTimersSnapshot}.
+ */
+ public interface InternalTimersSnapshotWriter {
+
+ /**
+ * Writes the timers snapshot to the output view.
+ *
+ * @param out the output view to write to
+ * @throws IOException
+ */
+ void writeTimersSnapshot(DataOutputView out) throws IOException;
+ }
+
+ private abstract static class AbstractInternalTimersSnapshotWriter<K, N> implements InternalTimersSnapshotWriter {
+
+ protected final InternalTimersSnapshot<K, N> timersSnapshot;
+
+ public AbstractInternalTimersSnapshotWriter(InternalTimersSnapshot<K, N> timersSnapshot) {
+ this.timersSnapshot = checkNotNull(timersSnapshot);
+ }
+
+ protected abstract void writeKeyAndNamespaceSerializers(DataOutputView out) throws IOException;
+
+ @Override
+ public final void writeTimersSnapshot(DataOutputView out) throws IOException {
+ writeKeyAndNamespaceSerializers(out);
+
+ InternalTimer.TimerSerializer<K, N> timerSerializer = new InternalTimer.TimerSerializer<>(
+ timersSnapshot.getKeySerializer(),
+ timersSnapshot.getNamespaceSerializer());
+
+ // write the event time timers
+ Set<InternalTimer<K, N>> eventTimers = timersSnapshot.getEventTimeTimers();
+ if (eventTimers != null) {
+ out.writeInt(eventTimers.size());
+ for (InternalTimer<K, N> eventTimer : eventTimers) {
+ timerSerializer.serialize(eventTimer, out);
+ }
+ } else {
+ out.writeInt(0);
+ }
+
+ // write the processing time timers
+ Set<InternalTimer<K, N>> processingTimers = timersSnapshot.getProcessingTimeTimers();
+ if (processingTimers != null) {
+ out.writeInt(processingTimers.size());
+ for (InternalTimer<K, N> processingTimer : processingTimers) {
+ timerSerializer.serialize(processingTimer, out);
+ }
+ } else {
+ out.writeInt(0);
+ }
+ }
+ }
+
+ private static class InternalTimersSnapshotWriterPreVersioned<K, N> extends AbstractInternalTimersSnapshotWriter<K, N> {
+
+ public InternalTimersSnapshotWriterPreVersioned(InternalTimersSnapshot<K, N> timersSnapshot) {
+ super(timersSnapshot);
+ }
+
+ @Override
+ protected void writeKeyAndNamespaceSerializers(DataOutputView out) throws IOException {
+ // the pre-versioned format only serializes the serializers, without their configuration snapshots
+ try (ByteArrayOutputStreamWithPos stream = new ByteArrayOutputStreamWithPos()) {
+ InstantiationUtil.serializeObject(stream, timersSnapshot.getKeySerializer());
+ InstantiationUtil.serializeObject(stream, timersSnapshot.getNamespaceSerializer());
+
+ out.write(stream.getBuf(), 0, stream.getPosition());
+ }
+ }
+ }
+
+ private static class InternalTimersSnapshotWriterV1<K, N> extends AbstractInternalTimersSnapshotWriter<K, N> {
+
+ public InternalTimersSnapshotWriterV1(InternalTimersSnapshot<K, N> timersSnapshot) {
+ super(timersSnapshot);
+ }
+
+ @Override
+ protected void writeKeyAndNamespaceSerializers(DataOutputView out) throws IOException {
+ // write key / namespace serializers, and their configuration snapshots
+ TypeSerializerSerializationUtil.writeSerializersAndConfigsWithResilience(
+ out,
+ Arrays.asList(
+ Tuple2.of(timersSnapshot.getKeySerializer(), timersSnapshot.getKeySerializerConfigSnapshot()),
+ Tuple2.of(timersSnapshot.getNamespaceSerializer(), timersSnapshot.getNamespaceSerializerConfigSnapshot())));
+ }
+ }
+
+ // -------------------------------------------------------------------------------
+ // Readers
+ // - pre-versioned: Flink 1.4.0
+ // - v1: Flink 1.4.1
+ // -------------------------------------------------------------------------------
+
+ public static <K, N> InternalTimersSnapshotReader<K, N> getReaderForVersion(
+ int version, ClassLoader userCodeClassLoader) {
+
+ switch (version) {
+ case NO_VERSION:
+ return new InternalTimersSnapshotReaderPreVersioned<>(userCodeClassLoader);
+
+ case InternalTimerServiceSerializationProxy.VERSION:
+ return new InternalTimersSnapshotReaderV1<>(userCodeClassLoader);
+
+ default:
+ // guard for future
+ throw new IllegalStateException(
+ "Unrecognized internal timers snapshot writer version: " + version);
+ }
+ }
+
+ /**
+ * A reader for a {@link InternalTimersSnapshot}.
+ */
+ public interface InternalTimersSnapshotReader<K, N> {
+
+ /**
+ * Reads a timers snapshot from the provided input view.
+ *
+ * @param in the input view
+ * @return the read timers snapshot
+ * @throws IOException
+ */
+ InternalTimersSnapshot<K, N> readTimersSnapshot(DataInputView in) throws IOException;
+ }
+
+ private abstract static class AbstractInternalTimersSnapshotReader<K, N> implements InternalTimersSnapshotReader<K, N> {
+
+ protected final ClassLoader userCodeClassLoader;
+
+ public AbstractInternalTimersSnapshotReader(ClassLoader userCodeClassLoader) {
+ this.userCodeClassLoader = checkNotNull(userCodeClassLoader);
+ }
+
+ protected abstract void restoreKeyAndNamespaceSerializers(
+ InternalTimersSnapshot<K, N> restoredTimersSnapshot,
+ DataInputView in) throws IOException;
+
+ @Override
+ public final InternalTimersSnapshot<K, N> readTimersSnapshot(DataInputView in) throws IOException {
+ InternalTimersSnapshot<K, N> restoredTimersSnapshot = new InternalTimersSnapshot<>();
+
+ restoreKeyAndNamespaceSerializers(restoredTimersSnapshot, in);
+
+ InternalTimer.TimerSerializer<K, N> timerSerializer = new InternalTimer.TimerSerializer<>(
+ restoredTimersSnapshot.getKeySerializer(),
+ restoredTimersSnapshot.getNamespaceSerializer());
+
+ // read the event time timers
+ int sizeOfEventTimeTimers = in.readInt();
+ Set<InternalTimer<K, N>> restoredEventTimers = new HashSet<>(sizeOfEventTimeTimers);
+ if (sizeOfEventTimeTimers > 0) {
+ for (int i = 0; i < sizeOfEventTimeTimers; i++) {
+ InternalTimer<K, N> timer = timerSerializer.deserialize(in);
+ restoredEventTimers.add(timer);
+ }
+ }
+ restoredTimersSnapshot.setEventTimeTimers(restoredEventTimers);
+
+ // read the processing time timers
+ int sizeOfProcessingTimeTimers = in.readInt();
+ Set<InternalTimer<K, N>> restoredProcessingTimers = new HashSet<>(sizeOfProcessingTimeTimers);
+ if (sizeOfProcessingTimeTimers > 0) {
+ for (int i = 0; i < sizeOfProcessingTimeTimers; i++) {
+ InternalTimer<K, N> timer = timerSerializer.deserialize(in);
+ restoredProcessingTimers.add(timer);
+ }
+ }
+ restoredTimersSnapshot.setProcessingTimeTimers(restoredProcessingTimers);
+
+ return restoredTimersSnapshot;
+ }
+ }
+
+ private static class InternalTimersSnapshotReaderPreVersioned<K, N> extends AbstractInternalTimersSnapshotReader<K, N> {
+
+ public InternalTimersSnapshotReaderPreVersioned(ClassLoader userCodeClassLoader) {
+ super(userCodeClassLoader);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ protected void restoreKeyAndNamespaceSerializers(
+ InternalTimersSnapshot<K, N> restoredTimersSnapshot,
+ DataInputView in) throws IOException {
+
+ DataInputViewStream dis = new DataInputViewStream(in);
+ try {
+ restoredTimersSnapshot.setKeySerializer(InstantiationUtil.deserializeObject(dis, userCodeClassLoader, true));
+ restoredTimersSnapshot.setNamespaceSerializer(InstantiationUtil.deserializeObject(dis, userCodeClassLoader, true));
+ } catch (ClassNotFoundException exception) {
+ throw new IOException(exception);
+ }
+ }
+ }
+
+ private static class InternalTimersSnapshotReaderV1<K, N> extends AbstractInternalTimersSnapshotReader<K, N> {
+
+ public InternalTimersSnapshotReaderV1(ClassLoader userCodeClassLoader) {
+ super(userCodeClassLoader);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ protected void restoreKeyAndNamespaceSerializers(
+ InternalTimersSnapshot<K, N> restoredTimersSnapshot,
+ DataInputView in) throws IOException {
+
+ List<Tuple2<TypeSerializer<?>, TypeSerializerConfigSnapshot>> serializersAndConfigs =
+ TypeSerializerSerializationUtil.readSerializersAndConfigsWithResilience(in, userCodeClassLoader);
+
+ restoredTimersSnapshot.setKeySerializer((TypeSerializer<K>) serializersAndConfigs.get(0).f0);
+ restoredTimersSnapshot.setKeySerializerConfigSnapshot(serializersAndConfigs.get(0).f1);
+ restoredTimersSnapshot.setNamespaceSerializer((TypeSerializer<N>) serializersAndConfigs.get(1).f0);
+ restoredTimersSnapshot.setNamespaceSerializerConfigSnapshot(serializersAndConfigs.get(1).f1);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/71d682eb/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/HeapInternalTimerServiceTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/HeapInternalTimerServiceTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/HeapInternalTimerServiceTest.java
index 5a4f1c4..aade876 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/HeapInternalTimerServiceTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/HeapInternalTimerServiceTest.java
@@ -548,13 +548,36 @@ public class HeapInternalTimerServiceTest {
@Test
public void testSnapshotAndRestore() throws Exception {
+ testSnapshotAndRestore(InternalTimerServiceSerializationProxy.VERSION);
+ }
+
+ @Test
+ public void testSnapshotAndRestorePreVersioned() throws Exception {
+ testSnapshotAndRestore(InternalTimersSnapshotReaderWriters.NO_VERSION);
+ }
+
+ /**
+ * This test checks whether timers are assigned to correct key groups
+ * and whether snapshot/restore respects key groups.
+ */
+ @Test
+ public void testSnapshotAndRebalancingRestore() throws Exception {
+ testSnapshotAndRebalancingRestore(InternalTimerServiceSerializationProxy.VERSION);
+ }
+
+ @Test
+ public void testSnapshotAndRebalancingRestorePreVersioned() throws Exception {
+ testSnapshotAndRebalancingRestore(InternalTimersSnapshotReaderWriters.NO_VERSION);
+ }
+
+ private void testSnapshotAndRestore(int snapshotVersion) throws Exception {
@SuppressWarnings("unchecked")
Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class);
TestKeyContext keyContext = new TestKeyContext();
TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
HeapInternalTimerService<Integer, String> timerService =
- createTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, maxParallelism);
+ createTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, maxParallelism);
// get two different keys
int key1 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
@@ -582,10 +605,15 @@ public class HeapInternalTimerServiceTest {
Map<Integer, byte[]> snapshot = new HashMap<>();
for (Integer keyGroupIndex : testKeyGroupRange) {
- ByteArrayOutputStream outStream = new ByteArrayOutputStream();
- timerService.snapshotTimersForKeyGroup(new DataOutputViewStreamWrapper(outStream), keyGroupIndex);
- outStream.close();
- snapshot.put(keyGroupIndex, outStream.toByteArray());
+ try (ByteArrayOutputStream outStream = new ByteArrayOutputStream()) {
+ InternalTimersSnapshot<?, ?> timersSnapshot = timerService.snapshotTimersForKeyGroup(keyGroupIndex);
+
+ InternalTimersSnapshotReaderWriters
+ .getWriterForVersion(snapshotVersion, timersSnapshot)
+ .writeTimersSnapshot(new DataOutputViewStreamWrapper(outStream));
+
+ snapshot.put(keyGroupIndex, outStream.toByteArray());
+ }
}
@SuppressWarnings("unchecked")
@@ -595,12 +623,13 @@ public class HeapInternalTimerServiceTest {
processingTimeService = new TestProcessingTimeService();
timerService = restoreTimerService(
- snapshot,
- mockTriggerable2,
- keyContext,
- processingTimeService,
- testKeyGroupRange,
- maxParallelism);
+ snapshot,
+ snapshotVersion,
+ mockTriggerable2,
+ keyContext,
+ processingTimeService,
+ testKeyGroupRange,
+ maxParallelism);
processingTimeService.setCurrentTime(10);
timerService.advanceWatermark(10);
@@ -615,22 +644,17 @@ public class HeapInternalTimerServiceTest {
assertEquals(0, timerService.numEventTimeTimers());
}
- /**
- * This test checks whether timers are assigned to correct key groups
- * and whether snapshot/restore respects key groups.
- */
- @Test
- public void testSnapshotAndRebalancingRestore() throws Exception {
+ private void testSnapshotAndRebalancingRestore(int snapshotVersion) throws Exception {
@SuppressWarnings("unchecked")
Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class);
TestKeyContext keyContext = new TestKeyContext();
TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
HeapInternalTimerService<Integer, String> timerService =
- createTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, maxParallelism);
+ createTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, maxParallelism);
int midpoint = testKeyGroupRange.getStartKeyGroup() +
- (testKeyGroupRange.getEndKeyGroup() - testKeyGroupRange.getStartKeyGroup()) / 2;
+ (testKeyGroupRange.getEndKeyGroup() - testKeyGroupRange.getStartKeyGroup()) / 2;
// get two sub key-ranges so that we can restore two ranges separately
KeyGroupRange subKeyGroupRange1 = new KeyGroupRange(testKeyGroupRange.getStartKeyGroup(), midpoint);
@@ -661,15 +685,20 @@ public class HeapInternalTimerServiceTest {
Map<Integer, byte[]> snapshot1 = new HashMap<>();
Map<Integer, byte[]> snapshot2 = new HashMap<>();
for (Integer keyGroupIndex : testKeyGroupRange) {
- ByteArrayOutputStream outStream = new ByteArrayOutputStream();
- timerService.snapshotTimersForKeyGroup(new DataOutputViewStreamWrapper(outStream), keyGroupIndex);
- outStream.close();
- if (subKeyGroupRange1.contains(keyGroupIndex)) {
- snapshot1.put(keyGroupIndex, outStream.toByteArray());
- } else if (subKeyGroupRange2.contains(keyGroupIndex)) {
- snapshot2.put(keyGroupIndex, outStream.toByteArray());
- } else {
- throw new IllegalStateException("Key-Group index doesn't belong to any sub range.");
+ try (ByteArrayOutputStream outStream = new ByteArrayOutputStream()) {
+ InternalTimersSnapshot<?, ?> timersSnapshot = timerService.snapshotTimersForKeyGroup(keyGroupIndex);
+
+ InternalTimersSnapshotReaderWriters
+ .getWriterForVersion(snapshotVersion, timersSnapshot)
+ .writeTimersSnapshot(new DataOutputViewStreamWrapper(outStream));
+
+ if (subKeyGroupRange1.contains(keyGroupIndex)) {
+ snapshot1.put(keyGroupIndex, outStream.toByteArray());
+ } else if (subKeyGroupRange2.contains(keyGroupIndex)) {
+ snapshot2.put(keyGroupIndex, outStream.toByteArray());
+ } else {
+ throw new IllegalStateException("Key-Group index doesn't belong to any sub range.");
+ }
}
}
@@ -687,20 +716,22 @@ public class HeapInternalTimerServiceTest {
TestProcessingTimeService processingTimeService2 = new TestProcessingTimeService();
HeapInternalTimerService<Integer, String> timerService1 = restoreTimerService(
- snapshot1,
- mockTriggerable1,
- keyContext1,
- processingTimeService1,
- subKeyGroupRange1,
- maxParallelism);
+ snapshot1,
+ snapshotVersion,
+ mockTriggerable1,
+ keyContext1,
+ processingTimeService1,
+ subKeyGroupRange1,
+ maxParallelism);
HeapInternalTimerService<Integer, String> timerService2 = restoreTimerService(
- snapshot2,
- mockTriggerable2,
- keyContext2,
- processingTimeService2,
- subKeyGroupRange2,
- maxParallelism);
+ snapshot2,
+ snapshotVersion,
+ mockTriggerable2,
+ keyContext2,
+ processingTimeService2,
+ subKeyGroupRange2,
+ maxParallelism);
processingTimeService1.setCurrentTime(10);
timerService1.advanceWatermark(10);
@@ -779,6 +810,7 @@ public class HeapInternalTimerServiceTest {
private static HeapInternalTimerService<Integer, String> restoreTimerService(
Map<Integer, byte[]> state,
+ int snapshotVersion,
Triggerable<Integer, String> triggerable,
KeyContext keyContext,
ProcessingTimeService processingTimeService,
@@ -796,10 +828,14 @@ public class HeapInternalTimerServiceTest {
// restore the timers
for (Integer keyGroupIndex : keyGroupsList) {
if (state.containsKey(keyGroupIndex)) {
- service.restoreTimersForKeyGroup(
- new DataInputViewStreamWrapper(new ByteArrayInputStream(state.get(keyGroupIndex))),
- keyGroupIndex,
- HeapInternalTimerServiceTest.class.getClassLoader());
+ try (ByteArrayInputStream inputStream = new ByteArrayInputStream(state.get(keyGroupIndex))) {
+ InternalTimersSnapshot<?, ?> restoredTimersSnapshot =
+ InternalTimersSnapshotReaderWriters
+ .getReaderForVersion(snapshotVersion, HeapInternalTimerServiceTest.class.getClassLoader())
+ .readTimersSnapshot(new DataInputViewStreamWrapper(inputStream));
+
+ service.restoreTimersForKeyGroup(restoredTimersSnapshot, keyGroupIndex);
+ }
}
}