You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tz...@apache.org on 2018/02/06 19:03:01 UTC
[01/28] flink git commit: [FLINK-8421] [core] Allow
InstantiationUtil.deserializeObject() to use FailureTolerantObjectInputStream
Repository: flink
Updated Branches:
refs/heads/release-1.4 c9322661e -> 82f395781
[FLINK-8421] [core] Allow InstantiationUtil.deserializeObject() to use FailureTolerantObjectInputStream
This makes it possible to use the FailureTolerantObjectInputStream in
InstantiationUtil.deserializeObject. This behaviour is controlled via a
isFailureTolerant flag argument. All previous calls to the method
remains to have identical behaviour (the ClassloaderObjectInputStream).
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/319d5101
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/319d5101
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/319d5101
Branch: refs/heads/release-1.4
Commit: 319d510174f7862d71a3081f5a9d71eb3ae1de00
Parents: 1f81f6a
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Wed Jan 24 17:15:08 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 17:31:32 2018 +0100
----------------------------------------------------------------------
.../TypeSerializerSerializationUtil.java | 83 +------------
.../apache/flink/util/InstantiationUtil.java | 116 ++++++++++++++++---
...ckendStateMetaInfoSnapshotReaderWriters.java | 5 +-
3 files changed, 107 insertions(+), 97 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/319d5101/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationUtil.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationUtil.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationUtil.java
index c6291ad..3f82530 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationUtil.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationUtil.java
@@ -20,7 +20,6 @@ package org.apache.flink.api.common.typeutils;
import org.apache.flink.annotation.Internal;
import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.typeutils.runtime.KryoRegistrationSerializerConfigSnapshot;
import org.apache.flink.core.io.VersionedIOReadableWritable;
import org.apache.flink.core.memory.ByteArrayInputStreamWithPos;
import org.apache.flink.core.memory.ByteArrayOutputStreamWithPos;
@@ -36,14 +35,9 @@ import org.slf4j.LoggerFactory;
import java.io.ByteArrayInputStream;
import java.io.IOException;
-import java.io.InputStream;
import java.io.InvalidClassException;
-import java.io.ObjectInputStream;
-import java.io.ObjectStreamClass;
import java.util.ArrayList;
-import java.util.HashSet;
import java.util.List;
-import java.util.Set;
/**
* Utility methods for serialization of {@link TypeSerializer} and {@link TypeSerializerConfigSnapshot}.
@@ -54,79 +48,6 @@ public class TypeSerializerSerializationUtil {
private static final Logger LOG = LoggerFactory.getLogger(TypeSerializerSerializationUtil.class);
/**
- * This is maintained as a temporary workaround for FLINK-6869.
- *
- * <p>Before 1.3, the Scala serializers did not specify the serialVersionUID.
- * Although since 1.3 they are properly specified, we still have to ignore them for now
- * as their previous serialVersionUIDs will vary depending on the Scala version.
- *
- * <p>This can be removed once 1.2 is no longer supported.
- */
- private static Set<String> scalaSerializerClassnames = new HashSet<>();
- static {
- scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.TraversableSerializer");
- scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.CaseClassSerializer");
- scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.EitherSerializer");
- scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.EnumValueSerializer");
- scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.OptionSerializer");
- scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.TrySerializer");
- scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.EitherSerializer");
- scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.UnitSerializer");
- }
-
- /**
- * An {@link ObjectInputStream} that ignores serialVersionUID mismatches when deserializing objects of
- * anonymous classes or our Scala serializer classes and also replaces occurences of GenericData.Array
- * (from Avro) by a dummy class so that the KryoSerializer can still be deserialized without
- * Avro being on the classpath.
- *
- * <p>The {@link TypeSerializerSerializationProxy} uses this specific object input stream to read serializers,
- * so that mismatching serialVersionUIDs of anonymous classes / Scala serializers are ignored.
- * This is a required workaround to maintain backwards compatibility for our pre-1.3 Scala serializers.
- * See FLINK-6869 for details.
- *
- * @see <a href="https://issues.apache.org/jira/browse/FLINK-6869">FLINK-6869</a>
- */
- public static class FailureTolerantObjectInputStream extends InstantiationUtil.ClassLoaderObjectInputStream {
-
- public FailureTolerantObjectInputStream(InputStream in, ClassLoader cl) throws IOException {
- super(in, cl);
- }
-
- @Override
- protected ObjectStreamClass readClassDescriptor() throws IOException, ClassNotFoundException {
- ObjectStreamClass streamClassDescriptor = super.readClassDescriptor();
-
- try {
- Class.forName(streamClassDescriptor.getName(), false, classLoader);
- } catch (ClassNotFoundException e) {
- if (streamClassDescriptor.getName().equals("org.apache.avro.generic.GenericData$Array")) {
- ObjectStreamClass result = ObjectStreamClass.lookup(
- KryoRegistrationSerializerConfigSnapshot.DummyRegisteredClass.class);
- return result;
- }
- }
-
- Class localClass = resolveClass(streamClassDescriptor);
- if (scalaSerializerClassnames.contains(localClass.getName()) || localClass.isAnonymousClass()
- // isAnonymousClass does not work for anonymous Scala classes; additionally check by classname
- || localClass.getName().contains("$anon$") || localClass.getName().contains("$anonfun")) {
-
- ObjectStreamClass localClassDescriptor = ObjectStreamClass.lookup(localClass);
- if (localClassDescriptor != null
- && localClassDescriptor.getSerialVersionUID() != streamClassDescriptor.getSerialVersionUID()) {
- LOG.warn("Ignoring serialVersionUID mismatch for anonymous class {}; was {}, now {}.",
- streamClassDescriptor.getName(), streamClassDescriptor.getSerialVersionUID(), localClassDescriptor.getSerialVersionUID());
-
- streamClassDescriptor = localClassDescriptor;
- }
- }
-
- return streamClassDescriptor;
- }
- }
-
- /**
* Writes a {@link TypeSerializer} to the provided data output view.
*
* <p>It is written with a format that can be later read again using
@@ -447,8 +368,8 @@ public class TypeSerializerSerializationUtil {
ClassLoader previousClassLoader = Thread.currentThread().getContextClassLoader();
try (
- FailureTolerantObjectInputStream ois =
- new FailureTolerantObjectInputStream(new ByteArrayInputStream(buffer), userClassLoader)) {
+ InstantiationUtil.FailureTolerantObjectInputStream ois =
+ new InstantiationUtil.FailureTolerantObjectInputStream(new ByteArrayInputStream(buffer), userClassLoader)) {
Thread.currentThread().setContextClassLoader(userClassLoader);
typeSerializer = (TypeSerializer<T>) ois.readObject();
http://git-wip-us.apache.org/repos/asf/flink/blob/319d5101/flink-core/src/main/java/org/apache/flink/util/InstantiationUtil.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/util/InstantiationUtil.java b/flink-core/src/main/java/org/apache/flink/util/InstantiationUtil.java
index 6441c86..7ffad55 100644
--- a/flink-core/src/main/java/org/apache/flink/util/InstantiationUtil.java
+++ b/flink-core/src/main/java/org/apache/flink/util/InstantiationUtil.java
@@ -20,11 +20,16 @@ package org.apache.flink.util;
import org.apache.flink.annotation.Internal;
import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerSerializationUtil;
+import org.apache.flink.api.java.typeutils.runtime.KryoRegistrationSerializerConfigSnapshot;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.io.IOReadableWritable;
import org.apache.flink.core.memory.DataInputViewStreamWrapper;
import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@@ -37,13 +42,17 @@ import java.io.Serializable;
import java.lang.reflect.Constructor;
import java.lang.reflect.Modifier;
import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Set;
/**
* Utility class to create instances from class objects and checking failure reasons.
*/
@Internal
public final class InstantiationUtil {
-
+
+ private static final Logger LOG = LoggerFactory.getLogger(InstantiationUtil.class);
+
/**
* A custom ObjectInputStream that can load classes using a specific ClassLoader.
*/
@@ -94,8 +103,80 @@ public final class InstantiationUtil {
primitiveClasses.put("void", void.class);
}
}
-
-
+
+ /**
+ * This is maintained as a temporary workaround for FLINK-6869.
+ *
+ * <p>Before 1.3, the Scala serializers did not specify the serialVersionUID.
+ * Although since 1.3 they are properly specified, we still have to ignore them for now
+ * as their previous serialVersionUIDs will vary depending on the Scala version.
+ *
+ * <p>This can be removed once 1.2 is no longer supported.
+ */
+ private static Set<String> scalaSerializerClassnames = new HashSet<>();
+ static {
+ scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.TraversableSerializer");
+ scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.CaseClassSerializer");
+ scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.EitherSerializer");
+ scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.EnumValueSerializer");
+ scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.OptionSerializer");
+ scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.TrySerializer");
+ scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.EitherSerializer");
+ scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.UnitSerializer");
+ }
+
+ /**
+ * An {@link ObjectInputStream} that ignores serialVersionUID mismatches when deserializing objects of
+ * anonymous classes or our Scala serializer classes and also replaces occurences of GenericData.Array
+ * (from Avro) by a dummy class so that the KryoSerializer can still be deserialized without
+ * Avro being on the classpath.
+ *
+ * <p>The {@link TypeSerializerSerializationUtil.TypeSerializerSerializationProxy} uses this specific object input stream to read serializers,
+ * so that mismatching serialVersionUIDs of anonymous classes / Scala serializers are ignored.
+ * This is a required workaround to maintain backwards compatibility for our pre-1.3 Scala serializers.
+ * See FLINK-6869 for details.
+ *
+ * @see <a href="https://issues.apache.org/jira/browse/FLINK-6869">FLINK-6869</a>
+ */
+ public static class FailureTolerantObjectInputStream extends InstantiationUtil.ClassLoaderObjectInputStream {
+
+ public FailureTolerantObjectInputStream(InputStream in, ClassLoader cl) throws IOException {
+ super(in, cl);
+ }
+
+ @Override
+ protected ObjectStreamClass readClassDescriptor() throws IOException, ClassNotFoundException {
+ ObjectStreamClass streamClassDescriptor = super.readClassDescriptor();
+
+ try {
+ Class.forName(streamClassDescriptor.getName(), false, classLoader);
+ } catch (ClassNotFoundException e) {
+ if (streamClassDescriptor.getName().equals("org.apache.avro.generic.GenericData$Array")) {
+ ObjectStreamClass result = ObjectStreamClass.lookup(
+ KryoRegistrationSerializerConfigSnapshot.DummyRegisteredClass.class);
+ return result;
+ }
+ }
+
+ Class localClass = resolveClass(streamClassDescriptor);
+ if (scalaSerializerClassnames.contains(localClass.getName()) || localClass.isAnonymousClass()
+ // isAnonymousClass does not work for anonymous Scala classes; additionally check by classname
+ || localClass.getName().contains("$anon$") || localClass.getName().contains("$anonfun")) {
+
+ ObjectStreamClass localClassDescriptor = ObjectStreamClass.lookup(localClass);
+ if (localClassDescriptor != null
+ && localClassDescriptor.getSerialVersionUID() != streamClassDescriptor.getSerialVersionUID()) {
+ LOG.warn("Ignoring serialVersionUID mismatch for anonymous class {}; was {}, now {}.",
+ streamClassDescriptor.getName(), streamClassDescriptor.getSerialVersionUID(), localClassDescriptor.getSerialVersionUID());
+
+ streamClassDescriptor = localClassDescriptor;
+ }
+ }
+
+ return streamClassDescriptor;
+ }
+ }
+
/**
* Creates a new instance of the given class.
*
@@ -284,23 +365,30 @@ public final class InstantiationUtil {
@SuppressWarnings("unchecked")
public static <T> T deserializeObject(byte[] bytes, ClassLoader cl) throws IOException, ClassNotFoundException {
- final ClassLoader old = Thread.currentThread().getContextClassLoader();
- try (ObjectInputStream oois = new ClassLoaderObjectInputStream(new ByteArrayInputStream(bytes), cl)) {
- Thread.currentThread().setContextClassLoader(cl);
- return (T) oois.readObject();
- }
- finally {
- Thread.currentThread().setContextClassLoader(old);
- }
+ return deserializeObject(bytes, cl, false);
}
@SuppressWarnings("unchecked")
public static <T> T deserializeObject(InputStream in, ClassLoader cl) throws IOException, ClassNotFoundException {
+ return deserializeObject(in, cl, false);
+ }
+
+ @SuppressWarnings("unchecked")
+ public static <T> T deserializeObject(byte[] bytes, ClassLoader cl, boolean isFailureTolerant)
+ throws IOException, ClassNotFoundException {
+
+ return deserializeObject(new ByteArrayInputStream(bytes), cl, isFailureTolerant);
+ }
+
+ @SuppressWarnings("unchecked")
+ public static <T> T deserializeObject(InputStream in, ClassLoader cl, boolean isFailureTolerant)
+ throws IOException, ClassNotFoundException {
+
final ClassLoader old = Thread.currentThread().getContextClassLoader();
- ObjectInputStream oois;
// not using resource try to avoid AutoClosable's close() on the given stream
- try {
- oois = new ClassLoaderObjectInputStream(in, cl);
+ try (ObjectInputStream oois = isFailureTolerant
+ ? new InstantiationUtil.FailureTolerantObjectInputStream(in, cl)
+ : new InstantiationUtil.ClassLoaderObjectInputStream(in, cl)) {
Thread.currentThread().setContextClassLoader(cl);
return (T) oois.readObject();
}
http://git-wip-us.apache.org/repos/asf/flink/blob/319d5101/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorBackendStateMetaInfoSnapshotReaderWriters.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorBackendStateMetaInfoSnapshotReaderWriters.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorBackendStateMetaInfoSnapshotReaderWriters.java
index c333397..03fe612 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorBackendStateMetaInfoSnapshotReaderWriters.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorBackendStateMetaInfoSnapshotReaderWriters.java
@@ -25,6 +25,7 @@ import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.typeutils.runtime.DataInputViewStream;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.util.InstantiationUtil;
import org.apache.flink.util.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -168,8 +169,8 @@ public class OperatorBackendStateMetaInfoSnapshotReaderWriters {
DataInputViewStream dis = new DataInputViewStream(in);
ClassLoader previousClassLoader = Thread.currentThread().getContextClassLoader();
try (
- TypeSerializerSerializationUtil.FailureTolerantObjectInputStream ois =
- new TypeSerializerSerializationUtil.FailureTolerantObjectInputStream(dis, userCodeClassLoader)) {
+ InstantiationUtil.FailureTolerantObjectInputStream ois =
+ new InstantiationUtil.FailureTolerantObjectInputStream(dis, userCodeClassLoader)) {
Thread.currentThread().setContextClassLoader(userCodeClassLoader);
TypeSerializer<S> stateSerializer = (TypeSerializer<S>) ois.readObject();
[05/28] flink git commit: [FLINK-8409] [kafka] Fix offset committing
race condition in KafkaConsumerThread
Posted by tz...@apache.org.
[FLINK-8409] [kafka] Fix offset committing race condition in KafkaConsumerThread
This closes #5329.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/08163009
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/08163009
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/08163009
Branch: refs/heads/release-1.4
Commit: 08163009e443d00379696f9f84b3ccb0af6a25b6
Parents: c932266
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Sat Jan 13 20:13:20 2018 +0800
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 17:31:32 2018 +0100
----------------------------------------------------------------------
.../kafka/internal/KafkaConsumerThread.java | 31 ++++++++++----------
1 file changed, 16 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/08163009/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java
index fc5f359..c462b26 100644
--- a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java
+++ b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java
@@ -19,6 +19,7 @@
package org.apache.flink.streaming.connectors.kafka.internal;
import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.streaming.connectors.kafka.internals.ClosableBlockingQueue;
import org.apache.flink.streaming.connectors.kafka.internals.KafkaCommitCallback;
@@ -68,8 +69,8 @@ public class KafkaConsumerThread extends Thread {
/** The handover of data and exceptions between the consumer thread and the task thread. */
private final Handover handover;
- /** The next offsets that the main thread should commit. */
- private final AtomicReference<Map<TopicPartition, OffsetAndMetadata>> nextOffsetsToCommit;
+ /** The next offsets that the main thread should commit and the commit callback. */
+ private final AtomicReference<Tuple2<Map<TopicPartition, OffsetAndMetadata>, KafkaCommitCallback>> nextOffsetsToCommit;
/** The configuration for the Kafka consumer. */
private final Properties kafkaProperties;
@@ -110,9 +111,6 @@ public class KafkaConsumerThread extends Thread {
/** Flag tracking whether the latest commit request has completed. */
private volatile boolean commitInProgress;
- /** User callback to be invoked when commits completed. */
- private volatile KafkaCommitCallback offsetCommitCallback;
-
public KafkaConsumerThread(
Logger log,
Handover handover,
@@ -188,9 +186,6 @@ public class KafkaConsumerThread extends Thread {
return;
}
- // The callback invoked by Kafka once an offset commit is complete
- final OffsetCommitCallback offsetCommitCallback = new CommitCallback();
-
// the latest bulk of records. May carry across the loop if the thread is woken up
// from blocking on the handover
ConsumerRecords<byte[], byte[]> records = null;
@@ -206,15 +201,16 @@ public class KafkaConsumerThread extends Thread {
// check if there is something to commit
if (!commitInProgress) {
// get and reset the work-to-be committed, so we don't repeatedly commit the same
- final Map<TopicPartition, OffsetAndMetadata> toCommit = nextOffsetsToCommit.getAndSet(null);
+ final Tuple2<Map<TopicPartition, OffsetAndMetadata>, KafkaCommitCallback> commitOffsetsAndCallback =
+ nextOffsetsToCommit.getAndSet(null);
- if (toCommit != null) {
+ if (commitOffsetsAndCallback != null) {
log.debug("Sending async offset commit request to Kafka broker");
// also record that a commit is already in progress
// the order here matters! first set the flag, then send the commit command.
commitInProgress = true;
- consumer.commitAsync(toCommit, offsetCommitCallback);
+ consumer.commitAsync(commitOffsetsAndCallback.f0, new CommitCallback(commitOffsetsAndCallback.f1));
}
}
@@ -324,12 +320,11 @@ public class KafkaConsumerThread extends Thread {
@Nonnull KafkaCommitCallback commitCallback) {
// record the work to be committed by the main consumer thread and make sure the consumer notices that
- if (nextOffsetsToCommit.getAndSet(offsetsToCommit) != null) {
+ if (nextOffsetsToCommit.getAndSet(Tuple2.of(offsetsToCommit, commitCallback)) != null) {
log.warn("Committing offsets to Kafka takes longer than the checkpoint interval. " +
"Skipping commit of previous offsets because newer complete checkpoint offsets are available. " +
"This does not compromise Flink's checkpoint integrity.");
}
- this.offsetCommitCallback = commitCallback;
// if the consumer is blocked in a poll() or handover operation, wake it up to commit soon
handover.wakeupProducer();
@@ -487,15 +482,21 @@ public class KafkaConsumerThread extends Thread {
private class CommitCallback implements OffsetCommitCallback {
+ private final KafkaCommitCallback internalCommitCallback;
+
+ CommitCallback(KafkaCommitCallback internalCommitCallback) {
+ this.internalCommitCallback = checkNotNull(internalCommitCallback);
+ }
+
@Override
public void onComplete(Map<TopicPartition, OffsetAndMetadata> offsets, Exception ex) {
commitInProgress = false;
if (ex != null) {
log.warn("Committing offsets to Kafka failed. This does not compromise Flink's checkpoints.", ex);
- offsetCommitCallback.onException(ex);
+ internalCommitCallback.onException(ex);
} else {
- offsetCommitCallback.onSuccess();
+ internalCommitCallback.onSuccess();
}
}
}
[22/28] flink git commit: [hotfix] [kinesis] Use ArrayList for faster
shard state updates
Posted by tz...@apache.org.
[hotfix] [kinesis] Use ArrayList for faster shard state updates
Previously, the Kinesis Consumer uses LinkedLists as the underlying
implementation for subscribedShardsState. This list is accessed on every
record, updating a shard's state using a shard state index (i.e., the
order of the shard state in the list). With LinkedLists, this access has
linear time complexity, and since this operation happens per record, can
have a big performance hit on executions with a huge number of Kinesis
shards.
This commit changes the list implementation to be an ArrayList for
constant-time shard state access. The downside is that when new shards
are discovered, there will be a costly array re-size. However, since
resharding is not expected to happen so often, this is acceptable.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/9276ee51
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/9276ee51
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/9276ee51
Branch: refs/heads/release-1.4
Commit: 9276ee51aa3d671095ddc8e1335e58f03700db39
Parents: 7884a4f
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Thu Feb 1 13:05:56 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 17:31:32 2018 +0100
----------------------------------------------------------------------
.../connectors/kinesis/internals/KinesisDataFetcher.java | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/9276ee51/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
index 79e4bfd..4634421 100644
--- a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
+++ b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
@@ -39,6 +39,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@@ -178,8 +179,8 @@ public class KinesisDataFetcher<T> {
runtimeContext,
configProps,
deserializationSchema,
- new AtomicReference<Throwable>(),
- new LinkedList<KinesisStreamShardState>(),
+ new AtomicReference<>(),
+ new ArrayList<>(),
createInitialSubscribedStreamsToLastDiscoveredShardsState(streams),
KinesisProxy.create(configProps));
}
@@ -192,7 +193,7 @@ public class KinesisDataFetcher<T> {
Properties configProps,
KinesisDeserializationSchema<T> deserializationSchema,
AtomicReference<Throwable> error,
- LinkedList<KinesisStreamShardState> subscribedShardsState,
+ List<KinesisStreamShardState> subscribedShardsState,
HashMap<String, String> subscribedStreamsToLastDiscoveredShardIds,
KinesisProxyInterface kinesis) {
this.streams = checkNotNull(streams);
[12/28] flink git commit: [FLINK-8472] [kafka,
test] Extend FlinkKafkaConsumerBaseMigrationTest for Flink 1.4
Posted by tz...@apache.org.
[FLINK-8472] [kafka, test] Extend FlinkKafkaConsumerBaseMigrationTest for Flink 1.4
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/da47bd2a
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/da47bd2a
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/da47bd2a
Branch: refs/heads/release-1.4
Commit: da47bd2ac83c8866cbeac8f0ff6a09bb79b546f9
Parents: 8cef788
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Thu Jan 25 17:23:22 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 17:31:32 2018 +0100
----------------------------------------------------------------------
.../kafka/FlinkKafkaConsumerBaseMigrationTest.java | 2 +-
...mer-migration-test-flink1.4-empty-state-snapshot | Bin 0 -> 6471 bytes
.../kafka-consumer-migration-test-flink1.4-snapshot | Bin 0 -> 6525 bytes
3 files changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/da47bd2a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java
index a533cca..1980ccc 100644
--- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java
+++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java
@@ -93,7 +93,7 @@ public class FlinkKafkaConsumerBaseMigrationTest {
@Parameterized.Parameters(name = "Migration Savepoint: {0}")
public static Collection<MigrationVersion> parameters () {
- return Arrays.asList(MigrationVersion.v1_2, MigrationVersion.v1_3);
+ return Arrays.asList(MigrationVersion.v1_2, MigrationVersion.v1_3, MigrationVersion.v1_4);
}
public FlinkKafkaConsumerBaseMigrationTest(MigrationVersion testMigrateVersion) {
http://git-wip-us.apache.org/repos/asf/flink/blob/da47bd2a/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.4-empty-state-snapshot
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.4-empty-state-snapshot b/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.4-empty-state-snapshot
new file mode 100644
index 0000000..2a06fb7
Binary files /dev/null and b/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.4-empty-state-snapshot differ
http://git-wip-us.apache.org/repos/asf/flink/blob/da47bd2a/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.4-snapshot
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.4-snapshot b/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.4-snapshot
new file mode 100644
index 0000000..d778ca0
Binary files /dev/null and b/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.4-snapshot differ
[07/28] flink git commit: [FLINK-8419] [kafka] Move consumer metric
group registration to FlinkKafkaConsumerBase
Posted by tz...@apache.org.
[FLINK-8419] [kafka] Move consumer metric group registration to FlinkKafkaConsumerBase
This commit is a refactor to move the registration of the consumer
metric group (user scope "KafkaConsumer") to FlinkKafkaConsumerBase.
Previously, the registration was scattered around in Kafka
version-specific subclasses.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/e5ac4b08
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/e5ac4b08
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/e5ac4b08
Branch: refs/heads/release-1.4
Commit: e5ac4b08f75e785f257d0fe5010377000e5a0185
Parents: a951852
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Mon Jan 22 13:36:20 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 17:31:32 2018 +0100
----------------------------------------------------------------------
.../connectors/kafka/FlinkKafkaConsumer010.java | 11 +++++-----
.../kafka/internal/Kafka010Fetcher.java | 6 ++++--
.../kafka/internal/Kafka010FetcherTest.java | 9 ++++++---
.../connectors/kafka/FlinkKafkaConsumer08.java | 12 +++++++----
.../kafka/internals/Kafka08Fetcher.java | 8 ++------
.../connectors/kafka/FlinkKafkaConsumer09.java | 16 +++++++++------
.../kafka/internal/Kafka09Fetcher.java | 13 ++++++------
.../kafka/internal/KafkaConsumerThread.java | 14 ++++++-------
.../kafka/internal/Kafka09FetcherTest.java | 9 ++++++---
.../kafka/internal/KafkaConsumerThreadTest.java | 6 +++---
.../kafka/FlinkKafkaConsumerBase.java | 21 +++++++++++++++++---
.../kafka/internals/AbstractFetcher.java | 9 +++++----
.../FlinkKafkaConsumerBaseMigrationTest.java | 8 ++++++--
.../kafka/FlinkKafkaConsumerBaseTest.java | 8 ++++++--
.../kafka/internals/AbstractFetcherTest.java | 2 ++
15 files changed, 95 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/e5ac4b08/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer010.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer010.java b/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer010.java
index 6fb63e1..b102ee0 100644
--- a/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer010.java
+++ b/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer010.java
@@ -19,6 +19,7 @@ package org.apache.flink.streaming.connectors.kafka;
import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.streaming.api.functions.AssignerWithPeriodicWatermarks;
import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks;
import org.apache.flink.streaming.api.operators.StreamingRuntimeContext;
@@ -31,7 +32,6 @@ import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition
import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicsDescriptor;
import org.apache.flink.streaming.util.serialization.KeyedDeserializationSchema;
import org.apache.flink.streaming.util.serialization.KeyedDeserializationSchemaWrapper;
-import org.apache.flink.util.PropertiesUtil;
import org.apache.flink.util.SerializedValue;
import org.apache.kafka.clients.consumer.ConsumerConfig;
@@ -178,9 +178,9 @@ public class FlinkKafkaConsumer010<T> extends FlinkKafkaConsumer09<T> {
SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
StreamingRuntimeContext runtimeContext,
- OffsetCommitMode offsetCommitMode) throws Exception {
-
- boolean useMetrics = !PropertiesUtil.getBoolean(properties, KEY_DISABLE_METRICS, false);
+ OffsetCommitMode offsetCommitMode,
+ MetricGroup consumerMetricGroup,
+ boolean useMetrics) throws Exception {
// make sure that auto commit is disabled when our offset commit mode is ON_CHECKPOINTS;
// this overwrites whatever setting the user configured in the properties
@@ -197,10 +197,11 @@ public class FlinkKafkaConsumer010<T> extends FlinkKafkaConsumer09<T> {
runtimeContext.getExecutionConfig().getAutoWatermarkInterval(),
runtimeContext.getUserCodeClassLoader(),
runtimeContext.getTaskNameWithSubtasks(),
- runtimeContext.getMetricGroup(),
deserializer,
properties,
pollTimeout,
+ runtimeContext.getMetricGroup(),
+ consumerMetricGroup,
useMetrics);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e5ac4b08/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010Fetcher.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010Fetcher.java b/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010Fetcher.java
index eb4dfee..c2639ca 100644
--- a/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010Fetcher.java
+++ b/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010Fetcher.java
@@ -53,10 +53,11 @@ public class Kafka010Fetcher<T> extends Kafka09Fetcher<T> {
long autoWatermarkInterval,
ClassLoader userCodeClassLoader,
String taskNameWithSubtasks,
- MetricGroup metricGroup,
KeyedDeserializationSchema<T> deserializer,
Properties kafkaProperties,
long pollTimeout,
+ MetricGroup subtaskMetricGroup,
+ MetricGroup consumerMetricGroup,
boolean useMetrics) throws Exception {
super(
sourceContext,
@@ -67,10 +68,11 @@ public class Kafka010Fetcher<T> extends Kafka09Fetcher<T> {
autoWatermarkInterval,
userCodeClassLoader,
taskNameWithSubtasks,
- metricGroup,
deserializer,
kafkaProperties,
pollTimeout,
+ subtaskMetricGroup,
+ consumerMetricGroup,
useMetrics);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e5ac4b08/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010FetcherTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010FetcherTest.java b/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010FetcherTest.java
index 45ceadc..f57fbea 100644
--- a/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010FetcherTest.java
+++ b/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010FetcherTest.java
@@ -126,10 +126,11 @@ public class Kafka010FetcherTest {
10,
getClass().getClassLoader(),
"taskname-with-subtask",
- new UnregisteredMetricsGroup(),
schema,
new Properties(),
0L,
+ new UnregisteredMetricsGroup(),
+ new UnregisteredMetricsGroup(),
false);
// ----- run the fetcher -----
@@ -262,10 +263,11 @@ public class Kafka010FetcherTest {
10,
getClass().getClassLoader(),
"taskname-with-subtask",
- new UnregisteredMetricsGroup(),
schema,
new Properties(),
0L,
+ new UnregisteredMetricsGroup(),
+ new UnregisteredMetricsGroup(),
false);
// ----- run the fetcher -----
@@ -376,10 +378,11 @@ public class Kafka010FetcherTest {
10, /* watermark interval */
this.getClass().getClassLoader(),
"task_name",
- new UnregisteredMetricsGroup(),
schema,
new Properties(),
0L,
+ new UnregisteredMetricsGroup(),
+ new UnregisteredMetricsGroup(),
false);
// ----- run the fetcher -----
http://git-wip-us.apache.org/repos/asf/flink/blob/e5ac4b08/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer08.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer08.java b/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer08.java
index f362046..af2efd1 100644
--- a/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer08.java
+++ b/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer08.java
@@ -19,6 +19,7 @@ package org.apache.flink.streaming.connectors.kafka;
import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.streaming.api.functions.AssignerWithPeriodicWatermarks;
import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks;
import org.apache.flink.streaming.api.operators.StreamingRuntimeContext;
@@ -43,6 +44,7 @@ import java.util.Properties;
import java.util.regex.Pattern;
import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.apache.flink.util.PropertiesUtil.getBoolean;
import static org.apache.flink.util.PropertiesUtil.getLong;
/**
@@ -216,7 +218,8 @@ public class FlinkKafkaConsumer08<T> extends FlinkKafkaConsumerBase<T> {
deserializer,
getLong(
checkNotNull(props, "props"),
- KEY_PARTITION_DISCOVERY_INTERVAL_MILLIS, PARTITION_DISCOVERY_DISABLED));
+ KEY_PARTITION_DISCOVERY_INTERVAL_MILLIS, PARTITION_DISCOVERY_DISABLED),
+ !getBoolean(props, KEY_DISABLE_METRICS, false));
this.kafkaProperties = props;
@@ -234,9 +237,9 @@ public class FlinkKafkaConsumer08<T> extends FlinkKafkaConsumerBase<T> {
SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
StreamingRuntimeContext runtimeContext,
- OffsetCommitMode offsetCommitMode) throws Exception {
-
- boolean useMetrics = !PropertiesUtil.getBoolean(kafkaProperties, KEY_DISABLE_METRICS, false);
+ OffsetCommitMode offsetCommitMode,
+ MetricGroup consumerMetricGroup,
+ boolean useMetrics) throws Exception {
long autoCommitInterval = (offsetCommitMode == OffsetCommitMode.KAFKA_PERIODIC)
? PropertiesUtil.getLong(kafkaProperties, "auto.commit.interval.ms", 60000)
@@ -251,6 +254,7 @@ public class FlinkKafkaConsumer08<T> extends FlinkKafkaConsumerBase<T> {
deserializer,
kafkaProperties,
autoCommitInterval,
+ consumerMetricGroup,
useMetrics);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e5ac4b08/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/Kafka08Fetcher.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/Kafka08Fetcher.java b/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/Kafka08Fetcher.java
index 8bcd663..d8bdd22 100644
--- a/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/Kafka08Fetcher.java
+++ b/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/Kafka08Fetcher.java
@@ -95,6 +95,7 @@ public class Kafka08Fetcher<T> extends AbstractFetcher<T, TopicAndPartition> {
KeyedDeserializationSchema<T> deserializer,
Properties kafkaProperties,
long autoCommitInterval,
+ MetricGroup consumerMetricGroup,
boolean useMetrics) throws Exception {
super(
sourceContext,
@@ -104,6 +105,7 @@ public class Kafka08Fetcher<T> extends AbstractFetcher<T, TopicAndPartition> {
runtimeContext.getProcessingTimeService(),
runtimeContext.getExecutionConfig().getAutoWatermarkInterval(),
runtimeContext.getUserCodeClassLoader(),
+ consumerMetricGroup,
useMetrics);
this.deserializer = checkNotNull(deserializer);
@@ -175,12 +177,6 @@ public class Kafka08Fetcher<T> extends AbstractFetcher<T, TopicAndPartition> {
periodicCommitter.start();
}
- // register offset metrics
- if (useMetrics) {
- final MetricGroup kafkaMetricGroup = runtimeContext.getMetricGroup().addGroup("KafkaConsumer");
- addOffsetStateGauge(kafkaMetricGroup);
- }
-
// Main loop polling elements from the unassignedPartitions queue to the threads
while (running) {
// re-throw any exception from the concurrent fetcher threads
http://git-wip-us.apache.org/repos/asf/flink/blob/e5ac4b08/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer09.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer09.java b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer09.java
index 79be73c..fe75006 100644
--- a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer09.java
+++ b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer09.java
@@ -19,6 +19,7 @@ package org.apache.flink.streaming.connectors.kafka;
import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.streaming.api.functions.AssignerWithPeriodicWatermarks;
import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks;
import org.apache.flink.streaming.api.operators.StreamingRuntimeContext;
@@ -46,6 +47,7 @@ import java.util.Properties;
import java.util.regex.Pattern;
import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.apache.flink.util.PropertiesUtil.getBoolean;
import static org.apache.flink.util.PropertiesUtil.getLong;
/**
@@ -207,7 +209,8 @@ public class FlinkKafkaConsumer09<T> extends FlinkKafkaConsumerBase<T> {
deserializer,
getLong(
checkNotNull(props, "props"),
- KEY_PARTITION_DISCOVERY_INTERVAL_MILLIS, PARTITION_DISCOVERY_DISABLED));
+ KEY_PARTITION_DISCOVERY_INTERVAL_MILLIS, PARTITION_DISCOVERY_DISABLED),
+ !getBoolean(props, KEY_DISABLE_METRICS, false));
this.properties = props;
setDeserializer(this.properties);
@@ -232,9 +235,9 @@ public class FlinkKafkaConsumer09<T> extends FlinkKafkaConsumerBase<T> {
SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
StreamingRuntimeContext runtimeContext,
- OffsetCommitMode offsetCommitMode) throws Exception {
-
- boolean useMetrics = !PropertiesUtil.getBoolean(properties, KEY_DISABLE_METRICS, false);
+ OffsetCommitMode offsetCommitMode,
+ MetricGroup consumerMetricGroup,
+ boolean useMetrics) throws Exception {
// make sure that auto commit is disabled when our offset commit mode is ON_CHECKPOINTS;
// this overwrites whatever setting the user configured in the properties
@@ -251,10 +254,11 @@ public class FlinkKafkaConsumer09<T> extends FlinkKafkaConsumerBase<T> {
runtimeContext.getExecutionConfig().getAutoWatermarkInterval(),
runtimeContext.getUserCodeClassLoader(),
runtimeContext.getTaskNameWithSubtasks(),
- runtimeContext.getMetricGroup(),
deserializer,
properties,
pollTimeout,
+ runtimeContext.getMetricGroup(),
+ consumerMetricGroup,
useMetrics);
}
@@ -269,7 +273,7 @@ public class FlinkKafkaConsumer09<T> extends FlinkKafkaConsumerBase<T> {
@Override
protected boolean getIsAutoCommitEnabled() {
- return PropertiesUtil.getBoolean(properties, ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, true) &&
+ return getBoolean(properties, ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, true) &&
PropertiesUtil.getLong(properties, ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, 5000) > 0;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e5ac4b08/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09Fetcher.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09Fetcher.java b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09Fetcher.java
index 51f69cd..aca0da1 100644
--- a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09Fetcher.java
+++ b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09Fetcher.java
@@ -80,10 +80,11 @@ public class Kafka09Fetcher<T> extends AbstractFetcher<T, TopicPartition> {
long autoWatermarkInterval,
ClassLoader userCodeClassLoader,
String taskNameWithSubtasks,
- MetricGroup metricGroup,
KeyedDeserializationSchema<T> deserializer,
Properties kafkaProperties,
long pollTimeout,
+ MetricGroup subtaskMetricGroup,
+ MetricGroup consumerMetricGroup,
boolean useMetrics) throws Exception {
super(
sourceContext,
@@ -92,25 +93,23 @@ public class Kafka09Fetcher<T> extends AbstractFetcher<T, TopicPartition> {
watermarksPunctuated,
processingTimeProvider,
autoWatermarkInterval,
- userCodeClassLoader,
+ userCodeClassLoader.getParent(),
+ consumerMetricGroup,
useMetrics);
this.deserializer = deserializer;
this.handover = new Handover();
- final MetricGroup kafkaMetricGroup = metricGroup.addGroup("KafkaConsumer");
- addOffsetStateGauge(kafkaMetricGroup);
-
this.consumerThread = new KafkaConsumerThread(
LOG,
handover,
kafkaProperties,
unassignedPartitionsQueue,
- kafkaMetricGroup,
createCallBridge(),
getFetcherName() + " for " + taskNameWithSubtasks,
pollTimeout,
- useMetrics);
+ useMetrics,
+ subtaskMetricGroup);
}
// ------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/e5ac4b08/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java
index ee0a63b..e21bd5f 100644
--- a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java
+++ b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java
@@ -78,9 +78,6 @@ public class KafkaConsumerThread extends Thread {
/** The queue of unassigned partitions that we need to assign to the Kafka consumer. */
private final ClosableBlockingQueue<KafkaTopicPartitionState<TopicPartition>> unassignedPartitionsQueue;
- /** We get this from the outside to publish metrics. **/
- private final MetricGroup kafkaMetricGroup;
-
/** The indirections on KafkaConsumer methods, for cases where KafkaConsumer compatibility is broken. */
private final KafkaConsumerCallBridge consumerCallBridge;
@@ -90,6 +87,9 @@ public class KafkaConsumerThread extends Thread {
/** Flag whether to add Kafka's metrics to the Flink metrics. */
private final boolean useMetrics;
+ /** We get this from the outside to publish metrics. */
+ private final MetricGroup subtaskMetricGroup;
+
/** Reference to the Kafka consumer, once it is created. */
private volatile KafkaConsumer<byte[], byte[]> consumer;
@@ -116,11 +116,11 @@ public class KafkaConsumerThread extends Thread {
Handover handover,
Properties kafkaProperties,
ClosableBlockingQueue<KafkaTopicPartitionState<TopicPartition>> unassignedPartitionsQueue,
- MetricGroup kafkaMetricGroup,
KafkaConsumerCallBridge consumerCallBridge,
String threadName,
long pollTimeout,
- boolean useMetrics) {
+ boolean useMetrics,
+ MetricGroup subtaskMetricGroup) {
super(threadName);
setDaemon(true);
@@ -128,7 +128,7 @@ public class KafkaConsumerThread extends Thread {
this.log = checkNotNull(log);
this.handover = checkNotNull(handover);
this.kafkaProperties = checkNotNull(kafkaProperties);
- this.kafkaMetricGroup = checkNotNull(kafkaMetricGroup);
+ this.subtaskMetricGroup = checkNotNull(subtaskMetricGroup);
this.consumerCallBridge = checkNotNull(consumerCallBridge);
this.unassignedPartitionsQueue = checkNotNull(unassignedPartitionsQueue);
@@ -176,7 +176,7 @@ public class KafkaConsumerThread extends Thread {
} else {
// we have Kafka metrics, register them
for (Map.Entry<MetricName, ? extends Metric> metric: metrics.entrySet()) {
- kafkaMetricGroup.gauge(metric.getKey().name(), new KafkaMetricWrapper(metric.getValue()));
+ subtaskMetricGroup.gauge(metric.getKey().name(), new KafkaMetricWrapper(metric.getValue()));
}
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e5ac4b08/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09FetcherTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09FetcherTest.java b/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09FetcherTest.java
index e4e276a..27b67f1 100644
--- a/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09FetcherTest.java
+++ b/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09FetcherTest.java
@@ -126,10 +126,11 @@ public class Kafka09FetcherTest {
10, /* watermark interval */
this.getClass().getClassLoader(),
"task_name",
- new UnregisteredMetricsGroup(),
schema,
new Properties(),
0L,
+ new UnregisteredMetricsGroup(),
+ new UnregisteredMetricsGroup(),
false);
// ----- run the fetcher -----
@@ -261,10 +262,11 @@ public class Kafka09FetcherTest {
10, /* watermark interval */
this.getClass().getClassLoader(),
"task_name",
- new UnregisteredMetricsGroup(),
schema,
new Properties(),
0L,
+ new UnregisteredMetricsGroup(),
+ new UnregisteredMetricsGroup(),
false);
// ----- run the fetcher -----
@@ -375,10 +377,11 @@ public class Kafka09FetcherTest {
10, /* watermark interval */
this.getClass().getClassLoader(),
"task_name",
- new UnregisteredMetricsGroup(),
schema,
new Properties(),
0L,
+ new UnregisteredMetricsGroup(),
+ new UnregisteredMetricsGroup(),
false);
// ----- run the fetcher -----
http://git-wip-us.apache.org/repos/asf/flink/blob/e5ac4b08/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThreadTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThreadTest.java b/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThreadTest.java
index 2368091..607a1a9 100644
--- a/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThreadTest.java
+++ b/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThreadTest.java
@@ -20,7 +20,7 @@ package org.apache.flink.streaming.connectors.kafka.internal;
import org.apache.flink.core.testutils.MultiShotLatch;
import org.apache.flink.core.testutils.OneShotLatch;
-import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
import org.apache.flink.streaming.connectors.kafka.internals.ClosableBlockingQueue;
import org.apache.flink.streaming.connectors.kafka.internals.KafkaCommitCallback;
import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition;
@@ -716,11 +716,11 @@ public class KafkaConsumerThreadTest {
handover,
new Properties(),
unassignedPartitionsQueue,
- mock(MetricGroup.class),
new KafkaConsumerCallBridge(),
"test-kafka-consumer-thread",
0,
- false);
+ false,
+ new UnregisteredMetricsGroup());
this.mockConsumer = mockConsumer;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e5ac4b08/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java
index d71827f..2645ddc 100644
--- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java
+++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java
@@ -28,6 +28,7 @@ import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.metrics.Counter;
+import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.runtime.state.CheckpointListener;
import org.apache.flink.runtime.state.DefaultOperatorStateBackend;
import org.apache.flink.runtime.state.FunctionInitializationContext;
@@ -188,6 +189,13 @@ public abstract class FlinkKafkaConsumerBase<T> extends RichParallelSourceFuncti
// internal metrics
// ------------------------------------------------------------------------
+ /**
+ * Flag indicating whether or not metrics should be exposed.
+ * If {@code true}, offset metrics (e.g. current offset, committed offset) and
+ * Kafka-shipped metrics will be registered.
+ */
+ private final boolean useMetrics;
+
/** Counter for successful Kafka offset commits. */
private transient Counter successfulCommits;
@@ -217,7 +225,8 @@ public abstract class FlinkKafkaConsumerBase<T> extends RichParallelSourceFuncti
List<String> topics,
Pattern topicPattern,
KeyedDeserializationSchema<T> deserializer,
- long discoveryIntervalMillis) {
+ long discoveryIntervalMillis,
+ boolean useMetrics) {
this.topicsDescriptor = new KafkaTopicsDescriptor(topics, topicPattern);
this.deserializer = checkNotNull(deserializer, "valueDeserializer");
@@ -225,6 +234,8 @@ public abstract class FlinkKafkaConsumerBase<T> extends RichParallelSourceFuncti
discoveryIntervalMillis == PARTITION_DISCOVERY_DISABLED || discoveryIntervalMillis >= 0,
"Cannot define a negative value for the topic / partition discovery interval.");
this.discoveryIntervalMillis = discoveryIntervalMillis;
+
+ this.useMetrics = useMetrics;
}
// ------------------------------------------------------------------------
@@ -556,7 +567,9 @@ public abstract class FlinkKafkaConsumerBase<T> extends RichParallelSourceFuncti
periodicWatermarkAssigner,
punctuatedWatermarkAssigner,
(StreamingRuntimeContext) getRuntimeContext(),
- offsetCommitMode);
+ offsetCommitMode,
+ getRuntimeContext().getMetricGroup().addGroup("KafkaConsumer"),
+ useMetrics);
if (!running) {
return;
@@ -835,7 +848,9 @@ public abstract class FlinkKafkaConsumerBase<T> extends RichParallelSourceFuncti
SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
StreamingRuntimeContext runtimeContext,
- OffsetCommitMode offsetCommitMode) throws Exception;
+ OffsetCommitMode offsetCommitMode,
+ MetricGroup kafkaMetricGroup,
+ boolean useMetrics) throws Exception;
/**
* Creates the partition discoverer that is used to find new partitions for this subtask.
http://git-wip-us.apache.org/repos/asf/flink/blob/e5ac4b08/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcher.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcher.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcher.java
index 5240326..441e660 100644
--- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcher.java
+++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcher.java
@@ -83,9 +83,6 @@ public abstract class AbstractFetcher<T, KPH> {
/** The mode describing whether the fetcher also generates timestamps and watermarks. */
protected final int timestampWatermarkMode;
- /** Flag whether to register metrics for the fetcher. */
- protected final boolean useMetrics;
-
/**
* Optional timestamp extractor / watermark generator that will be run per Kafka partition,
* to exploit per-partition timestamp characteristics.
@@ -116,10 +113,10 @@ public abstract class AbstractFetcher<T, KPH> {
ProcessingTimeService processingTimeProvider,
long autoWatermarkInterval,
ClassLoader userCodeClassLoader,
+ MetricGroup consumerMetricGroup,
boolean useMetrics) throws Exception {
this.sourceContext = checkNotNull(sourceContext);
this.checkpointLock = sourceContext.getCheckpointLock();
- this.useMetrics = useMetrics;
this.userCodeClassLoader = checkNotNull(userCodeClassLoader);
// figure out what we watermark mode we will be using
@@ -163,6 +160,10 @@ public abstract class AbstractFetcher<T, KPH> {
unassignedPartitionsQueue.add(partition);
}
+ if (useMetrics) {
+ addOffsetStateGauge(checkNotNull(consumerMetricGroup));
+ }
+
// if we have periodic watermarks, kick off the interval scheduler
if (timestampWatermarkMode == PERIODIC_WATERMARKS) {
@SuppressWarnings("unchecked")
http://git-wip-us.apache.org/repos/asf/flink/blob/e5ac4b08/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java
index 84f0e38..a533cca 100644
--- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java
+++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java
@@ -19,6 +19,7 @@
package org.apache.flink.streaming.connectors.kafka;
import org.apache.flink.core.testutils.OneShotLatch;
+import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.functions.AssignerWithPeriodicWatermarks;
import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks;
@@ -376,7 +377,8 @@ public class FlinkKafkaConsumerBaseMigrationTest {
Arrays.asList("dummy-topic"),
null,
(KeyedDeserializationSchema< T >) mock(KeyedDeserializationSchema.class),
- discoveryInterval);
+ discoveryInterval,
+ false);
this.fetcher = fetcher;
this.partitions = partitions;
@@ -393,7 +395,9 @@ public class FlinkKafkaConsumerBaseMigrationTest {
SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
StreamingRuntimeContext runtimeContext,
- OffsetCommitMode offsetCommitMode) throws Exception {
+ OffsetCommitMode offsetCommitMode,
+ MetricGroup consumerMetricGroup,
+ boolean useMetrics) throws Exception {
return fetcher;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e5ac4b08/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseTest.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseTest.java
index f091c08..b4ee378 100644
--- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseTest.java
+++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseTest.java
@@ -639,7 +639,8 @@ public class FlinkKafkaConsumerBaseTest {
Collections.singletonList("dummy-topic"),
null,
(KeyedDeserializationSchema < T >) mock(KeyedDeserializationSchema.class),
- PARTITION_DISCOVERY_DISABLED);
+ PARTITION_DISCOVERY_DISABLED,
+ false);
this.testFetcher = testFetcher;
this.testPartitionDiscoverer = testPartitionDiscoverer;
@@ -654,7 +655,9 @@ public class FlinkKafkaConsumerBaseTest {
SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
StreamingRuntimeContext runtimeContext,
- OffsetCommitMode offsetCommitMode) throws Exception {
+ OffsetCommitMode offsetCommitMode,
+ MetricGroup consumerMetricGroup,
+ boolean useMetrics) throws Exception {
return this.testFetcher;
}
@@ -737,6 +740,7 @@ public class FlinkKafkaConsumerBaseTest {
new TestProcessingTimeService(),
0,
MockFetcher.class.getClassLoader(),
+ new UnregisteredMetricsGroup(),
false);
this.stateSnapshotsToReturn.addAll(Arrays.asList(stateSnapshotsToReturn));
http://git-wip-us.apache.org/repos/asf/flink/blob/e5ac4b08/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcherTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcherTest.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcherTest.java
index 6fe1d6f..d276cfb 100644
--- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcherTest.java
+++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcherTest.java
@@ -18,6 +18,7 @@
package org.apache.flink.streaming.connectors.kafka.internals;
+import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
import org.apache.flink.streaming.api.functions.AssignerWithPeriodicWatermarks;
import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks;
import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext;
@@ -411,6 +412,7 @@ public class AbstractFetcherTest {
processingTimeProvider,
autoWatermarkInterval,
TestFetcher.class.getClassLoader(),
+ new UnregisteredMetricsGroup(),
false);
}
[16/28] flink git commit: [FLINK-8472] [DataStream,
test] Extend WindowOperatorMigrationTest for Flink 1.4
Posted by tz...@apache.org.
[FLINK-8472] [DataStream, test] Extend WindowOperatorMigrationTest for Flink 1.4
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/4af0b599
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/4af0b599
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/4af0b599
Branch: refs/heads/release-1.4
Commit: 4af0b59968eef6957ada307b9ba356d650585a7b
Parents: 9276ee5
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Thu Jan 25 16:08:53 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 17:31:32 2018 +0100
----------------------------------------------------------------------
.../windowing/WindowOperatorMigrationTest.java | 2 +-
.../streaming/util/migration/MigrationVersion.java | 3 ++-
...igration-test-apply-event-time-flink1.4-snapshot | Bin 0 -> 4630 bytes
...ion-test-apply-processing-time-flink1.4-snapshot | Bin 0 -> 4522 bytes
...gration-test-reduce-event-time-flink1.4-snapshot | Bin 0 -> 3497 bytes
...on-test-reduce-processing-time-flink1.4-snapshot | Bin 0 -> 3438 bytes
...-session-with-stateful-trigger-flink1.4-snapshot | Bin 0 -> 8680 bytes
...ion-with-stateful-trigger-mint-flink1.4-snapshot | Bin 0 -> 7651 bytes
8 files changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/4af0b599/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java
index d7df479..5b75d46 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java
@@ -83,7 +83,7 @@ public class WindowOperatorMigrationTest {
@Parameterized.Parameters(name = "Migration Savepoint: {0}")
public static Collection<MigrationVersion> parameters () {
- return Arrays.asList(MigrationVersion.v1_2, MigrationVersion.v1_3);
+ return Arrays.asList(MigrationVersion.v1_2, MigrationVersion.v1_3, MigrationVersion.v1_4);
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/4af0b599/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/migration/MigrationVersion.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/migration/MigrationVersion.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/migration/MigrationVersion.java
index 3e7998d..2a1eb98 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/migration/MigrationVersion.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/migration/MigrationVersion.java
@@ -28,7 +28,8 @@ public enum MigrationVersion {
// as they are used to locate snapshot file paths
v1_1("1.1"),
v1_2("1.2"),
- v1_3("1.3");
+ v1_3("1.3"),
+ v1_4("1.4");
private String versionStr;
http://git-wip-us.apache.org/repos/asf/flink/blob/4af0b599/flink-streaming-java/src/test/resources/win-op-migration-test-apply-event-time-flink1.4-snapshot
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-apply-event-time-flink1.4-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-apply-event-time-flink1.4-snapshot
new file mode 100644
index 0000000..5caddc2
Binary files /dev/null and b/flink-streaming-java/src/test/resources/win-op-migration-test-apply-event-time-flink1.4-snapshot differ
http://git-wip-us.apache.org/repos/asf/flink/blob/4af0b599/flink-streaming-java/src/test/resources/win-op-migration-test-apply-processing-time-flink1.4-snapshot
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-apply-processing-time-flink1.4-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-apply-processing-time-flink1.4-snapshot
new file mode 100644
index 0000000..a2c1148
Binary files /dev/null and b/flink-streaming-java/src/test/resources/win-op-migration-test-apply-processing-time-flink1.4-snapshot differ
http://git-wip-us.apache.org/repos/asf/flink/blob/4af0b599/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-event-time-flink1.4-snapshot
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-event-time-flink1.4-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-event-time-flink1.4-snapshot
new file mode 100644
index 0000000..b59d9ca
Binary files /dev/null and b/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-event-time-flink1.4-snapshot differ
http://git-wip-us.apache.org/repos/asf/flink/blob/4af0b599/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-processing-time-flink1.4-snapshot
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-processing-time-flink1.4-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-processing-time-flink1.4-snapshot
new file mode 100644
index 0000000..94909fc
Binary files /dev/null and b/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-processing-time-flink1.4-snapshot differ
http://git-wip-us.apache.org/repos/asf/flink/blob/4af0b599/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-flink1.4-snapshot
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-flink1.4-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-flink1.4-snapshot
new file mode 100644
index 0000000..9337718
Binary files /dev/null and b/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-flink1.4-snapshot differ
http://git-wip-us.apache.org/repos/asf/flink/blob/4af0b599/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-mint-flink1.4-snapshot
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-mint-flink1.4-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-mint-flink1.4-snapshot
new file mode 100644
index 0000000..ea48bdd
Binary files /dev/null and b/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-mint-flink1.4-snapshot differ
[20/28] flink git commit: [hotfix] [kinesis,
test] Use real test classes instead of Mockito in
FlinkKinesisConsumerMigrationTest
Posted by tz...@apache.org.
[hotfix] [kinesis, test] Use real test classes instead of Mockito in FlinkKinesisConsumerMigrationTest
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/f887575f
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/f887575f
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/f887575f
Branch: refs/heads/release-1.4
Commit: f887575f88770cfc0980a4aa0b48fa0bb75a6dd1
Parents: 426cf46
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Thu Feb 1 11:11:48 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 17:31:32 2018 +0100
----------------------------------------------------------------------
.../FlinkKinesisConsumerMigrationTest.java | 163 ++++++++++++++++---
1 file changed, 136 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/f887575f/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java
index 364560c..46010d1 100644
--- a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java
+++ b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java
@@ -25,24 +25,31 @@ import org.apache.flink.streaming.api.operators.StreamSource;
import org.apache.flink.streaming.connectors.kinesis.config.AWSConfigConstants;
import org.apache.flink.streaming.connectors.kinesis.internals.KinesisDataFetcher;
import org.apache.flink.streaming.connectors.kinesis.model.SequenceNumber;
+import org.apache.flink.streaming.connectors.kinesis.model.StreamShardHandle;
import org.apache.flink.streaming.connectors.kinesis.model.StreamShardMetadata;
import org.apache.flink.streaming.connectors.kinesis.serialization.KinesisDeserializationSchema;
+import org.apache.flink.streaming.connectors.kinesis.serialization.KinesisDeserializationSchemaWrapper;
import org.apache.flink.streaming.connectors.kinesis.testutils.KinesisShardIdGenerator;
+import org.apache.flink.streaming.connectors.kinesis.testutils.TestSourceContext;
import org.apache.flink.streaming.runtime.tasks.OperatorStateHandles;
import org.apache.flink.streaming.util.AbstractStreamOperatorTestHarness;
import org.apache.flink.streaming.util.OperatorSnapshotUtil;
import org.apache.flink.streaming.util.migration.MigrationTestUtil;
import org.apache.flink.streaming.util.migration.MigrationVersion;
+import org.apache.flink.streaming.util.serialization.SimpleStringSchema;
+
+import com.amazonaws.services.kinesis.model.Shard;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
+import org.mockito.Mockito;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Properties;
@@ -51,9 +58,6 @@ import java.util.concurrent.atomic.AtomicReference;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
/**
* Tests for checking whether {@link FlinkKinesisConsumer} can restore from snapshots that were
@@ -71,10 +75,12 @@ public class FlinkKinesisConsumerMigrationTest {
*/
private final MigrationVersion flinkGenerateSavepointVersion = null;
+ private static final String TEST_STREAM_NAME = "fakeStream1";
+
private static final HashMap<StreamShardMetadata, SequenceNumber> TEST_STATE = new HashMap<>();
static {
StreamShardMetadata shardMetadata = new StreamShardMetadata();
- shardMetadata.setStreamName("fakeStream1");
+ shardMetadata.setStreamName(TEST_STREAM_NAME);
shardMetadata.setShardId(KinesisShardIdGenerator.generateFromShardOrder(0));
TEST_STATE.put(shardMetadata, new SequenceNumber("987654321"));
@@ -105,7 +111,25 @@ public class FlinkKinesisConsumerMigrationTest {
@Test
public void testRestoreWithEmptyState() throws Exception {
- final DummyFlinkKinesisConsumer<String> consumerFunction = new DummyFlinkKinesisConsumer<>(mock(KinesisDataFetcher.class));
+ final List<StreamShardHandle> initialDiscoveryShards = new ArrayList<>(TEST_STATE.size());
+ for (StreamShardMetadata shardMetadata : TEST_STATE.keySet()) {
+ Shard shard = new Shard();
+ shard.setShardId(shardMetadata.getShardId());
+
+ initialDiscoveryShards.add(new StreamShardHandle(shardMetadata.getStreamName(), shard));
+ }
+
+ final TestFetcher<String> fetcher = new TestFetcher<>(
+ Collections.singletonList(TEST_STREAM_NAME),
+ new TestSourceContext<>(),
+ getMockRuntimeContext(1, 0),
+ getStandardProperties(),
+ new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
+ null,
+ initialDiscoveryShards);
+
+ final DummyFlinkKinesisConsumer<String> consumerFunction = new DummyFlinkKinesisConsumer<>(
+ fetcher, new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()));
StreamSource<String, DummyFlinkKinesisConsumer<String>> consumerOperator = new StreamSource<>(consumerFunction);
@@ -118,6 +142,8 @@ public class FlinkKinesisConsumerMigrationTest {
"src/test/resources/kinesis-consumer-migration-test-flink" + testMigrateVersion + "-empty-snapshot", testMigrateVersion);
testHarness.open();
+ consumerFunction.run(new TestSourceContext<>());
+
// assert that no state was restored
assertTrue(consumerFunction.getRestoredState().isEmpty());
@@ -127,7 +153,25 @@ public class FlinkKinesisConsumerMigrationTest {
@Test
public void testRestore() throws Exception {
- final DummyFlinkKinesisConsumer<String> consumerFunction = new DummyFlinkKinesisConsumer<>(mock(KinesisDataFetcher.class));
+ final List<StreamShardHandle> initialDiscoveryShards = new ArrayList<>(TEST_STATE.size());
+ for (StreamShardMetadata shardMetadata : TEST_STATE.keySet()) {
+ Shard shard = new Shard();
+ shard.setShardId(shardMetadata.getShardId());
+
+ initialDiscoveryShards.add(new StreamShardHandle(shardMetadata.getStreamName(), shard));
+ }
+
+ final TestFetcher<String> fetcher = new TestFetcher<>(
+ Collections.singletonList(TEST_STREAM_NAME),
+ new TestSourceContext<>(),
+ getMockRuntimeContext(1, 0),
+ getStandardProperties(),
+ new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
+ null,
+ initialDiscoveryShards);
+
+ final DummyFlinkKinesisConsumer<String> consumerFunction = new DummyFlinkKinesisConsumer<>(
+ fetcher, new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()));
StreamSource<String, DummyFlinkKinesisConsumer<String>> consumerOperator =
new StreamSource<>(consumerFunction);
@@ -141,6 +185,8 @@ public class FlinkKinesisConsumerMigrationTest {
"src/test/resources/kinesis-consumer-migration-test-flink" + testMigrateVersion + "-snapshot", testMigrateVersion);
testHarness.open();
+ consumerFunction.run(new TestSourceContext<>());
+
// assert that state is correctly restored
assertNotEquals(null, consumerFunction.getRestoredState());
assertEquals(1, consumerFunction.getRestoredState().size());
@@ -154,19 +200,17 @@ public class FlinkKinesisConsumerMigrationTest {
@SuppressWarnings("unchecked")
private void writeSnapshot(String path, HashMap<StreamShardMetadata, SequenceNumber> state) throws Exception {
- final OneShotLatch latch = new OneShotLatch();
-
- final KinesisDataFetcher<String> fetcher = mock(KinesisDataFetcher.class);
- doAnswer(new Answer() {
- @Override
- public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
- latch.trigger();
- return null;
- }
- }).when(fetcher).runFetcher();
- when(fetcher.snapshotState()).thenReturn(state);
-
- final DummyFlinkKinesisConsumer<String> consumer = new DummyFlinkKinesisConsumer<>(fetcher);
+ final TestFetcher<String> fetcher = new TestFetcher<>(
+ Collections.singletonList(TEST_STREAM_NAME),
+ new TestSourceContext<>(),
+ getMockRuntimeContext(1, 0),
+ new Properties(),
+ new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
+ state,
+ null);
+
+ final DummyFlinkKinesisConsumer<String> consumer = new DummyFlinkKinesisConsumer<>(
+ fetcher, new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()));
StreamSource<String, DummyFlinkKinesisConsumer<String>> consumerOperator = new StreamSource<>(consumer);
@@ -185,7 +229,7 @@ public class FlinkKinesisConsumerMigrationTest {
@Override
public void run() {
try {
- consumer.run(mock(SourceFunction.SourceContext.class));
+ consumer.run(new TestSourceContext<>());
} catch (Throwable t) {
t.printStackTrace();
error.set(t);
@@ -194,9 +238,7 @@ public class FlinkKinesisConsumerMigrationTest {
};
runner.start();
- if (!latch.isTriggered()) {
- latch.await();
- }
+ fetcher.waitUntilRun();
final OperatorStateHandles snapshot;
synchronized (testHarness.getCheckpointLock()) {
@@ -211,6 +253,8 @@ public class FlinkKinesisConsumerMigrationTest {
private static class DummyFlinkKinesisConsumer<T> extends FlinkKinesisConsumer<T> {
+ private static final long serialVersionUID = -1573896262106029446L;
+
private KinesisDataFetcher<T> mockFetcher;
private static Properties dummyConfig = new Properties();
@@ -220,8 +264,8 @@ public class FlinkKinesisConsumerMigrationTest {
dummyConfig.setProperty(AWSConfigConstants.AWS_SECRET_ACCESS_KEY, "secretKey");
}
- DummyFlinkKinesisConsumer(KinesisDataFetcher<T> mockFetcher) {
- super("dummy-topic", mock(KinesisDeserializationSchema.class), dummyConfig);
+ DummyFlinkKinesisConsumer(KinesisDataFetcher<T> mockFetcher, KinesisDeserializationSchema<T> schema) {
+ super(TEST_STREAM_NAME, schema, dummyConfig);
this.mockFetcher = mockFetcher;
}
@@ -235,4 +279,69 @@ public class FlinkKinesisConsumerMigrationTest {
return mockFetcher;
}
}
+
+ private static class TestFetcher<T> extends KinesisDataFetcher<T> {
+
+ final OneShotLatch runLatch = new OneShotLatch();
+
+ final HashMap<StreamShardMetadata, SequenceNumber> testStateSnapshot;
+ final List<StreamShardHandle> testInitialDiscoveryShards;
+
+ public TestFetcher(
+ List<String> streams,
+ SourceFunction.SourceContext<T> sourceContext,
+ RuntimeContext runtimeContext,
+ Properties configProps,
+ KinesisDeserializationSchema<T> deserializationSchema,
+ HashMap<StreamShardMetadata, SequenceNumber> testStateSnapshot,
+ List<StreamShardHandle> testInitialDiscoveryShards) {
+
+ super(streams, sourceContext, runtimeContext, configProps, deserializationSchema);
+
+ this.testStateSnapshot = testStateSnapshot;
+ this.testInitialDiscoveryShards = testInitialDiscoveryShards;
+ }
+
+ @Override
+ public void runFetcher() throws Exception {
+ runLatch.trigger();
+ }
+
+ @Override
+ public HashMap<StreamShardMetadata, SequenceNumber> snapshotState() {
+ return testStateSnapshot;
+ }
+
+ public void waitUntilRun() throws InterruptedException {
+ runLatch.await();
+ }
+
+ @Override
+ public List<StreamShardHandle> discoverNewShardsToSubscribe() throws InterruptedException {
+ return testInitialDiscoveryShards;
+ }
+
+ @Override
+ public void awaitTermination() throws InterruptedException {
+ // do nothing
+ }
+ }
+
+ private static Properties getStandardProperties() {
+ Properties config = new Properties();
+ config.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
+ config.setProperty(AWSConfigConstants.AWS_ACCESS_KEY_ID, "accessKeyId");
+ config.setProperty(AWSConfigConstants.AWS_SECRET_ACCESS_KEY, "secretKey");
+
+ return config;
+ }
+
+ private static RuntimeContext getMockRuntimeContext(int numSubtasks, int thisSubtaskIndex) {
+ RuntimeContext runtimeContext = Mockito.mock(RuntimeContext.class);
+
+ Mockito.when(runtimeContext.getNumberOfParallelSubtasks()).thenReturn(numSubtasks);
+ Mockito.when(runtimeContext.getNumberOfParallelSubtasks()).thenReturn(thisSubtaskIndex);
+
+ return runtimeContext;
+ }
}
[23/28] flink git commit: [FLINK-8472] [scala,
test] Extend StatefulJobSavepointMigrationITCase for Flink 1.4
Posted by tz...@apache.org.
[FLINK-8472] [scala, test] Extend StatefulJobSavepointMigrationITCase for Flink 1.4
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/4fc14ccf
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/4fc14ccf
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/4fc14ccf
Branch: refs/heads/release-1.4
Commit: 4fc14ccf7c9a28095c6838030bb8b0391c1e6262
Parents: 4af0b59
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Thu Jan 25 16:20:31 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 17:31:32 2018 +0100
----------------------------------------------------------------------
.../_metadata | Bin 0 -> 204543 bytes
.../_metadata | Bin 0 -> 204543 bytes
.../StatefulJobSavepointMigrationITCase.scala | 6 ++++--
3 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/4fc14ccf/flink-tests/src/test/resources/stateful-scala2.11-udf-migration-itcase-flink1.4-jobmanager-savepoint/_metadata
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/resources/stateful-scala2.11-udf-migration-itcase-flink1.4-jobmanager-savepoint/_metadata b/flink-tests/src/test/resources/stateful-scala2.11-udf-migration-itcase-flink1.4-jobmanager-savepoint/_metadata
new file mode 100644
index 0000000..69aa8d4
Binary files /dev/null and b/flink-tests/src/test/resources/stateful-scala2.11-udf-migration-itcase-flink1.4-jobmanager-savepoint/_metadata differ
http://git-wip-us.apache.org/repos/asf/flink/blob/4fc14ccf/flink-tests/src/test/resources/stateful-scala2.11-udf-migration-itcase-flink1.4-rocksdb-savepoint/_metadata
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/resources/stateful-scala2.11-udf-migration-itcase-flink1.4-rocksdb-savepoint/_metadata b/flink-tests/src/test/resources/stateful-scala2.11-udf-migration-itcase-flink1.4-rocksdb-savepoint/_metadata
new file mode 100644
index 0000000..7dcef9c
Binary files /dev/null and b/flink-tests/src/test/resources/stateful-scala2.11-udf-migration-itcase-flink1.4-rocksdb-savepoint/_metadata differ
http://git-wip-us.apache.org/repos/asf/flink/blob/4fc14ccf/flink-tests/src/test/scala/org/apache/flink/api/scala/migration/StatefulJobSavepointMigrationITCase.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/migration/StatefulJobSavepointMigrationITCase.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/migration/StatefulJobSavepointMigrationITCase.scala
index 1e67042..55cf04d 100644
--- a/flink-tests/src/test/scala/org/apache/flink/api/scala/migration/StatefulJobSavepointMigrationITCase.scala
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/migration/StatefulJobSavepointMigrationITCase.scala
@@ -54,13 +54,15 @@ object StatefulJobSavepointMigrationITCase {
(MigrationVersion.v1_2, AbstractStateBackend.MEMORY_STATE_BACKEND_NAME),
(MigrationVersion.v1_2, AbstractStateBackend.ROCKSDB_STATE_BACKEND_NAME),
(MigrationVersion.v1_3, AbstractStateBackend.MEMORY_STATE_BACKEND_NAME),
- (MigrationVersion.v1_3, AbstractStateBackend.ROCKSDB_STATE_BACKEND_NAME))
+ (MigrationVersion.v1_3, AbstractStateBackend.ROCKSDB_STATE_BACKEND_NAME),
+ (MigrationVersion.v1_4, AbstractStateBackend.MEMORY_STATE_BACKEND_NAME),
+ (MigrationVersion.v1_4, AbstractStateBackend.ROCKSDB_STATE_BACKEND_NAME))
}
// TODO to generate savepoints for a specific Flink version / backend type,
// TODO change these values accordingly, e.g. to generate for 1.3 with RocksDB,
// TODO set as (MigrationVersion.v1_3, AbstractStateBackend.ROCKSDB_STATE_BACKEND_NAME)
- val GENERATE_SAVEPOINT_VER: MigrationVersion = MigrationVersion.v1_3
+ val GENERATE_SAVEPOINT_VER: MigrationVersion = MigrationVersion.v1_4
val GENERATE_SAVEPOINT_BACKEND_TYPE: String = AbstractStateBackend.ROCKSDB_STATE_BACKEND_NAME
val SCALA_VERSION: String = {
[17/28] flink git commit: [hotfix] [test] Remove stale savepoint
files no longer used by migration tests
Posted by tz...@apache.org.
[hotfix] [test] Remove stale savepoint files no longer used by migration tests
This includes:
- Removing MigrationVersion.v1_1, since compatilbity for 1.1 is no
longer supported (and no migration tests still test that)
- Remove all 1.1 test savepoint files (which no migration tests still
use)
- Remove all 1.2 test savepoint files that are no longer in use (e.g.,
CEPMigrationTest does not test 1.2 restores anymore)
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/e41300d5
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/e41300d5
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/e41300d5
Branch: refs/heads/release-1.4
Commit: e41300d5a9165af3925aeeac3a23fa5c122482b9
Parents: f68fe3e
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Thu Jan 25 17:45:47 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 17:31:32 2018 +0100
----------------------------------------------------------------------
.../rolling-sink-migration-test-flink1.1-snapshot | Bin 1471 -> 0 bytes
.../kafka/FlinkKafkaConsumerBaseMigrationTest.java | 6 +-----
...er-migration-test-flink1.1-empty-state-snapshot | Bin 468 -> 0 bytes
...kafka-consumer-migration-test-flink1.1-snapshot | Bin 738 -> 0 bytes
.../ContinuousFileProcessingMigrationTest.java | 15 ++++-----------
...-migration-test-1482144479339-flink1.1-snapshot | Bin 468 -> 0 bytes
.../reader-migration-test-flink1.1-snapshot | Bin 979 -> 0 bytes
.../streaming/util/migration/MigrationVersion.java | 1 -
...-migration-test-accum-aligned-flink1.1-snapshot | Bin 621 -> 0 bytes
...p-migration-test-aggr-aligned-flink1.1-snapshot | Bin 586 -> 0 bytes
...gration-test-apply-event-time-flink1.1-snapshot | Bin 2427 -> 0 bytes
...on-test-apply-processing-time-flink1.1-snapshot | Bin 2327 -> 0 bytes
...ration-test-reduce-event-time-flink1.1-snapshot | Bin 2457 -> 0 bytes
...n-test-reduce-processing-time-flink1.1-snapshot | Bin 2406 -> 0 bytes
...session-with-stateful-trigger-flink1.1-snapshot | Bin 2988 -> 0 bytes
...on-with-stateful-trigger-mint-flink1.1-snapshot | Bin 538 -> 0 bytes
...udf-migration-itcase-flink1.1-rocksdb-savepoint | Bin 22283 -> 0 bytes
...tateful-udf-migration-itcase-flink1.1-savepoint | Bin 27146 -> 0 bytes
18 files changed, 5 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/e41300d5/flink-connectors/flink-connector-filesystem/src/test/resources/rolling-sink-migration-test-flink1.1-snapshot
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-filesystem/src/test/resources/rolling-sink-migration-test-flink1.1-snapshot b/flink-connectors/flink-connector-filesystem/src/test/resources/rolling-sink-migration-test-flink1.1-snapshot
deleted file mode 100644
index 2ebd70a..0000000
Binary files a/flink-connectors/flink-connector-filesystem/src/test/resources/rolling-sink-migration-test-flink1.1-snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/flink/blob/e41300d5/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java
index 1980ccc..94097e9 100644
--- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java
+++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java
@@ -350,11 +350,7 @@ public class FlinkKafkaConsumerBaseMigrationTest {
fail("Restore from savepoints from version before Flink 1.3.x should have failed if discovery is enabled.");
} catch (Exception e) {
- if (testMigrateVersion == MigrationVersion.v1_1) {
- Assert.assertTrue(e.getCause() instanceof IllegalArgumentException);
- } else {
- Assert.assertTrue(e instanceof IllegalArgumentException);
- }
+ Assert.assertTrue(e instanceof IllegalArgumentException);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e41300d5/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.1-empty-state-snapshot
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.1-empty-state-snapshot b/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.1-empty-state-snapshot
deleted file mode 100644
index f4dd96d..0000000
Binary files a/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.1-empty-state-snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/flink/blob/e41300d5/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.1-snapshot
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.1-snapshot b/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.1-snapshot
deleted file mode 100644
index 01c9e03..0000000
Binary files a/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.1-snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/flink/blob/e41300d5/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingMigrationTest.java
----------------------------------------------------------------------
diff --git a/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingMigrationTest.java b/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingMigrationTest.java
index 1c2ea6f..30a6119 100644
--- a/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingMigrationTest.java
+++ b/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingMigrationTest.java
@@ -203,17 +203,10 @@ public class ContinuousFileProcessingMigrationTest {
// compare if the results contain what they should contain and also if
// they are the same, as they should.
- if (testMigrateVersion == MigrationVersion.v1_1) {
- Assert.assertTrue(testHarness.getOutput().contains(new StreamRecord<>(createSplitFromTimestampedSplit(split1))));
- Assert.assertTrue(testHarness.getOutput().contains(new StreamRecord<>(createSplitFromTimestampedSplit(split2))));
- Assert.assertTrue(testHarness.getOutput().contains(new StreamRecord<>(createSplitFromTimestampedSplit(split3))));
- Assert.assertTrue(testHarness.getOutput().contains(new StreamRecord<>(createSplitFromTimestampedSplit(split4))));
- } else {
- Assert.assertTrue(testHarness.getOutput().contains(new StreamRecord<>(split1)));
- Assert.assertTrue(testHarness.getOutput().contains(new StreamRecord<>(split2)));
- Assert.assertTrue(testHarness.getOutput().contains(new StreamRecord<>(split3)));
- Assert.assertTrue(testHarness.getOutput().contains(new StreamRecord<>(split4)));
- }
+ Assert.assertTrue(testHarness.getOutput().contains(new StreamRecord<>(split1)));
+ Assert.assertTrue(testHarness.getOutput().contains(new StreamRecord<>(split2)));
+ Assert.assertTrue(testHarness.getOutput().contains(new StreamRecord<>(split3)));
+ Assert.assertTrue(testHarness.getOutput().contains(new StreamRecord<>(split4)));
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/e41300d5/flink-fs-tests/src/test/resources/monitoring-function-migration-test-1482144479339-flink1.1-snapshot
----------------------------------------------------------------------
diff --git a/flink-fs-tests/src/test/resources/monitoring-function-migration-test-1482144479339-flink1.1-snapshot b/flink-fs-tests/src/test/resources/monitoring-function-migration-test-1482144479339-flink1.1-snapshot
deleted file mode 100644
index 17eba99..0000000
Binary files a/flink-fs-tests/src/test/resources/monitoring-function-migration-test-1482144479339-flink1.1-snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/flink/blob/e41300d5/flink-fs-tests/src/test/resources/reader-migration-test-flink1.1-snapshot
----------------------------------------------------------------------
diff --git a/flink-fs-tests/src/test/resources/reader-migration-test-flink1.1-snapshot b/flink-fs-tests/src/test/resources/reader-migration-test-flink1.1-snapshot
deleted file mode 100644
index e47ebbd..0000000
Binary files a/flink-fs-tests/src/test/resources/reader-migration-test-flink1.1-snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/flink/blob/e41300d5/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/migration/MigrationVersion.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/migration/MigrationVersion.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/migration/MigrationVersion.java
index 2a1eb98..9b17439 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/migration/MigrationVersion.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/migration/MigrationVersion.java
@@ -26,7 +26,6 @@ public enum MigrationVersion {
// NOTE: the version strings must not change,
// as they are used to locate snapshot file paths
- v1_1("1.1"),
v1_2("1.2"),
v1_3("1.3"),
v1_4("1.4");
http://git-wip-us.apache.org/repos/asf/flink/blob/e41300d5/flink-streaming-java/src/test/resources/win-op-migration-test-accum-aligned-flink1.1-snapshot
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-accum-aligned-flink1.1-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-accum-aligned-flink1.1-snapshot
deleted file mode 100644
index 08ee86c..0000000
Binary files a/flink-streaming-java/src/test/resources/win-op-migration-test-accum-aligned-flink1.1-snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/flink/blob/e41300d5/flink-streaming-java/src/test/resources/win-op-migration-test-aggr-aligned-flink1.1-snapshot
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-aggr-aligned-flink1.1-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-aggr-aligned-flink1.1-snapshot
deleted file mode 100644
index c086a91..0000000
Binary files a/flink-streaming-java/src/test/resources/win-op-migration-test-aggr-aligned-flink1.1-snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/flink/blob/e41300d5/flink-streaming-java/src/test/resources/win-op-migration-test-apply-event-time-flink1.1-snapshot
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-apply-event-time-flink1.1-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-apply-event-time-flink1.1-snapshot
deleted file mode 100644
index 02f0df2..0000000
Binary files a/flink-streaming-java/src/test/resources/win-op-migration-test-apply-event-time-flink1.1-snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/flink/blob/e41300d5/flink-streaming-java/src/test/resources/win-op-migration-test-apply-processing-time-flink1.1-snapshot
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-apply-processing-time-flink1.1-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-apply-processing-time-flink1.1-snapshot
deleted file mode 100644
index 78963d6..0000000
Binary files a/flink-streaming-java/src/test/resources/win-op-migration-test-apply-processing-time-flink1.1-snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/flink/blob/e41300d5/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-event-time-flink1.1-snapshot
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-event-time-flink1.1-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-event-time-flink1.1-snapshot
deleted file mode 100644
index ed428be..0000000
Binary files a/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-event-time-flink1.1-snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/flink/blob/e41300d5/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-processing-time-flink1.1-snapshot
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-processing-time-flink1.1-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-processing-time-flink1.1-snapshot
deleted file mode 100644
index 9945be6..0000000
Binary files a/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-processing-time-flink1.1-snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/flink/blob/e41300d5/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-flink1.1-snapshot
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-flink1.1-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-flink1.1-snapshot
deleted file mode 100644
index 5f42359..0000000
Binary files a/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-flink1.1-snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/flink/blob/e41300d5/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-mint-flink1.1-snapshot
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-mint-flink1.1-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-mint-flink1.1-snapshot
deleted file mode 100644
index 72e8d90..0000000
Binary files a/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-mint-flink1.1-snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/flink/blob/e41300d5/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.1-rocksdb-savepoint
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.1-rocksdb-savepoint b/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.1-rocksdb-savepoint
deleted file mode 100644
index e63038b..0000000
Binary files a/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.1-rocksdb-savepoint and /dev/null differ
http://git-wip-us.apache.org/repos/asf/flink/blob/e41300d5/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.1-savepoint
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.1-savepoint b/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.1-savepoint
deleted file mode 100644
index f2f6dcd..0000000
Binary files a/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.1-savepoint and /dev/null differ
[09/28] flink git commit: [FLINK-8398] [kinesis,
tests] Cleanup confusing implementations in KinesisDataFetcherTest
and related classes
Posted by tz...@apache.org.
[FLINK-8398] [kinesis, tests] Cleanup confusing implementations in KinesisDataFetcherTest and related classes
The previous implementation of the TestableKinesisDataFetcher was
confusing in various ways, causing it hard to be re-used for other
tests. This commit contains the following various cleaups:
- Remove confusing mocks of source context and checkpoint lock. We now
allow users of the TestableKinesisDataFetcher to provide a source
context, which should provide the checkpoint lock.
- Remove override of emitRecordAndUpdateState(). Strictly speaking, that
method should be final. It was previously overriden to allow
verifying how many records were output by the fetcher. That
verification would be better implemented within a mock source context.
- Properly parameterize the output type for the
TestableKinesisDataFetcher.
- Remove use of PowerMockito in KinesisDataFetcherTest.
- Use CheckedThreads to properly capture any exceptions in fetcher /
consumer threads in unit tests.
- Use assertEquals / assertNull instead of assertTrue where-ever
appropriate.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/6922e4c6
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/6922e4c6
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/6922e4c6
Branch: refs/heads/release-1.4
Commit: 6922e4c6b6d6ffe455d550a9f9a7c9a38fe8b8d1
Parents: 0ad82b3
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Wed Jan 10 10:11:31 2018 +0800
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 17:31:32 2018 +0100
----------------------------------------------------------------------
.../kinesis/internals/KinesisDataFetcher.java | 10 +-
.../internals/KinesisDataFetcherTest.java | 239 +++++++++----------
.../kinesis/internals/ShardConsumerTest.java | 39 ++-
.../kinesis/testutils/TestSourceContext.java | 64 +++++
.../testutils/TestableKinesisDataFetcher.java | 104 +++-----
5 files changed, 252 insertions(+), 204 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/6922e4c6/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
index bbfbb20..a8f37a5 100644
--- a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
+++ b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
@@ -17,6 +17,7 @@
package org.apache.flink.streaming.connectors.kinesis.internals;
+import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.functions.RuntimeContext;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.flink.streaming.connectors.kinesis.config.ConsumerConfigConstants;
@@ -183,7 +184,7 @@ public class KinesisDataFetcher<T> {
KinesisProxy.create(configProps));
}
- /** This constructor is exposed for testing purposes. */
+ @VisibleForTesting
protected KinesisDataFetcher(List<String> streams,
SourceFunction.SourceContext<T> sourceContext,
Object checkpointLock,
@@ -481,7 +482,7 @@ public class KinesisDataFetcher<T> {
* when the shard state was registered.
* @param lastSequenceNumber the last sequence number value to update
*/
- protected void emitRecordAndUpdateState(T record, long recordTimestamp, int shardStateIndex, SequenceNumber lastSequenceNumber) {
+ protected final void emitRecordAndUpdateState(T record, long recordTimestamp, int shardStateIndex, SequenceNumber lastSequenceNumber) {
synchronized (checkpointLock) {
sourceContext.collectWithTimestamp(record, recordTimestamp);
updateState(shardStateIndex, lastSequenceNumber);
@@ -498,7 +499,7 @@ public class KinesisDataFetcher<T> {
* when the shard state was registered.
* @param lastSequenceNumber the last sequence number value to update
*/
- protected void updateState(int shardStateIndex, SequenceNumber lastSequenceNumber) {
+ protected final void updateState(int shardStateIndex, SequenceNumber lastSequenceNumber) {
synchronized (checkpointLock) {
subscribedShardsState.get(shardStateIndex).setLastProcessedSequenceNum(lastSequenceNumber);
@@ -559,7 +560,8 @@ public class KinesisDataFetcher<T> {
return (Math.abs(shard.hashCode() % totalNumberOfConsumerSubtasks)) == indexOfThisConsumerSubtask;
}
- private static ExecutorService createShardConsumersThreadPool(final String subtaskName) {
+ @VisibleForTesting
+ protected ExecutorService createShardConsumersThreadPool(final String subtaskName) {
return Executors.newCachedThreadPool(new ThreadFactory() {
@Override
public Thread newThread(Runnable runnable) {
http://git-wip-us.apache.org/repos/asf/flink/blob/6922e4c6/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcherTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcherTest.java b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcherTest.java
index 2e1adb6..56566c0 100644
--- a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcherTest.java
+++ b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcherTest.java
@@ -18,27 +18,26 @@
package org.apache.flink.streaming.connectors.kinesis.internals;
import org.apache.flink.api.common.functions.RuntimeContext;
+import org.apache.flink.api.common.serialization.SimpleStringSchema;
+import org.apache.flink.core.testutils.CheckedThread;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.flink.streaming.connectors.kinesis.FlinkKinesisConsumer;
-import org.apache.flink.streaming.connectors.kinesis.config.ConsumerConfigConstants;
import org.apache.flink.streaming.connectors.kinesis.model.KinesisStreamShardState;
import org.apache.flink.streaming.connectors.kinesis.model.SequenceNumber;
import org.apache.flink.streaming.connectors.kinesis.model.StreamShardHandle;
import org.apache.flink.streaming.connectors.kinesis.model.StreamShardMetadata;
import org.apache.flink.streaming.connectors.kinesis.serialization.KinesisDeserializationSchema;
+import org.apache.flink.streaming.connectors.kinesis.serialization.KinesisDeserializationSchemaWrapper;
import org.apache.flink.streaming.connectors.kinesis.testutils.FakeKinesisBehavioursFactory;
import org.apache.flink.streaming.connectors.kinesis.testutils.KinesisShardIdGenerator;
+import org.apache.flink.streaming.connectors.kinesis.testutils.TestSourceContext;
+import org.apache.flink.streaming.connectors.kinesis.testutils.TestUtils;
import org.apache.flink.streaming.connectors.kinesis.testutils.TestableKinesisDataFetcher;
import com.amazonaws.services.kinesis.model.HashKeyRange;
import com.amazonaws.services.kinesis.model.SequenceNumberRange;
import com.amazonaws.services.kinesis.model.Shard;
import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.Mockito;
-import org.powermock.api.mockito.PowerMockito;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
import java.util.HashMap;
import java.util.LinkedList;
@@ -51,6 +50,7 @@ import java.util.UUID;
import java.util.concurrent.atomic.AtomicReference;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -58,8 +58,6 @@ import static org.mockito.Mockito.when;
/**
* Tests for the {@link KinesisDataFetcher}.
*/
-@RunWith(PowerMockRunner.class)
-@PrepareForTest(TestableKinesisDataFetcher.class)
public class KinesisDataFetcherTest {
@Test(expected = RuntimeException.class)
@@ -71,14 +69,16 @@ public class KinesisDataFetcherTest {
HashMap<String, String> subscribedStreamsToLastSeenShardIdsUnderTest =
KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(fakeStreams);
- TestableKinesisDataFetcher fetcher =
- new TestableKinesisDataFetcher(
+ TestableKinesisDataFetcher<String> fetcher =
+ new TestableKinesisDataFetcher<>(
fakeStreams,
- new Properties(),
+ new TestSourceContext<>(),
+ TestUtils.getStandardProperties(),
+ new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
10,
2,
- new AtomicReference<Throwable>(),
- new LinkedList<KinesisStreamShardState>(),
+ new AtomicReference<>(),
+ new LinkedList<>(),
subscribedStreamsToLastSeenShardIdsUnderTest,
FakeKinesisBehavioursFactory.noShardsFoundForRequestedStreamsBehaviour());
@@ -102,52 +102,44 @@ public class KinesisDataFetcherTest {
streamToShardCount.put(fakeStream, rand.nextInt(5) + 1);
}
- final TestableKinesisDataFetcher fetcher =
- new TestableKinesisDataFetcher(
+ final TestableKinesisDataFetcher<String> fetcher =
+ new TestableKinesisDataFetcher<>(
fakeStreams,
- new Properties(),
+ new TestSourceContext<>(),
+ TestUtils.getStandardProperties(),
+ new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
10,
2,
- new AtomicReference<Throwable>(),
- new LinkedList<KinesisStreamShardState>(),
+ new AtomicReference<>(),
+ new LinkedList<>(),
subscribedStreamsToLastSeenShardIdsUnderTest,
FakeKinesisBehavioursFactory.nonReshardedStreamsBehaviour(streamToShardCount));
- Properties testConfig = new Properties();
- testConfig.setProperty(ConsumerConfigConstants.AWS_REGION, "us-east-1");
- testConfig.setProperty(ConsumerConfigConstants.AWS_CREDENTIALS_PROVIDER, "BASIC");
- testConfig.setProperty(ConsumerConfigConstants.AWS_ACCESS_KEY_ID, "accessKeyId");
- testConfig.setProperty(ConsumerConfigConstants.AWS_SECRET_ACCESS_KEY, "secretKey");
+ final DummyFlinkKafkaConsumer<String> consumer = new DummyFlinkKafkaConsumer<>(
+ TestUtils.getStandardProperties(), fetcher, 1, 0);
- final DummyFlinkKafkaConsumer<String> consumer = new DummyFlinkKafkaConsumer<>(testConfig, fetcher);
-
- PowerMockito.whenNew(ShardConsumer.class).withAnyArguments().thenReturn(Mockito.mock(ShardConsumer.class));
- Thread consumerThread = new Thread(new Runnable() {
+ CheckedThread consumerThread = new CheckedThread() {
@Override
- public void run() {
- try {
- consumer.run(mock(SourceFunction.SourceContext.class));
- } catch (Exception e) {
- //
- }
+ public void go() throws Exception {
+ consumer.run(new TestSourceContext<>());
}
- });
+ };
consumerThread.start();
fetcher.waitUntilRun();
consumer.cancel();
- consumerThread.join();
+ consumerThread.sync();
// assert that the streams tracked in the state are identical to the subscribed streams
Set<String> streamsInState = subscribedStreamsToLastSeenShardIdsUnderTest.keySet();
- assertTrue(streamsInState.size() == fakeStreams.size());
+ assertEquals(fakeStreams.size(), streamsInState.size());
assertTrue(streamsInState.containsAll(fakeStreams));
// assert that the last seen shards in state is correctly set
for (Map.Entry<String, String> streamToLastSeenShard : subscribedStreamsToLastSeenShardIdsUnderTest.entrySet()) {
- assertTrue(
- streamToLastSeenShard.getValue().equals(
- KinesisShardIdGenerator.generateFromShardOrder(streamToShardCount.get(streamToLastSeenShard.getKey()) - 1)));
+ assertEquals(
+ KinesisShardIdGenerator.generateFromShardOrder(streamToShardCount.get(streamToLastSeenShard.getKey()) - 1),
+ streamToLastSeenShard.getValue());
}
}
@@ -195,14 +187,16 @@ public class KinesisDataFetcherTest {
HashMap<String, String> subscribedStreamsToLastSeenShardIdsUnderTest =
KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(fakeStreams);
- final TestableKinesisDataFetcher fetcher =
- new TestableKinesisDataFetcher(
+ final TestableKinesisDataFetcher<String> fetcher =
+ new TestableKinesisDataFetcher<>(
fakeStreams,
- new Properties(),
+ new TestSourceContext<>(),
+ TestUtils.getStandardProperties(),
+ new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
10,
2,
- new AtomicReference<Throwable>(),
- new LinkedList<KinesisStreamShardState>(),
+ new AtomicReference<>(),
+ new LinkedList<>(),
subscribedStreamsToLastSeenShardIdsUnderTest,
FakeKinesisBehavioursFactory.nonReshardedStreamsBehaviour(streamToShardCount));
@@ -213,31 +207,27 @@ public class KinesisDataFetcherTest {
restoredState.getKey(), new SequenceNumber(restoredState.getValue())));
}
- PowerMockito.whenNew(ShardConsumer.class).withAnyArguments().thenReturn(Mockito.mock(ShardConsumer.class));
- Thread runFetcherThread = new Thread(new Runnable() {
+ CheckedThread runFetcherThread = new CheckedThread() {
@Override
- public void run() {
- try {
- fetcher.runFetcher();
- } catch (Exception e) {
- //
- }
+ public void go() throws Exception {
+ fetcher.runFetcher();
}
- });
+ };
runFetcherThread.start();
Thread.sleep(1000); // sleep a while before closing
fetcher.shutdownFetcher();
+ runFetcherThread.sync();
// assert that the streams tracked in the state are identical to the subscribed streams
Set<String> streamsInState = subscribedStreamsToLastSeenShardIdsUnderTest.keySet();
- assertTrue(streamsInState.size() == fakeStreams.size());
+ assertEquals(fakeStreams.size(), streamsInState.size());
assertTrue(streamsInState.containsAll(fakeStreams));
// assert that the last seen shards in state is correctly set
for (Map.Entry<String, String> streamToLastSeenShard : subscribedStreamsToLastSeenShardIdsUnderTest.entrySet()) {
- assertTrue(
- streamToLastSeenShard.getValue().equals(
- KinesisShardIdGenerator.generateFromShardOrder(streamToShardCount.get(streamToLastSeenShard.getKey()) - 1)));
+ assertEquals(
+ KinesisShardIdGenerator.generateFromShardOrder(streamToShardCount.get(streamToLastSeenShard.getKey()) - 1),
+ streamToLastSeenShard.getValue());
}
}
@@ -286,14 +276,16 @@ public class KinesisDataFetcherTest {
KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(fakeStreams);
// using a non-resharded streams kinesis behaviour to represent that Kinesis is not resharded AFTER the restore
- final TestableKinesisDataFetcher fetcher =
- new TestableKinesisDataFetcher(
+ final TestableKinesisDataFetcher<String> fetcher =
+ new TestableKinesisDataFetcher<>(
fakeStreams,
- new Properties(),
+ new TestSourceContext<>(),
+ TestUtils.getStandardProperties(),
+ new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
10,
2,
- new AtomicReference<Throwable>(),
- new LinkedList<KinesisStreamShardState>(),
+ new AtomicReference<>(),
+ new LinkedList<>(),
subscribedStreamsToLastSeenShardIdsUnderTest,
FakeKinesisBehavioursFactory.nonReshardedStreamsBehaviour(streamToShardCount));
@@ -304,31 +296,27 @@ public class KinesisDataFetcherTest {
restoredState.getKey(), new SequenceNumber(restoredState.getValue())));
}
- PowerMockito.whenNew(ShardConsumer.class).withAnyArguments().thenReturn(Mockito.mock(ShardConsumer.class));
- Thread runFetcherThread = new Thread(new Runnable() {
+ CheckedThread runFetcherThread = new CheckedThread() {
@Override
- public void run() {
- try {
- fetcher.runFetcher();
- } catch (Exception e) {
- //
- }
+ public void go() throws Exception {
+ fetcher.runFetcher();
}
- });
+ };
runFetcherThread.start();
Thread.sleep(1000); // sleep a while before closing
fetcher.shutdownFetcher();
+ runFetcherThread.sync();
// assert that the streams tracked in the state are identical to the subscribed streams
Set<String> streamsInState = subscribedStreamsToLastSeenShardIdsUnderTest.keySet();
- assertTrue(streamsInState.size() == fakeStreams.size());
+ assertEquals(fakeStreams.size(), streamsInState.size());
assertTrue(streamsInState.containsAll(fakeStreams));
// assert that the last seen shards in state is correctly set
for (Map.Entry<String, String> streamToLastSeenShard : subscribedStreamsToLastSeenShardIdsUnderTest.entrySet()) {
- assertTrue(
- streamToLastSeenShard.getValue().equals(
- KinesisShardIdGenerator.generateFromShardOrder(streamToShardCount.get(streamToLastSeenShard.getKey()) - 1)));
+ assertEquals(
+ KinesisShardIdGenerator.generateFromShardOrder(streamToShardCount.get(streamToLastSeenShard.getKey()) - 1),
+ streamToLastSeenShard.getValue());
}
}
@@ -381,14 +369,16 @@ public class KinesisDataFetcherTest {
KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(fakeStreams);
// using a non-resharded streams kinesis behaviour to represent that Kinesis is not resharded AFTER the restore
- final TestableKinesisDataFetcher fetcher =
- new TestableKinesisDataFetcher(
+ final TestableKinesisDataFetcher<String> fetcher =
+ new TestableKinesisDataFetcher<>(
fakeStreams,
- new Properties(),
+ new TestSourceContext<>(),
+ TestUtils.getStandardProperties(),
+ new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
10,
2,
- new AtomicReference<Throwable>(),
- new LinkedList<KinesisStreamShardState>(),
+ new AtomicReference<>(),
+ new LinkedList<>(),
subscribedStreamsToLastSeenShardIdsUnderTest,
FakeKinesisBehavioursFactory.nonReshardedStreamsBehaviour(streamToShardCount));
@@ -399,33 +389,31 @@ public class KinesisDataFetcherTest {
restoredState.getKey(), new SequenceNumber(restoredState.getValue())));
}
- PowerMockito.whenNew(ShardConsumer.class).withAnyArguments().thenReturn(Mockito.mock(ShardConsumer.class));
- Thread runFetcherThread = new Thread(new Runnable() {
+ CheckedThread runFetcherThread = new CheckedThread() {
@Override
- public void run() {
- try {
- fetcher.runFetcher();
- } catch (Exception e) {
- //
- }
+ public void go() throws Exception {
+ fetcher.runFetcher();
}
- });
+ };
runFetcherThread.start();
Thread.sleep(1000); // sleep a while before closing
fetcher.shutdownFetcher();
+ runFetcherThread.sync();
// assert that the streams tracked in the state are identical to the subscribed streams
Set<String> streamsInState = subscribedStreamsToLastSeenShardIdsUnderTest.keySet();
- assertTrue(streamsInState.size() == fakeStreams.size());
+ assertEquals(fakeStreams.size(), streamsInState.size());
assertTrue(streamsInState.containsAll(fakeStreams));
// assert that the last seen shards in state is correctly set
- assertTrue(subscribedStreamsToLastSeenShardIdsUnderTest.get("fakeStream1").equals(
- KinesisShardIdGenerator.generateFromShardOrder(2)));
- assertTrue(subscribedStreamsToLastSeenShardIdsUnderTest.get("fakeStream2").equals(
- KinesisShardIdGenerator.generateFromShardOrder(1)));
- assertTrue(subscribedStreamsToLastSeenShardIdsUnderTest.get("fakeStream3") == null);
- assertTrue(subscribedStreamsToLastSeenShardIdsUnderTest.get("fakeStream4") == null);
+ assertEquals(
+ KinesisShardIdGenerator.generateFromShardOrder(2),
+ subscribedStreamsToLastSeenShardIdsUnderTest.get("fakeStream1"));
+ assertEquals(
+ KinesisShardIdGenerator.generateFromShardOrder(1),
+ subscribedStreamsToLastSeenShardIdsUnderTest.get("fakeStream2"));
+ assertNull(subscribedStreamsToLastSeenShardIdsUnderTest.get("fakeStream3"));
+ assertNull(subscribedStreamsToLastSeenShardIdsUnderTest.get("fakeStream4"));
}
@Test
@@ -477,14 +465,16 @@ public class KinesisDataFetcherTest {
KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(fakeStreams);
// using a non-resharded streams kinesis behaviour to represent that Kinesis is not resharded AFTER the restore
- final TestableKinesisDataFetcher fetcher =
- new TestableKinesisDataFetcher(
+ final TestableKinesisDataFetcher<String> fetcher =
+ new TestableKinesisDataFetcher<>(
fakeStreams,
- new Properties(),
+ new TestSourceContext<>(),
+ TestUtils.getStandardProperties(),
+ new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
10,
2,
- new AtomicReference<Throwable>(),
- new LinkedList<KinesisStreamShardState>(),
+ new AtomicReference<>(),
+ new LinkedList<>(),
subscribedStreamsToLastSeenShardIdsUnderTest,
FakeKinesisBehavioursFactory.nonReshardedStreamsBehaviour(streamToShardCount));
@@ -495,33 +485,31 @@ public class KinesisDataFetcherTest {
restoredState.getKey(), new SequenceNumber(restoredState.getValue())));
}
- PowerMockito.whenNew(ShardConsumer.class).withAnyArguments().thenReturn(Mockito.mock(ShardConsumer.class));
- Thread runFetcherThread = new Thread(new Runnable() {
+ CheckedThread runFetcherThread = new CheckedThread() {
@Override
- public void run() {
- try {
- fetcher.runFetcher();
- } catch (Exception e) {
- //
- }
+ public void go() throws Exception {
+ fetcher.runFetcher();
}
- });
+ };
runFetcherThread.start();
Thread.sleep(1000); // sleep a while before closing
fetcher.shutdownFetcher();
+ runFetcherThread.sync();
// assert that the streams tracked in the state are identical to the subscribed streams
Set<String> streamsInState = subscribedStreamsToLastSeenShardIdsUnderTest.keySet();
- assertTrue(streamsInState.size() == fakeStreams.size());
+ assertEquals(fakeStreams.size(), streamsInState.size());
assertTrue(streamsInState.containsAll(fakeStreams));
// assert that the last seen shards in state is correctly set
- assertTrue(subscribedStreamsToLastSeenShardIdsUnderTest.get("fakeStream1").equals(
- KinesisShardIdGenerator.generateFromShardOrder(3)));
- assertTrue(subscribedStreamsToLastSeenShardIdsUnderTest.get("fakeStream2").equals(
- KinesisShardIdGenerator.generateFromShardOrder(4)));
- assertTrue(subscribedStreamsToLastSeenShardIdsUnderTest.get("fakeStream3") == null);
- assertTrue(subscribedStreamsToLastSeenShardIdsUnderTest.get("fakeStream4") == null);
+ assertEquals(
+ KinesisShardIdGenerator.generateFromShardOrder(3),
+ subscribedStreamsToLastSeenShardIdsUnderTest.get("fakeStream1"));
+ assertEquals(
+ KinesisShardIdGenerator.generateFromShardOrder(4),
+ subscribedStreamsToLastSeenShardIdsUnderTest.get("fakeStream2"));
+ assertNull(subscribedStreamsToLastSeenShardIdsUnderTest.get("fakeStream3"));
+ assertNull(subscribedStreamsToLastSeenShardIdsUnderTest.get("fakeStream4"));
}
@Test
@@ -564,12 +552,21 @@ public class KinesisDataFetcherTest {
private static class DummyFlinkKafkaConsumer<T> extends FlinkKinesisConsumer<T> {
private static final long serialVersionUID = 1L;
- private KinesisDataFetcher<T> fetcher;
+ private final KinesisDataFetcher<T> fetcher;
+
+ private final int numParallelSubtasks;
+ private final int subtaskIndex;
@SuppressWarnings("unchecked")
- DummyFlinkKafkaConsumer(Properties properties, KinesisDataFetcher<T> fetcher) {
+ DummyFlinkKafkaConsumer(
+ Properties properties,
+ KinesisDataFetcher<T> fetcher,
+ int numParallelSubtasks,
+ int subtaskIndex) {
super("test", mock(KinesisDeserializationSchema.class), properties);
this.fetcher = fetcher;
+ this.numParallelSubtasks = numParallelSubtasks;
+ this.subtaskIndex = subtaskIndex;
}
@Override
@@ -585,8 +582,8 @@ public class KinesisDataFetcherTest {
@Override
public RuntimeContext getRuntimeContext() {
RuntimeContext context = mock(RuntimeContext.class);
- when(context.getIndexOfThisSubtask()).thenReturn(0);
- when(context.getNumberOfParallelSubtasks()).thenReturn(1);
+ when(context.getIndexOfThisSubtask()).thenReturn(subtaskIndex);
+ when(context.getNumberOfParallelSubtasks()).thenReturn(numParallelSubtasks);
return context;
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/6922e4c6/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/ShardConsumerTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/ShardConsumerTest.java b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/ShardConsumerTest.java
index a194835..efc98a4 100644
--- a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/ShardConsumerTest.java
+++ b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/ShardConsumerTest.java
@@ -22,9 +22,12 @@ import org.apache.flink.streaming.connectors.kinesis.model.SentinelSequenceNumbe
import org.apache.flink.streaming.connectors.kinesis.model.SequenceNumber;
import org.apache.flink.streaming.connectors.kinesis.model.StreamShardHandle;
import org.apache.flink.streaming.connectors.kinesis.proxy.KinesisProxyInterface;
+import org.apache.flink.streaming.connectors.kinesis.serialization.KinesisDeserializationSchemaWrapper;
import org.apache.flink.streaming.connectors.kinesis.testutils.FakeKinesisBehavioursFactory;
import org.apache.flink.streaming.connectors.kinesis.testutils.KinesisShardIdGenerator;
+import org.apache.flink.streaming.connectors.kinesis.testutils.TestSourceContext;
import org.apache.flink.streaming.connectors.kinesis.testutils.TestableKinesisDataFetcher;
+import org.apache.flink.streaming.util.serialization.SimpleStringSchema;
import com.amazonaws.services.kinesis.model.HashKeyRange;
import com.amazonaws.services.kinesis.model.Shard;
@@ -38,7 +41,7 @@ import java.util.LinkedList;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicReference;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
/**
* Tests for the {@link ShardConsumer}.
@@ -61,13 +64,17 @@ public class ShardConsumerTest {
new KinesisStreamShardState(KinesisDataFetcher.convertToStreamShardMetadata(fakeToBeConsumedShard),
fakeToBeConsumedShard, new SequenceNumber("fakeStartingState")));
- TestableKinesisDataFetcher fetcher =
- new TestableKinesisDataFetcher(
+ TestSourceContext<String> sourceContext = new TestSourceContext<>();
+
+ TestableKinesisDataFetcher<String> fetcher =
+ new TestableKinesisDataFetcher<>(
Collections.singletonList("fakeStream"),
+ sourceContext,
new Properties(),
+ new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
10,
2,
- new AtomicReference<Throwable>(),
+ new AtomicReference<>(),
subscribedShardsStateUnderTest,
KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")),
Mockito.mock(KinesisProxyInterface.class));
@@ -79,9 +86,10 @@ public class ShardConsumerTest {
subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum(),
FakeKinesisBehavioursFactory.totalNumOfRecordsAfterNumOfGetRecordsCalls(1000, 9)).run();
- assertTrue(fetcher.getNumOfElementsCollected() == 1000);
- assertTrue(subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum().equals(
- SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get()));
+ assertEquals(1000, sourceContext.getCollectedOutputs().size());
+ assertEquals(
+ SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get(),
+ subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum());
}
@Test
@@ -100,13 +108,17 @@ public class ShardConsumerTest {
new KinesisStreamShardState(KinesisDataFetcher.convertToStreamShardMetadata(fakeToBeConsumedShard),
fakeToBeConsumedShard, new SequenceNumber("fakeStartingState")));
- TestableKinesisDataFetcher fetcher =
- new TestableKinesisDataFetcher(
+ TestSourceContext<String> sourceContext = new TestSourceContext<>();
+
+ TestableKinesisDataFetcher<String> fetcher =
+ new TestableKinesisDataFetcher<>(
Collections.singletonList("fakeStream"),
+ sourceContext,
new Properties(),
+ new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
10,
2,
- new AtomicReference<Throwable>(),
+ new AtomicReference<>(),
subscribedShardsStateUnderTest,
KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")),
Mockito.mock(KinesisProxyInterface.class));
@@ -120,9 +132,10 @@ public class ShardConsumerTest {
// and the 7th getRecords() call will encounter an unexpected expired shard iterator
FakeKinesisBehavioursFactory.totalNumOfRecordsAfterNumOfGetRecordsCallsWithUnexpectedExpiredIterator(1000, 9, 7)).run();
- assertTrue(fetcher.getNumOfElementsCollected() == 1000);
- assertTrue(subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum().equals(
- SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get()));
+ assertEquals(1000, sourceContext.getCollectedOutputs().size());
+ assertEquals(
+ SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get(),
+ subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum());
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/6922e4c6/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestSourceContext.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestSourceContext.java b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestSourceContext.java
new file mode 100644
index 0000000..4fe5c54
--- /dev/null
+++ b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestSourceContext.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kinesis.testutils;
+
+import org.apache.flink.streaming.api.functions.source.SourceFunction;
+import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+
+import java.util.concurrent.ConcurrentLinkedQueue;
+
+/**
+ * A testable {@link SourceFunction.SourceContext}.
+ */
+public class TestSourceContext<T> implements SourceFunction.SourceContext<T> {
+
+ private final Object checkpointLock = new Object();
+
+ private ConcurrentLinkedQueue<StreamRecord<T>> collectedOutputs = new ConcurrentLinkedQueue<>();
+
+ @Override
+ public void collect(T element) {
+ this.collectedOutputs.add(new StreamRecord<>(element));
+ }
+
+ @Override
+ public void collectWithTimestamp(T element, long timestamp) {
+ this.collectedOutputs.add(new StreamRecord<>(element, timestamp));
+ }
+
+ @Override
+ public void emitWatermark(Watermark mark) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void markAsTemporarilyIdle() {}
+
+ @Override
+ public Object getCheckpointLock() {
+ return checkpointLock;
+ }
+
+ @Override
+ public void close() {}
+
+ public ConcurrentLinkedQueue<StreamRecord<T>> getCollectedOutputs() {
+ return collectedOutputs;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/6922e4c6/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestableKinesisDataFetcher.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestableKinesisDataFetcher.java b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestableKinesisDataFetcher.java
index b6f3cbc..65ae6cc 100644
--- a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestableKinesisDataFetcher.java
+++ b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestableKinesisDataFetcher.java
@@ -22,72 +22,55 @@ import org.apache.flink.core.testutils.OneShotLatch;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.flink.streaming.connectors.kinesis.internals.KinesisDataFetcher;
import org.apache.flink.streaming.connectors.kinesis.model.KinesisStreamShardState;
-import org.apache.flink.streaming.connectors.kinesis.model.SequenceNumber;
import org.apache.flink.streaming.connectors.kinesis.proxy.KinesisProxyInterface;
import org.apache.flink.streaming.connectors.kinesis.serialization.KinesisDeserializationSchema;
-import org.apache.flink.streaming.connectors.kinesis.serialization.KinesisDeserializationSchemaWrapper;
-import org.apache.flink.streaming.util.serialization.SimpleStringSchema;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
+import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicReference;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
/**
* Extension of the {@link KinesisDataFetcher} for testing.
*/
-public class TestableKinesisDataFetcher extends KinesisDataFetcher<String> {
-
- private static final Object fakeCheckpointLock = new Object();
-
- private long numElementsCollected;
+public class TestableKinesisDataFetcher<T> extends KinesisDataFetcher<T> {
private OneShotLatch runWaiter;
+ private volatile boolean running;
public TestableKinesisDataFetcher(
List<String> fakeStreams,
+ SourceFunction.SourceContext<T> sourceContext,
Properties fakeConfiguration,
+ KinesisDeserializationSchema<T> deserializationSchema,
int fakeTotalCountOfSubtasks,
- int fakeTndexOfThisSubtask,
+ int fakeIndexOfThisSubtask,
AtomicReference<Throwable> thrownErrorUnderTest,
LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest,
HashMap<String, String> subscribedStreamsToLastDiscoveredShardIdsStateUnderTest,
KinesisProxyInterface fakeKinesis) {
- super(fakeStreams,
- getMockedSourceContext(),
- fakeCheckpointLock,
- getMockedRuntimeContext(fakeTotalCountOfSubtasks, fakeTndexOfThisSubtask),
+ super(
+ fakeStreams,
+ sourceContext,
+ sourceContext.getCheckpointLock(),
+ getMockedRuntimeContext(fakeTotalCountOfSubtasks, fakeIndexOfThisSubtask),
fakeConfiguration,
- new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
+ deserializationSchema,
thrownErrorUnderTest,
subscribedShardsStateUnderTest,
subscribedStreamsToLastDiscoveredShardIdsStateUnderTest,
fakeKinesis);
- this.numElementsCollected = 0;
this.runWaiter = new OneShotLatch();
- }
-
- public long getNumOfElementsCollected() {
- return numElementsCollected;
- }
-
- @Override
- protected KinesisDeserializationSchema<String> getClonedDeserializationSchema() {
- return new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema());
- }
-
- @Override
- protected void emitRecordAndUpdateState(String record, long recordTimestamp, int shardStateIndex, SequenceNumber lastSequenceNumber) {
- synchronized (fakeCheckpointLock) {
- this.numElementsCollected++;
- updateState(shardStateIndex, lastSequenceNumber);
- }
+ this.running = true;
}
@Override
@@ -100,41 +83,30 @@ public class TestableKinesisDataFetcher extends KinesisDataFetcher<String> {
runWaiter.await();
}
- @SuppressWarnings("unchecked")
- private static SourceFunction.SourceContext<String> getMockedSourceContext() {
- return Mockito.mock(SourceFunction.SourceContext.class);
+ @Override
+ protected ExecutorService createShardConsumersThreadPool(String subtaskName) {
+ // this is just a dummy fetcher, so no need to create a thread pool for shard consumers
+ ExecutorService mockExecutor = mock(ExecutorService.class);
+ when(mockExecutor.isTerminated()).thenAnswer((InvocationOnMock invocation) -> !running);
+ return mockExecutor;
+ }
+
+ @Override
+ public void awaitTermination() throws InterruptedException {
+ this.running = false;
+ super.awaitTermination();
}
- private static RuntimeContext getMockedRuntimeContext(final int fakeTotalCountOfSubtasks, final int fakeTndexOfThisSubtask) {
- RuntimeContext mockedRuntimeContext = Mockito.mock(RuntimeContext.class);
-
- Mockito.when(mockedRuntimeContext.getNumberOfParallelSubtasks()).thenAnswer(new Answer<Integer>() {
- @Override
- public Integer answer(InvocationOnMock invocationOnMock) throws Throwable {
- return fakeTotalCountOfSubtasks;
- }
- });
-
- Mockito.when(mockedRuntimeContext.getIndexOfThisSubtask()).thenAnswer(new Answer<Integer>() {
- @Override
- public Integer answer(InvocationOnMock invocationOnMock) throws Throwable {
- return fakeTndexOfThisSubtask;
- }
- });
-
- Mockito.when(mockedRuntimeContext.getTaskName()).thenAnswer(new Answer<String>() {
- @Override
- public String answer(InvocationOnMock invocationOnMock) throws Throwable {
- return "Fake Task";
- }
- });
-
- Mockito.when(mockedRuntimeContext.getTaskNameWithSubtasks()).thenAnswer(new Answer<String>() {
- @Override
- public String answer(InvocationOnMock invocationOnMock) throws Throwable {
- return "Fake Task (" + fakeTndexOfThisSubtask + "/" + fakeTotalCountOfSubtasks + ")";
- }
- });
+ private static RuntimeContext getMockedRuntimeContext(final int fakeTotalCountOfSubtasks, final int fakeIndexOfThisSubtask) {
+ RuntimeContext mockedRuntimeContext = mock(RuntimeContext.class);
+
+ Mockito.when(mockedRuntimeContext.getNumberOfParallelSubtasks()).thenReturn(fakeTotalCountOfSubtasks);
+ Mockito.when(mockedRuntimeContext.getIndexOfThisSubtask()).thenReturn(fakeIndexOfThisSubtask);
+ Mockito.when(mockedRuntimeContext.getTaskName()).thenReturn("Fake Task");
+ Mockito.when(mockedRuntimeContext.getTaskNameWithSubtasks()).thenReturn(
+ "Fake Task (" + fakeIndexOfThisSubtask + "/" + fakeTotalCountOfSubtasks + ")");
+ Mockito.when(mockedRuntimeContext.getUserCodeClassLoader()).thenReturn(
+ Thread.currentThread().getContextClassLoader());
return mockedRuntimeContext;
}
[08/28] flink git commit: [FLINK-8419] [kafka] Register metrics for
dynamically discovered Kafka partitions
Posted by tz...@apache.org.
[FLINK-8419] [kafka] Register metrics for dynamically discovered Kafka partitions
This closes #5335.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/432273aa
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/432273aa
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/432273aa
Branch: refs/heads/release-1.4
Commit: 432273aa963f46dc032c5007389a1a681b5188e6
Parents: e5ac4b0
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Mon Jan 22 14:14:44 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 17:31:32 2018 +0100
----------------------------------------------------------------------
.../kafka/internals/AbstractFetcher.java | 45 +++++++++++++++-----
1 file changed, 35 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/432273aa/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcher.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcher.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcher.java
index 441e660..90c1ae9 100644
--- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcher.java
+++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcher.java
@@ -104,6 +104,24 @@ public abstract class AbstractFetcher<T, KPH> {
private volatile long maxWatermarkSoFar = Long.MIN_VALUE;
// ------------------------------------------------------------------------
+ // Metrics
+ // ------------------------------------------------------------------------
+
+ /**
+ * Flag indicating whether or not metrics should be exposed.
+ * If {@code true}, offset metrics (e.g. current offset, committed offset) and
+ * Kafka-shipped metrics will be registered.
+ */
+ private final boolean useMetrics;
+
+ /**
+ * The metric group which all metrics for the consumer should be registered to.
+ * This metric group is defined under the user scope "KafkaConsumer".
+ */
+ private final MetricGroup consumerMetricGroup;
+
+ private final MetricGroup currentOffsetsMetricGroup;
+ private final MetricGroup committedOffsetsMetricGroup;
protected AbstractFetcher(
SourceContext<T> sourceContext,
@@ -119,6 +137,11 @@ public abstract class AbstractFetcher<T, KPH> {
this.checkpointLock = sourceContext.getCheckpointLock();
this.userCodeClassLoader = checkNotNull(userCodeClassLoader);
+ this.useMetrics = useMetrics;
+ this.consumerMetricGroup = checkNotNull(consumerMetricGroup);
+ this.currentOffsetsMetricGroup = consumerMetricGroup.addGroup("current-offsets");
+ this.committedOffsetsMetricGroup = consumerMetricGroup.addGroup("committed-offsets");
+
// figure out what we watermark mode we will be using
this.watermarksPeriodic = watermarksPeriodic;
this.watermarksPunctuated = watermarksPunctuated;
@@ -160,8 +183,9 @@ public abstract class AbstractFetcher<T, KPH> {
unassignedPartitionsQueue.add(partition);
}
+ // register metrics for the initial seed partitions
if (useMetrics) {
- addOffsetStateGauge(checkNotNull(consumerMetricGroup));
+ addOffsetStateGauge(subscribedPartitionStates);
}
// if we have periodic watermarks, kick off the interval scheduler
@@ -199,6 +223,10 @@ public abstract class AbstractFetcher<T, KPH> {
watermarksPunctuated,
userCodeClassLoader);
+ if (useMetrics) {
+ addOffsetStateGauge(newPartitionStates);
+ }
+
for (KafkaTopicPartitionState<KPH> newPartitionState : newPartitionStates) {
subscribedPartitionStates.add(newPartitionState);
unassignedPartitionsQueue.add(newPartitionState);
@@ -560,17 +588,14 @@ public abstract class AbstractFetcher<T, KPH> {
// ------------------------- Metrics ----------------------------------
/**
- * Add current and committed offsets to metric group.
+ * For each partition, register a gauge to expose current offsets and committed offsets.
*
- * @param metricGroup The metric group to use
+ * @param partitionOffsetStates The partition offset state holders, whose values will be used to update metrics
*/
- protected void addOffsetStateGauge(MetricGroup metricGroup) {
- // add current offsets to gage
- MetricGroup currentOffsets = metricGroup.addGroup("current-offsets");
- MetricGroup committedOffsets = metricGroup.addGroup("committed-offsets");
- for (KafkaTopicPartitionState<KPH> ktp : subscribedPartitionStates) {
- currentOffsets.gauge(ktp.getTopic() + "-" + ktp.getPartition(), new OffsetGauge(ktp, OffsetGaugeType.CURRENT_OFFSET));
- committedOffsets.gauge(ktp.getTopic() + "-" + ktp.getPartition(), new OffsetGauge(ktp, OffsetGaugeType.COMMITTED_OFFSET));
+ private void addOffsetStateGauge(List<KafkaTopicPartitionState<KPH>> partitionOffsetStates) {
+ for (KafkaTopicPartitionState<KPH> ktp : partitionOffsetStates) {
+ currentOffsetsMetricGroup.gauge(ktp.getTopic() + "-" + ktp.getPartition(), new OffsetGauge(ktp, OffsetGaugeType.CURRENT_OFFSET));
+ committedOffsetsMetricGroup.gauge(ktp.getTopic() + "-" + ktp.getPartition(), new OffsetGauge(ktp, OffsetGaugeType.COMMITTED_OFFSET));
}
}
[24/28] flink git commit: [FLINK-8421] [core] Introduce
PostVersionedIOReadableWritable
Posted by tz...@apache.org.
[FLINK-8421] [core] Introduce PostVersionedIOReadableWritable
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/93d514e6
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/93d514e6
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/93d514e6
Branch: refs/heads/release-1.4
Commit: 93d514e628478a14b2a01a955ca428d1bc7abd74
Parents: 319d510
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Wed Jan 24 18:08:52 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 18:11:26 2018 +0100
----------------------------------------------------------------------
.../io/PostVersionedIOReadableWritable.java | 88 +++++++++++++
.../io/PostVersionedIOReadableWritableTest.java | 132 +++++++++++++++++++
2 files changed, 220 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/93d514e6/flink-core/src/main/java/org/apache/flink/core/io/PostVersionedIOReadableWritable.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/core/io/PostVersionedIOReadableWritable.java b/flink-core/src/main/java/org/apache/flink/core/io/PostVersionedIOReadableWritable.java
new file mode 100644
index 0000000..6edc983
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/core/io/PostVersionedIOReadableWritable.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.core.io;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataInputViewStreamWrapper;
+import org.apache.flink.core.memory.DataOutputView;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PushbackInputStream;
+import java.util.Arrays;
+
+/**
+ * A {@link VersionedIOReadableWritable} which allows to differentiate whether the previous
+ * data was versioned with a {@link VersionedIOReadableWritable}. This can be used if previously
+ * written data was not versioned, and is to be migrated to a versioned format.
+ */
+@Internal
+public abstract class PostVersionedIOReadableWritable extends VersionedIOReadableWritable {
+
+ /** NOTE: CANNOT CHANGE! */
+ private static final byte[] VERSIONED_IDENTIFIER = new byte[] {-15, -51, -123, -97};
+
+ /**
+ * Read from the provided {@link DataInputView in}. A flag {@code wasVersioned} can be
+ * used to determine whether or not the data to read was previously written
+ * by a {@link VersionedIOReadableWritable}.
+ */
+ protected abstract void read(DataInputView in, boolean wasVersioned) throws IOException;
+
+ @Override
+ public void write(DataOutputView out) throws IOException {
+ out.write(VERSIONED_IDENTIFIER);
+ super.write(out);
+ }
+
+ /**
+ * This read attempts to first identify if the input view contains the special
+ * {@link #VERSIONED_IDENTIFIER} by reading and buffering the first few bytes.
+ * If identified to be versioned, the usual version resolution read path
+ * in {@link VersionedIOReadableWritable#read(DataInputView)} is invoked.
+ * Otherwise, we "reset" the input stream by pushing back the read buffered bytes
+ * into the stream.
+ */
+ public final void read(InputStream inputStream) throws IOException {
+ byte[] tmp = new byte[VERSIONED_IDENTIFIER.length];
+ inputStream.read(tmp);
+
+ if (Arrays.equals(tmp, VERSIONED_IDENTIFIER)) {
+ DataInputView inputView = new DataInputViewStreamWrapper(inputStream);
+
+ super.read(inputView);
+ read(inputView, true);
+ } else {
+ PushbackInputStream resetStream = new PushbackInputStream(inputStream, VERSIONED_IDENTIFIER.length);
+ resetStream.unread(tmp);
+
+ read(new DataInputViewStreamWrapper(resetStream), false);
+ }
+ }
+
+ /**
+ * We do not support reading from a {@link DataInputView}, because it does not
+ * support pushing back already read bytes.
+ */
+ @Override
+ public final void read(DataInputView in) throws IOException {
+ throw new UnsupportedOperationException("PostVersionedIOReadableWritable cannot read from a DataInputView.");
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/93d514e6/flink-core/src/test/java/org/apache/flink/core/io/PostVersionedIOReadableWritableTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/core/io/PostVersionedIOReadableWritableTest.java b/flink-core/src/test/java/org/apache/flink/core/io/PostVersionedIOReadableWritableTest.java
new file mode 100644
index 0000000..2954536
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/core/io/PostVersionedIOReadableWritableTest.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.core.io;
+
+import org.apache.flink.core.memory.ByteArrayInputStreamWithPos;
+import org.apache.flink.core.memory.ByteArrayOutputStreamWithPos;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+
+/**
+ * Suite of tests for {@link PostVersionedIOReadableWritable}.
+ */
+public class PostVersionedIOReadableWritableTest {
+
+ @Test
+ public void testReadVersioned() throws IOException {
+
+ String payload = "test-data";
+ TestPostVersionedReadableWritable versionedReadableWritable = new TestPostVersionedReadableWritable(payload);
+
+ byte[] serialized;
+ try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
+ versionedReadableWritable.write(new DataOutputViewStreamWrapper(out));
+ serialized = out.toByteArray();
+ }
+
+ TestPostVersionedReadableWritable restoredVersionedReadableWritable = new TestPostVersionedReadableWritable();
+ try(ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
+ restoredVersionedReadableWritable.read(in);
+ }
+
+ Assert.assertEquals(payload, restoredVersionedReadableWritable.getData());
+ }
+
+ @Test
+ public void testReadNonVersioned() throws IOException {
+ int preVersionedPayload = 563;
+
+ TestNonVersionedReadableWritable nonVersionedReadableWritable = new TestNonVersionedReadableWritable(preVersionedPayload);
+
+ byte[] serialized;
+ try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
+ nonVersionedReadableWritable.write(new DataOutputViewStreamWrapper(out));
+ serialized = out.toByteArray();
+ }
+
+ TestPostVersionedReadableWritable restoredVersionedReadableWritable = new TestPostVersionedReadableWritable();
+ try(ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
+ restoredVersionedReadableWritable.read(in);
+ }
+
+ Assert.assertEquals(String.valueOf(preVersionedPayload), restoredVersionedReadableWritable.getData());
+ }
+
+ static class TestPostVersionedReadableWritable extends PostVersionedIOReadableWritable {
+
+ private static final int VERSION = 1;
+ private String data;
+
+ TestPostVersionedReadableWritable() {}
+
+ TestPostVersionedReadableWritable(String data) {
+ this.data = data;
+ }
+
+ @Override
+ public int getVersion() {
+ return VERSION;
+ }
+
+ @Override
+ public void write(DataOutputView out) throws IOException {
+ super.write(out);
+ out.writeUTF(data);
+ }
+
+ @Override
+ protected void read(DataInputView in, boolean wasVersioned) throws IOException {
+ if (wasVersioned) {
+ this.data = in.readUTF();
+ } else {
+ // in the previous non-versioned format, we wrote integers instead
+ this.data = String.valueOf(in.readInt());
+ }
+ }
+
+ public String getData() {
+ return data;
+ }
+ }
+
+ static class TestNonVersionedReadableWritable implements IOReadableWritable {
+
+ private int data;
+
+ TestNonVersionedReadableWritable(int data) {
+ this.data = data;
+ }
+
+ @Override
+ public void write(DataOutputView out) throws IOException {
+ out.writeInt(data);
+ }
+
+ @Override
+ public void read(DataInputView in) throws IOException {
+ this.data = in.readInt();
+ }
+ }
+
+}
[25/28] flink git commit: [FLINK-8421] [DataStream] Make timer
serializers reconfigurable on restore
Posted by tz...@apache.org.
[FLINK-8421] [DataStream] Make timer serializers reconfigurable on restore
Previously, the key and namespace serializers for the
HeapInternalTimerService were not reconfigured on restore to be compatible
with previously written serializers.
This caused an immediate error to restore savepoints in Flink 1.4.0,
since in Flink 1.4.0 we changed the base registrations in the Kryo
serializer. That change requires serializer reconfiguration.
This commit fixes this by writing also the serializer configuration
snapshots of the key and namespace serializer into savepoints, and use
them to reconfigure the new serializers on rrestore. This improvement also
comes along with making the written data for timer service snapshots
versioned. Backwards compatibility with previous non-versioned formats
is not broken.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/71d682eb
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/71d682eb
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/71d682eb
Branch: refs/heads/release-1.4
Commit: 71d682eb7fef37481669f2ad07f891411223c0f0
Parents: 93d514e
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Wed Jan 24 18:09:44 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 18:30:31 2018 +0100
----------------------------------------------------------------------
.../api/operators/AbstractStreamOperator.java | 3 +-
.../api/operators/HeapInternalTimerService.java | 122 ++++----
.../operators/InternalTimeServiceManager.java | 51 ++--
.../InternalTimerServiceSerializationProxy.java | 132 +++++++++
.../api/operators/InternalTimersSnapshot.java | 120 ++++++++
.../InternalTimersSnapshotReaderWriters.java | 291 +++++++++++++++++++
.../operators/HeapInternalTimerServiceTest.java | 124 +++++---
7 files changed, 696 insertions(+), 147 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/71d682eb/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
index 58022a9..a8d9555 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
@@ -28,7 +28,6 @@ import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.MetricOptions;
-import org.apache.flink.core.memory.DataInputViewStreamWrapper;
import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
import org.apache.flink.metrics.Counter;
import org.apache.flink.metrics.Gauge;
@@ -437,7 +436,7 @@ public abstract class AbstractStreamOperator<OUT>
"Key Group " + keyGroupIdx + " does not belong to the local range.");
timeServiceManager.restoreStateForKeyGroup(
- new DataInputViewStreamWrapper(streamProvider.getStream()),
+ streamProvider.getStream(),
keyGroupIdx, getUserCodeClassloader());
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/71d682eb/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/HeapInternalTimerService.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/HeapInternalTimerService.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/HeapInternalTimerService.java
index 1b531aa..f6c4d28 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/HeapInternalTimerService.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/HeapInternalTimerService.java
@@ -19,14 +19,13 @@
package org.apache.flink.streaming.api.operators;
import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
+import org.apache.flink.api.common.typeutils.CompatibilityUtil;
import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.core.memory.DataInputViewStreamWrapper;
-import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
import org.apache.flink.runtime.state.KeyGroupRangeAssignment;
import org.apache.flink.runtime.state.KeyGroupsList;
import org.apache.flink.streaming.runtime.tasks.ProcessingTimeCallback;
import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService;
-import org.apache.flink.util.InstantiationUtil;
import org.apache.flink.util.Preconditions;
import java.io.IOException;
@@ -84,8 +83,6 @@ public class HeapInternalTimerService<K, N> implements InternalTimerService<N>,
private TypeSerializer<N> namespaceSerializer;
- private InternalTimer.TimerSerializer<K, N> timerSerializer;
-
private Triggerable<K, N> triggerTarget;
private volatile boolean isInitialized;
@@ -94,6 +91,9 @@ public class HeapInternalTimerService<K, N> implements InternalTimerService<N>,
private TypeSerializer<N> namespaceDeserializer;
+ /** The restored timers snapshot, if any. */
+ private InternalTimersSnapshot<K, N> restoredTimersSnapshot;
+
public HeapInternalTimerService(
int totalKeyGroups,
KeyGroupsList localKeyGroupRange,
@@ -148,10 +148,23 @@ public class HeapInternalTimerService<K, N> implements InternalTimerService<N>,
}
// the following is the case where we restore
- if ((this.keyDeserializer != null && !this.keyDeserializer.equals(keySerializer)) ||
- (this.namespaceDeserializer != null && !this.namespaceDeserializer.equals(namespaceSerializer))) {
- throw new IllegalStateException("Tried to initialize restored TimerService " +
- "with different serializers than those used to snapshot its state.");
+ if (restoredTimersSnapshot != null) {
+ CompatibilityResult<K> keySerializerCompatibility = CompatibilityUtil.resolveCompatibilityResult(
+ this.keyDeserializer,
+ null,
+ restoredTimersSnapshot.getKeySerializerConfigSnapshot(),
+ keySerializer);
+
+ CompatibilityResult<N> namespaceSerializerCompatibility = CompatibilityUtil.resolveCompatibilityResult(
+ this.namespaceDeserializer,
+ null,
+ restoredTimersSnapshot.getNamespaceSerializerConfigSnapshot(),
+ namespaceSerializer);
+
+ if (keySerializerCompatibility.isRequiresMigration() || namespaceSerializerCompatibility.isRequiresMigration()) {
+ throw new IllegalStateException("Tried to initialize restored TimerService " +
+ "with incompatible serializers than those used to snapshot its state.");
+ }
}
this.keySerializer = keySerializer;
@@ -161,8 +174,6 @@ public class HeapInternalTimerService<K, N> implements InternalTimerService<N>,
this.triggerTarget = Preconditions.checkNotNull(triggerTarget);
- this.timerSerializer = new InternalTimer.TimerSerializer<>(this.keySerializer, this.namespaceSerializer);
-
// re-register the restored timers (if any)
if (processingTimeTimersQueue.size() > 0) {
nextTimer = processingTimeService.registerTimer(processingTimeTimersQueue.peek().getTimestamp(), this);
@@ -280,86 +291,53 @@ public class HeapInternalTimerService<K, N> implements InternalTimerService<N>,
/**
* Snapshots the timers (both processing and event time ones) for a given {@code keyGroupIdx}.
- * @param stream the stream to write to.
+ *
* @param keyGroupIdx the id of the key-group to be put in the snapshot.
+ * @return a snapshot containing the timers for the given key-group, and the serializers for them
*/
- public void snapshotTimersForKeyGroup(DataOutputViewStreamWrapper stream, int keyGroupIdx) throws Exception {
- InstantiationUtil.serializeObject(stream, keySerializer);
- InstantiationUtil.serializeObject(stream, namespaceSerializer);
-
- // write the event time timers
- Set<InternalTimer<K, N>> eventTimers = getEventTimeTimerSetForKeyGroup(keyGroupIdx);
- if (eventTimers != null) {
- stream.writeInt(eventTimers.size());
- for (InternalTimer<K, N> timer : eventTimers) {
- this.timerSerializer.serialize(timer, stream);
- }
- } else {
- stream.writeInt(0);
- }
-
- // write the processing time timers
- Set<InternalTimer<K, N>> processingTimers = getProcessingTimeTimerSetForKeyGroup(keyGroupIdx);
- if (processingTimers != null) {
- stream.writeInt(processingTimers.size());
- for (InternalTimer<K, N> timer : processingTimers) {
- this.timerSerializer.serialize(timer, stream);
- }
- } else {
- stream.writeInt(0);
- }
+ public InternalTimersSnapshot<K, N> snapshotTimersForKeyGroup(int keyGroupIdx) {
+ return new InternalTimersSnapshot<>(
+ keySerializer,
+ keySerializer.snapshotConfiguration(),
+ namespaceSerializer,
+ namespaceSerializer.snapshotConfiguration(),
+ getEventTimeTimerSetForKeyGroup(keyGroupIdx),
+ getProcessingTimeTimerSetForKeyGroup(keyGroupIdx));
}
/**
* Restore the timers (both processing and event time ones) for a given {@code keyGroupIdx}.
- * @param stream the stream to read from.
+ *
+ * @param restoredTimersSnapshot the restored snapshot containing the key-group's timers,
+ * and the serializers that were used to write them
* @param keyGroupIdx the id of the key-group to be put in the snapshot.
- * @param userCodeClassLoader the class loader that will be used to deserialize
- * the local key and namespace serializers.
*/
- public void restoreTimersForKeyGroup(DataInputViewStreamWrapper stream, int keyGroupIdx,
- ClassLoader userCodeClassLoader) throws IOException, ClassNotFoundException {
-
- TypeSerializer<K> tmpKeyDeserializer = InstantiationUtil.deserializeObject(stream, userCodeClassLoader);
- TypeSerializer<N> tmpNamespaceDeserializer = InstantiationUtil.deserializeObject(stream, userCodeClassLoader);
+ @SuppressWarnings("unchecked")
+ public void restoreTimersForKeyGroup(InternalTimersSnapshot<?, ?> restoredTimersSnapshot, int keyGroupIdx) throws IOException {
+ this.restoredTimersSnapshot = (InternalTimersSnapshot<K, N>) restoredTimersSnapshot;
- if ((this.keyDeserializer != null && !this.keyDeserializer.equals(tmpKeyDeserializer)) ||
- (this.namespaceDeserializer != null && !this.namespaceDeserializer.equals(tmpNamespaceDeserializer))) {
+ if ((this.keyDeserializer != null && !this.keyDeserializer.equals(restoredTimersSnapshot.getKeySerializer())) ||
+ (this.namespaceDeserializer != null && !this.namespaceDeserializer.equals(restoredTimersSnapshot.getNamespaceSerializer()))) {
throw new IllegalArgumentException("Tried to restore timers " +
"for the same service with different serializers.");
}
- this.keyDeserializer = tmpKeyDeserializer;
- this.namespaceDeserializer = tmpNamespaceDeserializer;
-
- InternalTimer.TimerSerializer<K, N> timerSerializer =
- new InternalTimer.TimerSerializer<>(this.keyDeserializer, this.namespaceDeserializer);
+ this.keyDeserializer = this.restoredTimersSnapshot.getKeySerializer();
+ this.namespaceDeserializer = this.restoredTimersSnapshot.getNamespaceSerializer();
checkArgument(localKeyGroupRange.contains(keyGroupIdx),
"Key Group " + keyGroupIdx + " does not belong to the local range.");
- // read the event time timers
- int sizeOfEventTimeTimers = stream.readInt();
- if (sizeOfEventTimeTimers > 0) {
- Set<InternalTimer<K, N>> eventTimers = getEventTimeTimerSetForKeyGroup(keyGroupIdx);
- for (int i = 0; i < sizeOfEventTimeTimers; i++) {
- InternalTimer<K, N> timer = timerSerializer.deserialize(stream);
- eventTimers.add(timer);
- eventTimeTimersQueue.add(timer);
- }
- }
+ // restore the event time timers
+ Set<InternalTimer<K, N>> eventTimers = getEventTimeTimerSetForKeyGroup(keyGroupIdx);
+ eventTimers.addAll(this.restoredTimersSnapshot.getEventTimeTimers());
+ eventTimeTimersQueue.addAll(this.restoredTimersSnapshot.getEventTimeTimers());
- // read the processing time timers
- int sizeOfProcessingTimeTimers = stream.readInt();
- if (sizeOfProcessingTimeTimers > 0) {
- Set<InternalTimer<K, N>> processingTimers = getProcessingTimeTimerSetForKeyGroup(keyGroupIdx);
- for (int i = 0; i < sizeOfProcessingTimeTimers; i++) {
- InternalTimer<K, N> timer = timerSerializer.deserialize(stream);
- processingTimers.add(timer);
- processingTimeTimersQueue.add(timer);
- }
- }
+ // restore the processing time timers
+ Set<InternalTimer<K, N>> processingTimers = getProcessingTimeTimerSetForKeyGroup(keyGroupIdx);
+ processingTimers.addAll(this.restoredTimersSnapshot.getProcessingTimeTimers());
+ processingTimeTimersQueue.addAll(this.restoredTimersSnapshot.getProcessingTimeTimers());
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/71d682eb/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManager.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManager.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManager.java
index 7d5cb91..84c7bf1 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManager.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManager.java
@@ -21,8 +21,7 @@ package org.apache.flink.streaming.api.operators;
import org.apache.flink.annotation.Internal;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.core.memory.DataInputViewStreamWrapper;
-import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
+import org.apache.flink.core.memory.DataOutputView;
import org.apache.flink.runtime.state.KeyGroupsList;
import org.apache.flink.runtime.state.VoidNamespaceSerializer;
import org.apache.flink.streaming.api.watermark.Watermark;
@@ -30,6 +29,7 @@ import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService;
import org.apache.flink.util.Preconditions;
import java.io.IOException;
+import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;
@@ -111,36 +111,29 @@ public class InternalTimeServiceManager<K, N> {
////////////////// Fault Tolerance Methods ///////////////////
- public void snapshotStateForKeyGroup(DataOutputViewStreamWrapper stream, int keyGroupIdx) throws Exception {
- stream.writeInt(timerServices.size());
+ public void snapshotStateForKeyGroup(DataOutputView stream, int keyGroupIdx) throws IOException {
+ InternalTimerServiceSerializationProxy<K, N> serializationProxy =
+ new InternalTimerServiceSerializationProxy<>(timerServices, keyGroupIdx);
- for (Map.Entry<String, HeapInternalTimerService<K, N>> entry : timerServices.entrySet()) {
- String serviceName = entry.getKey();
- HeapInternalTimerService<?, ?> timerService = entry.getValue();
-
- stream.writeUTF(serviceName);
- timerService.snapshotTimersForKeyGroup(stream, keyGroupIdx);
- }
+ serializationProxy.write(stream);
}
- public void restoreStateForKeyGroup(DataInputViewStreamWrapper stream, int keyGroupIdx,
- ClassLoader userCodeClassLoader) throws IOException, ClassNotFoundException {
-
- int noOfTimerServices = stream.readInt();
- for (int i = 0; i < noOfTimerServices; i++) {
- String serviceName = stream.readUTF();
-
- HeapInternalTimerService<K, N> timerService = timerServices.get(serviceName);
- if (timerService == null) {
- timerService = new HeapInternalTimerService<>(
- totalKeyGroups,
- localKeyGroupRange,
- keyContext,
- processingTimeService);
- timerServices.put(serviceName, timerService);
- }
- timerService.restoreTimersForKeyGroup(stream, keyGroupIdx, userCodeClassLoader);
- }
+ public void restoreStateForKeyGroup(
+ InputStream stream,
+ int keyGroupIdx,
+ ClassLoader userCodeClassLoader) throws IOException {
+
+ InternalTimerServiceSerializationProxy<K, N> serializationProxy =
+ new InternalTimerServiceSerializationProxy<>(
+ timerServices,
+ userCodeClassLoader,
+ totalKeyGroups,
+ localKeyGroupRange,
+ keyContext,
+ processingTimeService,
+ keyGroupIdx);
+
+ serializationProxy.read(stream);
}
//////////////////// Methods used ONLY IN TESTS ////////////////////
http://git-wip-us.apache.org/repos/asf/flink/blob/71d682eb/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimerServiceSerializationProxy.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimerServiceSerializationProxy.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimerServiceSerializationProxy.java
new file mode 100644
index 0000000..53d1507
--- /dev/null
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimerServiceSerializationProxy.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.operators;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.core.io.PostVersionedIOReadableWritable;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.runtime.state.KeyGroupsList;
+import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * Serialization proxy for the timer services for a given key-group.
+ */
+@Internal
+public class InternalTimerServiceSerializationProxy<K, N> extends PostVersionedIOReadableWritable {
+
+ public static final int VERSION = 1;
+
+ /** The key-group timer services to write / read. */
+ private Map<String, HeapInternalTimerService<K, N>> timerServices;
+
+ /** The user classloader; only relevant if the proxy is used to restore timer services. */
+ private ClassLoader userCodeClassLoader;
+
+ /** Properties of restored timer services. */
+ private int keyGroupIdx;
+ private int totalKeyGroups;
+ private KeyGroupsList localKeyGroupRange;
+ private KeyContext keyContext;
+ private ProcessingTimeService processingTimeService;
+
+ /**
+ * Constructor to use when restoring timer services.
+ */
+ public InternalTimerServiceSerializationProxy(
+ Map<String, HeapInternalTimerService<K, N>> timerServicesMapToPopulate,
+ ClassLoader userCodeClassLoader,
+ int totalKeyGroups,
+ KeyGroupsList localKeyGroupRange,
+ KeyContext keyContext,
+ ProcessingTimeService processingTimeService,
+ int keyGroupIdx) {
+
+ this.timerServices = checkNotNull(timerServicesMapToPopulate);
+ this.userCodeClassLoader = checkNotNull(userCodeClassLoader);
+ this.totalKeyGroups = totalKeyGroups;
+ this.localKeyGroupRange = checkNotNull(localKeyGroupRange);
+ this.keyContext = checkNotNull(keyContext);
+ this.processingTimeService = checkNotNull(processingTimeService);
+ this.keyGroupIdx = keyGroupIdx;
+ }
+
+ /**
+ * Constructor to use when writing timer services.
+ */
+ public InternalTimerServiceSerializationProxy(
+ Map<String, HeapInternalTimerService<K, N>> timerServices,
+ int keyGroupIdx) {
+
+ this.timerServices = checkNotNull(timerServices);
+ this.keyGroupIdx = keyGroupIdx;
+ }
+
+ @Override
+ public int getVersion() {
+ return VERSION;
+ }
+
+ @Override
+ public void write(DataOutputView out) throws IOException {
+ super.write(out);
+
+ out.writeInt(timerServices.size());
+ for (Map.Entry<String, HeapInternalTimerService<K, N>> entry : timerServices.entrySet()) {
+ String serviceName = entry.getKey();
+ HeapInternalTimerService<K, N> timerService = entry.getValue();
+
+ out.writeUTF(serviceName);
+ InternalTimersSnapshotReaderWriters
+ .getWriterForVersion(VERSION, timerService.snapshotTimersForKeyGroup(keyGroupIdx))
+ .writeTimersSnapshot(out);
+ }
+ }
+
+ @Override
+ protected void read(DataInputView in, boolean wasVersioned) throws IOException {
+ int noOfTimerServices = in.readInt();
+
+ for (int i = 0; i < noOfTimerServices; i++) {
+ String serviceName = in.readUTF();
+
+ HeapInternalTimerService<K, N> timerService = timerServices.get(serviceName);
+ if (timerService == null) {
+ timerService = new HeapInternalTimerService<>(
+ totalKeyGroups,
+ localKeyGroupRange,
+ keyContext,
+ processingTimeService);
+ timerServices.put(serviceName, timerService);
+ }
+
+ int readerVersion = wasVersioned ? getReadVersion() : InternalTimersSnapshotReaderWriters.NO_VERSION;
+ InternalTimersSnapshot<?, ?> restoredTimersSnapshot = InternalTimersSnapshotReaderWriters
+ .getReaderForVersion(readerVersion, userCodeClassLoader)
+ .readTimersSnapshot(in);
+
+ timerService.restoreTimersForKeyGroup(restoredTimersSnapshot, keyGroupIdx);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/71d682eb/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimersSnapshot.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimersSnapshot.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimersSnapshot.java
new file mode 100644
index 0000000..ddaee6b
--- /dev/null
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimersSnapshot.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.operators;
+
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
+import org.apache.flink.util.Preconditions;
+
+import javax.annotation.Nullable;
+
+import java.util.Set;
+
+/**
+ * A snapshot of internal timers, containing event and processing timers and
+ * the serializers to use to write / read them.
+ */
+public class InternalTimersSnapshot<K, N> {
+
+ private TypeSerializer<K> keySerializer;
+ private TypeSerializerConfigSnapshot keySerializerConfigSnapshot;
+ private TypeSerializer<N> namespaceSerializer;
+ private TypeSerializerConfigSnapshot namespaceSerializerConfigSnapshot;
+
+ private Set<InternalTimer<K, N>> eventTimeTimers;
+ private Set<InternalTimer<K, N>> processingTimeTimers;
+
+ /** Empty constructor used when restoring the timers. */
+ public InternalTimersSnapshot() {}
+
+ /** Constructor to use when snapshotting the timers. */
+ public InternalTimersSnapshot(
+ TypeSerializer<K> keySerializer,
+ TypeSerializerConfigSnapshot keySerializerConfigSnapshot,
+ TypeSerializer<N> namespaceSerializer,
+ TypeSerializerConfigSnapshot namespaceSerializerConfigSnapshot,
+ @Nullable Set<InternalTimer<K, N>> eventTimeTimers,
+ @Nullable Set<InternalTimer<K, N>> processingTimeTimers) {
+
+ this.keySerializer = Preconditions.checkNotNull(keySerializer);
+ this.keySerializerConfigSnapshot = Preconditions.checkNotNull(keySerializerConfigSnapshot);
+ this.namespaceSerializer = Preconditions.checkNotNull(namespaceSerializer);
+ this.namespaceSerializerConfigSnapshot = Preconditions.checkNotNull(namespaceSerializerConfigSnapshot);
+ this.eventTimeTimers = eventTimeTimers;
+ this.processingTimeTimers = processingTimeTimers;
+ }
+
+ public TypeSerializer<K> getKeySerializer() {
+ return keySerializer;
+ }
+
+ public void setKeySerializer(TypeSerializer<K> keySerializer) {
+ this.keySerializer = keySerializer;
+ }
+
+ public TypeSerializerConfigSnapshot getKeySerializerConfigSnapshot() {
+ return keySerializerConfigSnapshot;
+ }
+
+ public void setKeySerializerConfigSnapshot(TypeSerializerConfigSnapshot keySerializerConfigSnapshot) {
+ this.keySerializerConfigSnapshot = keySerializerConfigSnapshot;
+ }
+
+ public TypeSerializer<N> getNamespaceSerializer() {
+ return namespaceSerializer;
+ }
+
+ public void setNamespaceSerializer(TypeSerializer<N> namespaceSerializer) {
+ this.namespaceSerializer = namespaceSerializer;
+ }
+
+ public TypeSerializerConfigSnapshot getNamespaceSerializerConfigSnapshot() {
+ return namespaceSerializerConfigSnapshot;
+ }
+
+ public void setNamespaceSerializerConfigSnapshot(TypeSerializerConfigSnapshot namespaceSerializerConfigSnapshot) {
+ this.namespaceSerializerConfigSnapshot = namespaceSerializerConfigSnapshot;
+ }
+
+ public Set<InternalTimer<K, N>> getEventTimeTimers() {
+ return eventTimeTimers;
+ }
+
+ public void setEventTimeTimers(Set<InternalTimer<K, N>> eventTimeTimers) {
+ this.eventTimeTimers = eventTimeTimers;
+ }
+
+ public Set<InternalTimer<K, N>> getProcessingTimeTimers() {
+ return processingTimeTimers;
+ }
+
+ public void setProcessingTimeTimers(Set<InternalTimer<K, N>> processingTimeTimers) {
+ this.processingTimeTimers = processingTimeTimers;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return super.equals(obj);
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/71d682eb/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimersSnapshotReaderWriters.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimersSnapshotReaderWriters.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimersSnapshotReaderWriters.java
new file mode 100644
index 0000000..3173625
--- /dev/null
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimersSnapshotReaderWriters.java
@@ -0,0 +1,291 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.operators;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
+import org.apache.flink.api.common.typeutils.TypeSerializerSerializationUtil;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.typeutils.runtime.DataInputViewStream;
+import org.apache.flink.core.memory.ByteArrayOutputStreamWithPos;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.util.InstantiationUtil;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * Readers and writers for different versions of the {@link InternalTimersSnapshot}.
+ * Outdated formats are also kept here for documentation of history backlog.
+ */
+@Internal
+public class InternalTimersSnapshotReaderWriters {
+
+ public static final int NO_VERSION = Integer.MIN_VALUE;
+
+ // -------------------------------------------------------------------------------
+ // Writers
+ // - pre-versioned: Flink 1.4.0
+ // - v1: Flink 1.4.1
+ // -------------------------------------------------------------------------------
+
+ public static <K, N> InternalTimersSnapshotWriter getWriterForVersion(int version, InternalTimersSnapshot<K, N> timersSnapshot) {
+
+ switch (version) {
+ case NO_VERSION:
+ return new InternalTimersSnapshotWriterPreVersioned<>(timersSnapshot);
+
+ case InternalTimerServiceSerializationProxy.VERSION:
+ return new InternalTimersSnapshotWriterV1<>(timersSnapshot);
+
+ default:
+ // guard for future
+ throw new IllegalStateException(
+ "Unrecognized internal timers snapshot writer version: " + version);
+ }
+ }
+
+ /**
+ * A writer for a {@link InternalTimersSnapshot}.
+ */
+ public interface InternalTimersSnapshotWriter {
+
+ /**
+ * Writes the timers snapshot to the output view.
+ *
+ * @param out the output view to write to
+ * @throws IOException
+ */
+ void writeTimersSnapshot(DataOutputView out) throws IOException;
+ }
+
+ private abstract static class AbstractInternalTimersSnapshotWriter<K, N> implements InternalTimersSnapshotWriter {
+
+ protected final InternalTimersSnapshot<K, N> timersSnapshot;
+
+ public AbstractInternalTimersSnapshotWriter(InternalTimersSnapshot<K, N> timersSnapshot) {
+ this.timersSnapshot = checkNotNull(timersSnapshot);
+ }
+
+ protected abstract void writeKeyAndNamespaceSerializers(DataOutputView out) throws IOException;
+
+ @Override
+ public final void writeTimersSnapshot(DataOutputView out) throws IOException {
+ writeKeyAndNamespaceSerializers(out);
+
+ InternalTimer.TimerSerializer<K, N> timerSerializer = new InternalTimer.TimerSerializer<>(
+ timersSnapshot.getKeySerializer(),
+ timersSnapshot.getNamespaceSerializer());
+
+ // write the event time timers
+ Set<InternalTimer<K, N>> eventTimers = timersSnapshot.getEventTimeTimers();
+ if (eventTimers != null) {
+ out.writeInt(eventTimers.size());
+ for (InternalTimer<K, N> eventTimer : eventTimers) {
+ timerSerializer.serialize(eventTimer, out);
+ }
+ } else {
+ out.writeInt(0);
+ }
+
+ // write the processing time timers
+ Set<InternalTimer<K, N>> processingTimers = timersSnapshot.getProcessingTimeTimers();
+ if (processingTimers != null) {
+ out.writeInt(processingTimers.size());
+ for (InternalTimer<K, N> processingTimer : processingTimers) {
+ timerSerializer.serialize(processingTimer, out);
+ }
+ } else {
+ out.writeInt(0);
+ }
+ }
+ }
+
+ private static class InternalTimersSnapshotWriterPreVersioned<K, N> extends AbstractInternalTimersSnapshotWriter<K, N> {
+
+ public InternalTimersSnapshotWriterPreVersioned(InternalTimersSnapshot<K, N> timersSnapshot) {
+ super(timersSnapshot);
+ }
+
+ @Override
+ protected void writeKeyAndNamespaceSerializers(DataOutputView out) throws IOException {
+ // the pre-versioned format only serializes the serializers, without their configuration snapshots
+ try (ByteArrayOutputStreamWithPos stream = new ByteArrayOutputStreamWithPos()) {
+ InstantiationUtil.serializeObject(stream, timersSnapshot.getKeySerializer());
+ InstantiationUtil.serializeObject(stream, timersSnapshot.getNamespaceSerializer());
+
+ out.write(stream.getBuf(), 0, stream.getPosition());
+ }
+ }
+ }
+
+ private static class InternalTimersSnapshotWriterV1<K, N> extends AbstractInternalTimersSnapshotWriter<K, N> {
+
+ public InternalTimersSnapshotWriterV1(InternalTimersSnapshot<K, N> timersSnapshot) {
+ super(timersSnapshot);
+ }
+
+ @Override
+ protected void writeKeyAndNamespaceSerializers(DataOutputView out) throws IOException {
+ // write key / namespace serializers, and their configuration snapshots
+ TypeSerializerSerializationUtil.writeSerializersAndConfigsWithResilience(
+ out,
+ Arrays.asList(
+ Tuple2.of(timersSnapshot.getKeySerializer(), timersSnapshot.getKeySerializerConfigSnapshot()),
+ Tuple2.of(timersSnapshot.getNamespaceSerializer(), timersSnapshot.getNamespaceSerializerConfigSnapshot())));
+ }
+ }
+
+ // -------------------------------------------------------------------------------
+ // Readers
+ // - pre-versioned: Flink 1.4.0
+ // - v1: Flink 1.4.1
+ // -------------------------------------------------------------------------------
+
+ public static <K, N> InternalTimersSnapshotReader<K, N> getReaderForVersion(
+ int version, ClassLoader userCodeClassLoader) {
+
+ switch (version) {
+ case NO_VERSION:
+ return new InternalTimersSnapshotReaderPreVersioned<>(userCodeClassLoader);
+
+ case InternalTimerServiceSerializationProxy.VERSION:
+ return new InternalTimersSnapshotReaderV1<>(userCodeClassLoader);
+
+ default:
+ // guard for future
+ throw new IllegalStateException(
+ "Unrecognized internal timers snapshot writer version: " + version);
+ }
+ }
+
+ /**
+ * A reader for a {@link InternalTimersSnapshot}.
+ */
+ public interface InternalTimersSnapshotReader<K, N> {
+
+ /**
+ * Reads a timers snapshot from the provided input view.
+ *
+ * @param in the input view
+ * @return the read timers snapshot
+ * @throws IOException
+ */
+ InternalTimersSnapshot<K, N> readTimersSnapshot(DataInputView in) throws IOException;
+ }
+
+ private abstract static class AbstractInternalTimersSnapshotReader<K, N> implements InternalTimersSnapshotReader<K, N> {
+
+ protected final ClassLoader userCodeClassLoader;
+
+ public AbstractInternalTimersSnapshotReader(ClassLoader userCodeClassLoader) {
+ this.userCodeClassLoader = checkNotNull(userCodeClassLoader);
+ }
+
+ protected abstract void restoreKeyAndNamespaceSerializers(
+ InternalTimersSnapshot<K, N> restoredTimersSnapshot,
+ DataInputView in) throws IOException;
+
+ @Override
+ public final InternalTimersSnapshot<K, N> readTimersSnapshot(DataInputView in) throws IOException {
+ InternalTimersSnapshot<K, N> restoredTimersSnapshot = new InternalTimersSnapshot<>();
+
+ restoreKeyAndNamespaceSerializers(restoredTimersSnapshot, in);
+
+ InternalTimer.TimerSerializer<K, N> timerSerializer = new InternalTimer.TimerSerializer<>(
+ restoredTimersSnapshot.getKeySerializer(),
+ restoredTimersSnapshot.getNamespaceSerializer());
+
+ // read the event time timers
+ int sizeOfEventTimeTimers = in.readInt();
+ Set<InternalTimer<K, N>> restoredEventTimers = new HashSet<>(sizeOfEventTimeTimers);
+ if (sizeOfEventTimeTimers > 0) {
+ for (int i = 0; i < sizeOfEventTimeTimers; i++) {
+ InternalTimer<K, N> timer = timerSerializer.deserialize(in);
+ restoredEventTimers.add(timer);
+ }
+ }
+ restoredTimersSnapshot.setEventTimeTimers(restoredEventTimers);
+
+ // read the processing time timers
+ int sizeOfProcessingTimeTimers = in.readInt();
+ Set<InternalTimer<K, N>> restoredProcessingTimers = new HashSet<>(sizeOfProcessingTimeTimers);
+ if (sizeOfProcessingTimeTimers > 0) {
+ for (int i = 0; i < sizeOfProcessingTimeTimers; i++) {
+ InternalTimer<K, N> timer = timerSerializer.deserialize(in);
+ restoredProcessingTimers.add(timer);
+ }
+ }
+ restoredTimersSnapshot.setProcessingTimeTimers(restoredProcessingTimers);
+
+ return restoredTimersSnapshot;
+ }
+ }
+
+ private static class InternalTimersSnapshotReaderPreVersioned<K, N> extends AbstractInternalTimersSnapshotReader<K, N> {
+
+ public InternalTimersSnapshotReaderPreVersioned(ClassLoader userCodeClassLoader) {
+ super(userCodeClassLoader);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ protected void restoreKeyAndNamespaceSerializers(
+ InternalTimersSnapshot<K, N> restoredTimersSnapshot,
+ DataInputView in) throws IOException {
+
+ DataInputViewStream dis = new DataInputViewStream(in);
+ try {
+ restoredTimersSnapshot.setKeySerializer(InstantiationUtil.deserializeObject(dis, userCodeClassLoader, true));
+ restoredTimersSnapshot.setNamespaceSerializer(InstantiationUtil.deserializeObject(dis, userCodeClassLoader, true));
+ } catch (ClassNotFoundException exception) {
+ throw new IOException(exception);
+ }
+ }
+ }
+
+ private static class InternalTimersSnapshotReaderV1<K, N> extends AbstractInternalTimersSnapshotReader<K, N> {
+
+ public InternalTimersSnapshotReaderV1(ClassLoader userCodeClassLoader) {
+ super(userCodeClassLoader);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ protected void restoreKeyAndNamespaceSerializers(
+ InternalTimersSnapshot<K, N> restoredTimersSnapshot,
+ DataInputView in) throws IOException {
+
+ List<Tuple2<TypeSerializer<?>, TypeSerializerConfigSnapshot>> serializersAndConfigs =
+ TypeSerializerSerializationUtil.readSerializersAndConfigsWithResilience(in, userCodeClassLoader);
+
+ restoredTimersSnapshot.setKeySerializer((TypeSerializer<K>) serializersAndConfigs.get(0).f0);
+ restoredTimersSnapshot.setKeySerializerConfigSnapshot(serializersAndConfigs.get(0).f1);
+ restoredTimersSnapshot.setNamespaceSerializer((TypeSerializer<N>) serializersAndConfigs.get(1).f0);
+ restoredTimersSnapshot.setNamespaceSerializerConfigSnapshot(serializersAndConfigs.get(1).f1);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/71d682eb/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/HeapInternalTimerServiceTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/HeapInternalTimerServiceTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/HeapInternalTimerServiceTest.java
index 5a4f1c4..aade876 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/HeapInternalTimerServiceTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/HeapInternalTimerServiceTest.java
@@ -548,13 +548,36 @@ public class HeapInternalTimerServiceTest {
@Test
public void testSnapshotAndRestore() throws Exception {
+ testSnapshotAndRestore(InternalTimerServiceSerializationProxy.VERSION);
+ }
+
+ @Test
+ public void testSnapshotAndRestorePreVersioned() throws Exception {
+ testSnapshotAndRestore(InternalTimersSnapshotReaderWriters.NO_VERSION);
+ }
+
+ /**
+ * This test checks whether timers are assigned to correct key groups
+ * and whether snapshot/restore respects key groups.
+ */
+ @Test
+ public void testSnapshotAndRebalancingRestore() throws Exception {
+ testSnapshotAndRebalancingRestore(InternalTimerServiceSerializationProxy.VERSION);
+ }
+
+ @Test
+ public void testSnapshotAndRebalancingRestorePreVersioned() throws Exception {
+ testSnapshotAndRebalancingRestore(InternalTimersSnapshotReaderWriters.NO_VERSION);
+ }
+
+ private void testSnapshotAndRestore(int snapshotVersion) throws Exception {
@SuppressWarnings("unchecked")
Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class);
TestKeyContext keyContext = new TestKeyContext();
TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
HeapInternalTimerService<Integer, String> timerService =
- createTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, maxParallelism);
+ createTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, maxParallelism);
// get two different keys
int key1 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
@@ -582,10 +605,15 @@ public class HeapInternalTimerServiceTest {
Map<Integer, byte[]> snapshot = new HashMap<>();
for (Integer keyGroupIndex : testKeyGroupRange) {
- ByteArrayOutputStream outStream = new ByteArrayOutputStream();
- timerService.snapshotTimersForKeyGroup(new DataOutputViewStreamWrapper(outStream), keyGroupIndex);
- outStream.close();
- snapshot.put(keyGroupIndex, outStream.toByteArray());
+ try (ByteArrayOutputStream outStream = new ByteArrayOutputStream()) {
+ InternalTimersSnapshot<?, ?> timersSnapshot = timerService.snapshotTimersForKeyGroup(keyGroupIndex);
+
+ InternalTimersSnapshotReaderWriters
+ .getWriterForVersion(snapshotVersion, timersSnapshot)
+ .writeTimersSnapshot(new DataOutputViewStreamWrapper(outStream));
+
+ snapshot.put(keyGroupIndex, outStream.toByteArray());
+ }
}
@SuppressWarnings("unchecked")
@@ -595,12 +623,13 @@ public class HeapInternalTimerServiceTest {
processingTimeService = new TestProcessingTimeService();
timerService = restoreTimerService(
- snapshot,
- mockTriggerable2,
- keyContext,
- processingTimeService,
- testKeyGroupRange,
- maxParallelism);
+ snapshot,
+ snapshotVersion,
+ mockTriggerable2,
+ keyContext,
+ processingTimeService,
+ testKeyGroupRange,
+ maxParallelism);
processingTimeService.setCurrentTime(10);
timerService.advanceWatermark(10);
@@ -615,22 +644,17 @@ public class HeapInternalTimerServiceTest {
assertEquals(0, timerService.numEventTimeTimers());
}
- /**
- * This test checks whether timers are assigned to correct key groups
- * and whether snapshot/restore respects key groups.
- */
- @Test
- public void testSnapshotAndRebalancingRestore() throws Exception {
+ private void testSnapshotAndRebalancingRestore(int snapshotVersion) throws Exception {
@SuppressWarnings("unchecked")
Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class);
TestKeyContext keyContext = new TestKeyContext();
TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
HeapInternalTimerService<Integer, String> timerService =
- createTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, maxParallelism);
+ createTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, maxParallelism);
int midpoint = testKeyGroupRange.getStartKeyGroup() +
- (testKeyGroupRange.getEndKeyGroup() - testKeyGroupRange.getStartKeyGroup()) / 2;
+ (testKeyGroupRange.getEndKeyGroup() - testKeyGroupRange.getStartKeyGroup()) / 2;
// get two sub key-ranges so that we can restore two ranges separately
KeyGroupRange subKeyGroupRange1 = new KeyGroupRange(testKeyGroupRange.getStartKeyGroup(), midpoint);
@@ -661,15 +685,20 @@ public class HeapInternalTimerServiceTest {
Map<Integer, byte[]> snapshot1 = new HashMap<>();
Map<Integer, byte[]> snapshot2 = new HashMap<>();
for (Integer keyGroupIndex : testKeyGroupRange) {
- ByteArrayOutputStream outStream = new ByteArrayOutputStream();
- timerService.snapshotTimersForKeyGroup(new DataOutputViewStreamWrapper(outStream), keyGroupIndex);
- outStream.close();
- if (subKeyGroupRange1.contains(keyGroupIndex)) {
- snapshot1.put(keyGroupIndex, outStream.toByteArray());
- } else if (subKeyGroupRange2.contains(keyGroupIndex)) {
- snapshot2.put(keyGroupIndex, outStream.toByteArray());
- } else {
- throw new IllegalStateException("Key-Group index doesn't belong to any sub range.");
+ try (ByteArrayOutputStream outStream = new ByteArrayOutputStream()) {
+ InternalTimersSnapshot<?, ?> timersSnapshot = timerService.snapshotTimersForKeyGroup(keyGroupIndex);
+
+ InternalTimersSnapshotReaderWriters
+ .getWriterForVersion(snapshotVersion, timersSnapshot)
+ .writeTimersSnapshot(new DataOutputViewStreamWrapper(outStream));
+
+ if (subKeyGroupRange1.contains(keyGroupIndex)) {
+ snapshot1.put(keyGroupIndex, outStream.toByteArray());
+ } else if (subKeyGroupRange2.contains(keyGroupIndex)) {
+ snapshot2.put(keyGroupIndex, outStream.toByteArray());
+ } else {
+ throw new IllegalStateException("Key-Group index doesn't belong to any sub range.");
+ }
}
}
@@ -687,20 +716,22 @@ public class HeapInternalTimerServiceTest {
TestProcessingTimeService processingTimeService2 = new TestProcessingTimeService();
HeapInternalTimerService<Integer, String> timerService1 = restoreTimerService(
- snapshot1,
- mockTriggerable1,
- keyContext1,
- processingTimeService1,
- subKeyGroupRange1,
- maxParallelism);
+ snapshot1,
+ snapshotVersion,
+ mockTriggerable1,
+ keyContext1,
+ processingTimeService1,
+ subKeyGroupRange1,
+ maxParallelism);
HeapInternalTimerService<Integer, String> timerService2 = restoreTimerService(
- snapshot2,
- mockTriggerable2,
- keyContext2,
- processingTimeService2,
- subKeyGroupRange2,
- maxParallelism);
+ snapshot2,
+ snapshotVersion,
+ mockTriggerable2,
+ keyContext2,
+ processingTimeService2,
+ subKeyGroupRange2,
+ maxParallelism);
processingTimeService1.setCurrentTime(10);
timerService1.advanceWatermark(10);
@@ -779,6 +810,7 @@ public class HeapInternalTimerServiceTest {
private static HeapInternalTimerService<Integer, String> restoreTimerService(
Map<Integer, byte[]> state,
+ int snapshotVersion,
Triggerable<Integer, String> triggerable,
KeyContext keyContext,
ProcessingTimeService processingTimeService,
@@ -796,10 +828,14 @@ public class HeapInternalTimerServiceTest {
// restore the timers
for (Integer keyGroupIndex : keyGroupsList) {
if (state.containsKey(keyGroupIndex)) {
- service.restoreTimersForKeyGroup(
- new DataInputViewStreamWrapper(new ByteArrayInputStream(state.get(keyGroupIndex))),
- keyGroupIndex,
- HeapInternalTimerServiceTest.class.getClassLoader());
+ try (ByteArrayInputStream inputStream = new ByteArrayInputStream(state.get(keyGroupIndex))) {
+ InternalTimersSnapshot<?, ?> restoredTimersSnapshot =
+ InternalTimersSnapshotReaderWriters
+ .getReaderForVersion(snapshotVersion, HeapInternalTimerServiceTest.class.getClassLoader())
+ .readTimersSnapshot(new DataInputViewStreamWrapper(inputStream));
+
+ service.restoreTimersForKeyGroup(restoredTimersSnapshot, keyGroupIndex);
+ }
}
}
[28/28] flink git commit: [FLINK-8275] [security,
yarn] Remove test-specific code path in YarnTaskManagerRunner
Posted by tz...@apache.org.
[FLINK-8275] [security, yarn] Remove test-specific code path in YarnTaskManagerRunner
Previously, the YarnTaskManagerRunner contained a code path that exists
for the sole purpose of injecting mock runners. Having code paths just
to utilize tests in production code is in general a bad idea.
This commit fixes this be making YarnTaskManagerRunner a factory-like
class, which creates a Runner that contains all the runner’s properties,
such as configuration. Unit tests can than test against the contained
configuration in the created Runner to validate that everything is
configured properly.
This closes #5172.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/82f39578
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/82f39578
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/82f39578
Branch: refs/heads/release-1.4
Commit: 82f3957811d2e5bcefabaa42326d3d4476e45df0
Parents: 2014665
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Fri Feb 2 15:53:13 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 18:30:32 2018 +0100
----------------------------------------------------------------------
.../yarn/TestingYarnTaskManagerRunner.java | 12 +-
.../flink/yarn/TestingYarnTaskManager.scala | 14 +-
.../flink/yarn/YarnApplicationMasterRunner.java | 8 +-
.../flink/yarn/YarnTaskManagerRunner.java | 166 ----------------
.../yarn/YarnTaskManagerRunnerFactory.java | 187 +++++++++++++++++++
.../org/apache/flink/yarn/YarnTaskManager.scala | 30 ++-
.../yarn/YarnTaskManagerRunnerFactoryTest.java | 76 ++++++++
.../flink/yarn/YarnTaskManagerRunnerTest.java | 74 --------
8 files changed, 313 insertions(+), 254 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/82f39578/flink-yarn-tests/src/test/java/org/apache/flink/yarn/TestingYarnTaskManagerRunner.java
----------------------------------------------------------------------
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/TestingYarnTaskManagerRunner.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/TestingYarnTaskManagerRunner.java
index 90cdf9f..ff030be 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/TestingYarnTaskManagerRunner.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/TestingYarnTaskManagerRunner.java
@@ -18,6 +18,8 @@
package org.apache.flink.yarn;
+import org.apache.flink.runtime.security.SecurityUtils;
+
import java.io.IOException;
/**
@@ -25,7 +27,13 @@ import java.io.IOException;
*/
public class TestingYarnTaskManagerRunner {
public static void main(String[] args) throws IOException {
- YarnTaskManagerRunner.runYarnTaskManager(
- args, TestingYarnTaskManager.class, System.getenv(), null);
+ YarnTaskManagerRunnerFactory.Runner tmRunner = YarnTaskManagerRunnerFactory.create(
+ args, TestingYarnTaskManager.class, System.getenv());
+
+ try {
+ SecurityUtils.getInstalledContext().runSecured(tmRunner);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/82f39578/flink-yarn-tests/src/test/scala/org/apache/flink/yarn/TestingYarnTaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-yarn-tests/src/test/scala/org/apache/flink/yarn/TestingYarnTaskManager.scala b/flink-yarn-tests/src/test/scala/org/apache/flink/yarn/TestingYarnTaskManager.scala
index a194e79..ed2b2f9 100644
--- a/flink-yarn-tests/src/test/scala/org/apache/flink/yarn/TestingYarnTaskManager.scala
+++ b/flink-yarn-tests/src/test/scala/org/apache/flink/yarn/TestingYarnTaskManager.scala
@@ -24,6 +24,7 @@ import org.apache.flink.runtime.io.disk.iomanager.IOManager
import org.apache.flink.runtime.io.network.NetworkEnvironment
import org.apache.flink.runtime.memory.MemoryManager
import org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup
+import org.apache.flink.runtime.security.SecurityUtils
import org.apache.flink.runtime.taskexecutor.TaskManagerConfiguration
import org.apache.flink.runtime.taskmanager.TaskManagerLocation
import org.apache.flink.runtime.testingUtils.TestingTaskManagerLike
@@ -68,12 +69,19 @@ class TestingYarnTaskManager(
object YarnTaskManager {
/** Entry point (main method) to run the TaskManager on YARN.
- *
+ *
* @param args The command line arguments.
*/
def main(args: Array[String]): Unit = {
- YarnTaskManagerRunner.runYarnTaskManager(
- args, classOf[TestingYarnTaskManager], System.getenv(), null)
+ val tmRunner = YarnTaskManagerRunnerFactory.create(
+ args, classOf[TestingYarnTaskManager], System.getenv())
+
+ try {
+ SecurityUtils.getInstalledContext.runSecured(tmRunner)
+ } catch {
+ case e: Exception =>
+ throw new RuntimeException(e)
+ }
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/82f39578/flink-yarn/src/main/java/org/apache/flink/yarn/YarnApplicationMasterRunner.java
----------------------------------------------------------------------
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnApplicationMasterRunner.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnApplicationMasterRunner.java
index 1c18e90..462682f 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnApplicationMasterRunner.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnApplicationMasterRunner.java
@@ -162,6 +162,9 @@ public class YarnApplicationMasterRunner {
File f = new File(currDir, Utils.KEYTAB_FILE_NAME);
if (remoteKeytabPrincipal != null && f.exists()) {
+ String keytabPath = f.getAbsolutePath();
+ LOG.debug("keytabPath: {}", keytabPath);
+
// set keytab principal and replace path with the local path of the shipped keytab file in NodeManager
flinkConfig.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, f.getAbsolutePath());
flinkConfig.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL, remoteKeytabPrincipal);
@@ -247,7 +250,10 @@ public class YarnApplicationMasterRunner {
File f = new File(currDir, Utils.KEYTAB_FILE_NAME);
if (remoteKeytabPrincipal != null && f.exists()) {
- config.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, f.getAbsolutePath());
+ String keytabPath = f.getAbsolutePath();
+ LOG.debug("keytabPath: {}", keytabPath);
+
+ config.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, keytabPath);
config.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL, remoteKeytabPrincipal);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/82f39578/flink-yarn/src/main/java/org/apache/flink/yarn/YarnTaskManagerRunner.java
----------------------------------------------------------------------
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnTaskManagerRunner.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnTaskManagerRunner.java
deleted file mode 100644
index d1eb1ed..0000000
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnTaskManagerRunner.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.yarn;
-
-import org.apache.flink.configuration.AkkaOptions;
-import org.apache.flink.configuration.ConfigConstants;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.configuration.SecurityOptions;
-import org.apache.flink.runtime.clusterframework.types.ResourceID;
-import org.apache.flink.runtime.security.SecurityConfiguration;
-import org.apache.flink.runtime.security.SecurityUtils;
-import org.apache.flink.runtime.security.modules.HadoopModule;
-import org.apache.flink.runtime.taskmanager.TaskManager;
-import org.apache.flink.runtime.util.EnvironmentInformation;
-import org.apache.flink.runtime.util.JvmShutdownSafeguard;
-import org.apache.flink.runtime.util.SignalHandler;
-import org.apache.flink.util.Preconditions;
-
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Map;
-import java.util.concurrent.Callable;
-
-/**
- * The entry point for running a TaskManager in a YARN container.
- */
-public class YarnTaskManagerRunner {
-
- private static final Logger LOG = LoggerFactory.getLogger(YarnTaskManagerRunner.class);
-
- private static Callable<Object> createMainRunner(
- Configuration configuration,
- ResourceID resourceId,
- final Class<? extends YarnTaskManager> taskManager) {
- return new Callable<Object>() {
- @Override
- public Integer call() {
- try {
- TaskManager.selectNetworkInterfaceAndRunTaskManager(
- configuration, resourceId, taskManager);
- } catch (Throwable t) {
- LOG.error("Error while starting the TaskManager", t);
- System.exit(TaskManager.STARTUP_FAILURE_RETURN_CODE());
- }
- return null;
- }
- };
- }
-
- public static void runYarnTaskManager(String[] args,
- final Class<? extends YarnTaskManager> taskManager,
- Map<String, String> envs,
- Callable<Object> mainRunner) throws IOException {
- EnvironmentInformation.logEnvironmentInfo(LOG, "YARN TaskManager", args);
- SignalHandler.register(LOG);
- JvmShutdownSafeguard.installAsShutdownHook(LOG);
-
- // try to parse the command line arguments
- final Configuration configuration;
- try {
- configuration = TaskManager.parseArgsAndLoadConfig(args);
- }
- catch (Throwable t) {
- LOG.error(t.getMessage(), t);
- System.exit(TaskManager.STARTUP_FAILURE_RETURN_CODE());
- return;
- }
-
- // read the environment variables for YARN
- final String yarnClientUsername = envs.get(YarnConfigKeys.ENV_HADOOP_USER_NAME);
- final String localDirs = envs.get(Environment.LOCAL_DIRS.key());
- LOG.info("Current working/local Directory: {}", localDirs);
-
- final String currDir = envs.get(Environment.PWD.key());
- LOG.info("Current working Directory: {}", currDir);
-
- final String remoteKeytabPrincipal = envs.get(YarnConfigKeys.KEYTAB_PRINCIPAL);
- LOG.info("TM: remoteKeytabPrincipal obtained {}", remoteKeytabPrincipal);
-
- // configure local directory
- String flinkTempDirs = configuration.getString(ConfigConstants.TASK_MANAGER_TMP_DIR_KEY, null);
- if (flinkTempDirs == null) {
- LOG.info("Setting directories for temporary file " + localDirs);
- configuration.setString(ConfigConstants.TASK_MANAGER_TMP_DIR_KEY, localDirs);
- }
- else {
- LOG.info("Overriding YARN's temporary file directories with those " +
- "specified in the Flink config: " + flinkTempDirs);
- }
-
- // tell akka to die in case of an error
- configuration.setBoolean(AkkaOptions.JVM_EXIT_ON_FATAL_ERROR, true);
-
- UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
-
- LOG.info("YARN daemon is running as: {} Yarn client user obtainer: {}",
- currentUser.getShortUserName(), yarnClientUsername);
-
- // Infer the resource identifier from the environment variable
- String containerID = Preconditions.checkNotNull(envs.get(YarnFlinkResourceManager.ENV_FLINK_CONTAINER_ID));
- final ResourceID resourceId = new ResourceID(containerID);
- LOG.info("ResourceID assigned for this container: {}", resourceId);
-
- File f = new File(currDir, Utils.KEYTAB_FILE_NAME);
- if (remoteKeytabPrincipal != null && f.exists()) {
- // set keytab principal and replace path with the local path of the shipped keytab file in NodeManager
- configuration.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, f.getAbsolutePath());
- configuration.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL, remoteKeytabPrincipal);
- }
- try {
-
- SecurityConfiguration sc;
-
- //To support Yarn Secure Integration Test Scenario
- File krb5Conf = new File(currDir, Utils.KRB5_FILE_NAME);
- if (krb5Conf.exists() && krb5Conf.canRead()) {
- String krb5Path = krb5Conf.getAbsolutePath();
- LOG.info("KRB5 Conf: {}", krb5Path);
- org.apache.hadoop.conf.Configuration hadoopConfiguration = new org.apache.hadoop.conf.Configuration();
- hadoopConfiguration.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
- hadoopConfiguration.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, "true");
-
- sc = new SecurityConfiguration(configuration,
- Collections.singletonList(securityConfig -> new HadoopModule(securityConfig, hadoopConfiguration)));
-
- } else {
- sc = new SecurityConfiguration(configuration);
-
- }
-
- SecurityUtils.install(sc);
-
- if (mainRunner == null) {
- mainRunner = createMainRunner(configuration, resourceId, taskManager);
- }
- SecurityUtils.getInstalledContext().runSecured(mainRunner);
- } catch (Exception e) {
- LOG.error("Exception occurred while launching Task Manager", e);
- throw new RuntimeException(e);
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/82f39578/flink-yarn/src/main/java/org/apache/flink/yarn/YarnTaskManagerRunnerFactory.java
----------------------------------------------------------------------
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnTaskManagerRunnerFactory.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnTaskManagerRunnerFactory.java
new file mode 100644
index 0000000..eef176b
--- /dev/null
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnTaskManagerRunnerFactory.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.yarn;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.configuration.AkkaOptions;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.SecurityOptions;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.security.SecurityConfiguration;
+import org.apache.flink.runtime.security.SecurityUtils;
+import org.apache.flink.runtime.security.modules.HadoopModule;
+import org.apache.flink.runtime.taskmanager.TaskManager;
+import org.apache.flink.runtime.util.EnvironmentInformation;
+import org.apache.flink.runtime.util.JvmShutdownSafeguard;
+import org.apache.flink.runtime.util.SignalHandler;
+import org.apache.flink.util.Preconditions;
+
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.Callable;
+
+/**
+ * The entry point for running a TaskManager in a YARN container.
+ */
+public class YarnTaskManagerRunnerFactory {
+
+ private static final Logger LOG = LoggerFactory.getLogger(YarnTaskManagerRunnerFactory.class);
+
+ /**
+ * Runner for a {@link YarnTaskManager}.
+ */
+ public static class Runner implements Callable<Object> {
+
+ private final Configuration configuration;
+ private final ResourceID resourceId;
+ private final Class<? extends YarnTaskManager> taskManager;
+
+ Runner(Configuration configuration, ResourceID resourceId, Class<? extends YarnTaskManager> taskManager) {
+ this.configuration = Preconditions.checkNotNull(configuration);
+ this.resourceId = Preconditions.checkNotNull(resourceId);
+ this.taskManager = Preconditions.checkNotNull(taskManager);
+ }
+
+ @Override
+ public Object call() throws Exception {
+ try {
+ TaskManager.selectNetworkInterfaceAndRunTaskManager(
+ configuration, resourceId, taskManager);
+ } catch (Throwable t) {
+ LOG.error("Error while starting the TaskManager", t);
+ System.exit(TaskManager.STARTUP_FAILURE_RETURN_CODE());
+ }
+ return null;
+ }
+
+ @VisibleForTesting
+ Configuration getConfiguration() {
+ return configuration;
+ }
+
+ @VisibleForTesting
+ ResourceID getResourceId() {
+ return resourceId;
+ }
+ }
+
+ /**
+ * Creates a {@link YarnTaskManagerRunnerFactory.Runner}.
+ */
+ public static Runner create(
+ String[] args,
+ final Class<? extends YarnTaskManager> taskManager,
+ Map<String, String> envs) throws IOException {
+
+ EnvironmentInformation.logEnvironmentInfo(LOG, "YARN TaskManager", args);
+ SignalHandler.register(LOG);
+ JvmShutdownSafeguard.installAsShutdownHook(LOG);
+
+ // try to parse the command line arguments
+ final Configuration configuration;
+ try {
+ configuration = TaskManager.parseArgsAndLoadConfig(args);
+ }
+ catch (Throwable t) {
+ LOG.error(t.getMessage(), t);
+ System.exit(TaskManager.STARTUP_FAILURE_RETURN_CODE());
+ return null;
+ }
+
+ // read the environment variables for YARN
+ final String yarnClientUsername = envs.get(YarnConfigKeys.ENV_HADOOP_USER_NAME);
+ final String localDirs = envs.get(Environment.LOCAL_DIRS.key());
+ LOG.info("Current working/local Directory: {}", localDirs);
+
+ final String currDir = envs.get(Environment.PWD.key());
+ LOG.info("Current working Directory: {}", currDir);
+
+ final String remoteKeytabPrincipal = envs.get(YarnConfigKeys.KEYTAB_PRINCIPAL);
+ LOG.info("TM: remoteKeytabPrincipal obtained {}", remoteKeytabPrincipal);
+
+ // configure local directory
+ String flinkTempDirs = configuration.getString(ConfigConstants.TASK_MANAGER_TMP_DIR_KEY, null);
+ if (flinkTempDirs == null) {
+ LOG.info("Setting directories for temporary file " + localDirs);
+ configuration.setString(ConfigConstants.TASK_MANAGER_TMP_DIR_KEY, localDirs);
+ }
+ else {
+ LOG.info("Overriding YARN's temporary file directories with those " +
+ "specified in the Flink config: " + flinkTempDirs);
+ }
+
+ // tell akka to die in case of an error
+ configuration.setBoolean(AkkaOptions.JVM_EXIT_ON_FATAL_ERROR, true);
+
+ UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
+
+ LOG.info("YARN daemon is running as: {} Yarn client user obtainer: {}",
+ currentUser.getShortUserName(), yarnClientUsername);
+
+ // Infer the resource identifier from the environment variable
+ String containerID = Preconditions.checkNotNull(envs.get(YarnFlinkResourceManager.ENV_FLINK_CONTAINER_ID));
+ final ResourceID resourceId = new ResourceID(containerID);
+ LOG.info("ResourceID assigned for this container: {}", resourceId);
+
+ File f = new File(currDir, Utils.KEYTAB_FILE_NAME);
+ if (remoteKeytabPrincipal != null && f.exists()) {
+ // set keytab principal and replace path with the local path of the shipped keytab file in NodeManager
+ configuration.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, f.getAbsolutePath());
+ configuration.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL, remoteKeytabPrincipal);
+ }
+
+ try {
+ SecurityConfiguration sc;
+
+ //To support Yarn Secure Integration Test Scenario
+ File krb5Conf = new File(currDir, Utils.KRB5_FILE_NAME);
+ if (krb5Conf.exists() && krb5Conf.canRead()) {
+ String krb5Path = krb5Conf.getAbsolutePath();
+ LOG.info("KRB5 Conf: {}", krb5Path);
+ org.apache.hadoop.conf.Configuration hadoopConfiguration = new org.apache.hadoop.conf.Configuration();
+ hadoopConfiguration.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
+ hadoopConfiguration.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, "true");
+
+ sc = new SecurityConfiguration(configuration,
+ Collections.singletonList(securityConfig -> new HadoopModule(securityConfig, hadoopConfiguration)));
+
+ } else {
+ sc = new SecurityConfiguration(configuration);
+
+ }
+
+ SecurityUtils.install(sc);
+
+ return new Runner(configuration, resourceId, taskManager);
+ } catch (Exception e) {
+ LOG.error("Exception occurred while building Task Manager runner", e);
+ throw new RuntimeException(e);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/82f39578/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnTaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnTaskManager.scala b/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnTaskManager.scala
index 5b8f4ca..08d24b4 100644
--- a/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnTaskManager.scala
+++ b/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnTaskManager.scala
@@ -24,9 +24,12 @@ import org.apache.flink.runtime.io.disk.iomanager.IOManager
import org.apache.flink.runtime.io.network.NetworkEnvironment
import org.apache.flink.runtime.memory.MemoryManager
import org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup
+import org.apache.flink.runtime.security.SecurityUtils
import org.apache.flink.runtime.taskexecutor.TaskManagerConfiguration
import org.apache.flink.runtime.taskmanager.{TaskManager, TaskManagerLocation}
+import grizzled.slf4j.Logger
+
/** An extension of the TaskManager that listens for additional YARN related
* messages.
*/
@@ -56,14 +59,25 @@ class YarnTaskManager(
}
}
- object YarnTaskManager {
- /** Entry point (main method) to run the TaskManager on YARN.
- *
- * @param args The command line arguments.
- */
- def main(args: Array[String]): Unit = {
- YarnTaskManagerRunner.runYarnTaskManager(
- args, classOf[YarnTaskManager], System.getenv(), null)
+object YarnTaskManager {
+
+ val LOG = Logger(classOf[TaskManager])
+
+ /** Entry point (main method) to run the TaskManager on YARN.
+ *
+ * @param args The command line arguments.
+ */
+ def main(args: Array[String]): Unit = {
+ val tmRunner = YarnTaskManagerRunnerFactory.create(
+ args, classOf[YarnTaskManager], System.getenv())
+
+ try {
+ SecurityUtils.getInstalledContext.runSecured(tmRunner)
+ } catch {
+ case e: Exception =>
+ LOG.error("Exception occurred while launching Task Manager runner", e)
+ throw new RuntimeException(e)
}
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/82f39578/flink-yarn/src/test/java/org/apache/flink/yarn/YarnTaskManagerRunnerFactoryTest.java
----------------------------------------------------------------------
diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnTaskManagerRunnerFactoryTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnTaskManagerRunnerFactoryTest.java
new file mode 100644
index 0000000..7f518f1
--- /dev/null
+++ b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnTaskManagerRunnerFactoryTest.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.yarn;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.SecurityOptions;
+import org.apache.flink.runtime.security.SecurityUtils;
+import org.apache.flink.runtime.security.modules.HadoopModule;
+import org.apache.flink.runtime.security.modules.SecurityModule;
+
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * Tests for {@link YarnTaskManagerRunnerFactory}.
+ */
+public class YarnTaskManagerRunnerFactoryTest {
+
+ @Test
+ public void testKerberosKeytabConfiguration() throws IOException {
+ final String resourceDirPath =
+ Paths.get("src", "test", "resources").toAbsolutePath().toString();
+ final Map<String, String> envs = new HashMap<>();
+ envs.put(YarnFlinkResourceManager.ENV_FLINK_CONTAINER_ID, "test_container_00001");
+ envs.put(YarnConfigKeys.KEYTAB_PRINCIPAL, "testuser1@domain");
+ envs.put(ApplicationConstants.Environment.PWD.key(), resourceDirPath);
+
+ final YarnTaskManagerRunnerFactory.Runner tmRunner = YarnTaskManagerRunnerFactory.create(
+ new String[]{"--configDir", resourceDirPath},
+ YarnTaskManager.class,
+ envs);
+
+ final List<SecurityModule> modules = SecurityUtils.getInstalledModules();
+ Optional<SecurityModule> moduleOpt =
+ modules.stream().filter(s -> s instanceof HadoopModule).findFirst();
+ if (moduleOpt.isPresent()) {
+ HadoopModule hadoopModule = (HadoopModule) moduleOpt.get();
+ assertEquals("testuser1@domain", hadoopModule.getSecurityConfig().getPrincipal());
+ assertEquals(resourceDirPath + "/" + Utils.KEYTAB_FILE_NAME,
+ hadoopModule.getSecurityConfig().getKeytab());
+ } else {
+ fail("Can not find HadoopModule!");
+ }
+
+ final Configuration configuration = tmRunner.getConfiguration();
+ assertEquals(resourceDirPath + "/" + Utils.KEYTAB_FILE_NAME, configuration.getString(SecurityOptions.KERBEROS_LOGIN_KEYTAB));
+ assertEquals("testuser1@domain", configuration.getString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL));
+ assertEquals("test_container_00001", tmRunner.getResourceId().toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/82f39578/flink-yarn/src/test/java/org/apache/flink/yarn/YarnTaskManagerRunnerTest.java
----------------------------------------------------------------------
diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnTaskManagerRunnerTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnTaskManagerRunnerTest.java
deleted file mode 100644
index cdacfc0..0000000
--- a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnTaskManagerRunnerTest.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.yarn;
-
-import org.apache.flink.runtime.security.SecurityUtils;
-import org.apache.flink.runtime.security.modules.HadoopModule;
-import org.apache.flink.runtime.security.modules.SecurityModule;
-
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import java.io.IOException;
-import java.nio.file.Paths;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.concurrent.Callable;
-
-/**
- * Tests for {@link YarnTaskManagerRunner}.
- */
-public class YarnTaskManagerRunnerTest {
-
- @Test
- public void testKerberosKeytabConfiguration() throws IOException {
- final String resourceDirPath =
- Paths.get("src", "test", "resources").toAbsolutePath().toString();
- final Map<String, String> envs = new HashMap<>();
- envs.put(YarnFlinkResourceManager.ENV_FLINK_CONTAINER_ID, "test_container_00001");
- envs.put(YarnConfigKeys.KEYTAB_PRINCIPAL, "testuser1@domain");
- envs.put(ApplicationConstants.Environment.PWD.key(), resourceDirPath);
- YarnTaskManagerRunner.runYarnTaskManager(
- new String[]{"--configDir", resourceDirPath},
- YarnTaskManager.class,
- envs,
- new Callable<Object>() {
- @Override
- public Integer call() {
- final List<SecurityModule> modules = SecurityUtils.getInstalledModules();
- Optional<SecurityModule> moduleOpt =
- modules.stream().filter(s -> s instanceof HadoopModule).findFirst();
- if (moduleOpt.isPresent()) {
- HadoopModule hadoopModule = (HadoopModule) moduleOpt.get();
- assertEquals("testuser1@domain", hadoopModule.getSecurityConfig().getPrincipal());
- assertEquals(resourceDirPath + "/krb5.keytab",
- hadoopModule.getSecurityConfig().getKeytab());
- } else {
- fail("Can not find HadoopModule!");
- }
- return null;
- }
- });
- }
-}
[02/28] flink git commit: [FLINK-8421] [DataStream,
tests] Add WindowOperator migration test for Kryo-serialized window
keys
Posted by tz...@apache.org.
[FLINK-8421] [DataStream, tests] Add WindowOperator migration test for Kryo-serialized window keys
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/1f81f6ac
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/1f81f6ac
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/1f81f6ac
Branch: refs/heads/release-1.4
Commit: 1f81f6acf4a729c892c329b7357d3da6ade4ef3f
Parents: e41300d
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Wed Jan 24 17:08:13 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 17:31:32 2018 +0100
----------------------------------------------------------------------
.../operators/windowing/NonPojoType.java | 68 ++++++
.../windowing/WindowOperatorMigrationTest.java | 219 +++++++++++++++----
...n-test-kryo-serialized-key-flink1.2-snapshot | Bin 0 -> 3686 bytes
...n-test-kryo-serialized-key-flink1.3-snapshot | Bin 0 -> 11298 bytes
...n-test-kryo-serialized-key-flink1.4-snapshot | Bin 0 -> 9539 bytes
5 files changed, 241 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/1f81f6ac/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/NonPojoType.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/NonPojoType.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/NonPojoType.java
new file mode 100644
index 0000000..e70b847
--- /dev/null
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/NonPojoType.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.runtime.operators.windowing;
+
+/**
+ * A non-Pojo type used in {@link WindowOperatorMigrationTest} to test
+ * migration behaviours for window operators that have a Kryo-serialized key.
+ *
+ * <p>NOTE: modifying this class would cause tests in {@link WindowOperatorMigrationTest} to fail,
+ * since it is serialized in older version test savepoints.
+ */
+public class NonPojoType implements Comparable<NonPojoType> {
+
+ private final String data;
+
+ NonPojoType(String data) {
+ this.data = data;
+ }
+
+ public String getData() {
+ return data;
+ }
+
+ @Override
+ public int compareTo(NonPojoType o) {
+ return data.compareTo(o.data);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this) {
+ return true;
+ }
+
+ if (obj == null) {
+ return false;
+ }
+
+ if (!(obj instanceof NonPojoType)) {
+ return false;
+ }
+
+ NonPojoType other = (NonPojoType) obj;
+
+ return data.equals(other.data);
+ }
+
+ @Override
+ public int hashCode() {
+ return data.hashCode();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1f81f6ac/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java
index 5b75d46..9ad987f 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java
@@ -23,11 +23,14 @@ import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.common.state.ListStateDescriptor;
import org.apache.flink.api.common.state.ReducingStateDescriptor;
import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.common.typeinfo.TypeHint;
import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.typeutils.TypeInfoParser;
+import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.windowing.PassThroughWindowFunction;
import org.apache.flink.streaming.api.functions.windowing.RichWindowFunction;
@@ -66,6 +69,7 @@ import java.util.Comparator;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.TimeUnit;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
/**
@@ -114,7 +118,7 @@ public class WindowOperatorMigrationTest {
WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple3<String, Long, Long>, TimeWindow> operator = new WindowOperator<>(
EventTimeSessionWindows.withGap(Time.seconds(sessionSize)),
new TimeWindow.Serializer(),
- new TupleKeySelector(),
+ new TupleKeySelector<String>(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalIterableWindowFunction<>(new SessionWindowFunction()),
@@ -123,7 +127,7 @@ public class WindowOperatorMigrationTest {
null /* late data output tag */);
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness =
- new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+ new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO);
testHarness.setup();
testHarness.open();
@@ -160,7 +164,7 @@ public class WindowOperatorMigrationTest {
WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple3<String, Long, Long>, TimeWindow> operator = new WindowOperator<>(
EventTimeSessionWindows.withGap(Time.seconds(sessionSize)),
new TimeWindow.Serializer(),
- new TupleKeySelector(),
+ new TupleKeySelector<String>(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalIterableWindowFunction<>(new SessionWindowFunction()),
@@ -171,7 +175,7 @@ public class WindowOperatorMigrationTest {
ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness =
- new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+ new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO);
testHarness.setup();
@@ -218,7 +222,7 @@ public class WindowOperatorMigrationTest {
WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple3<String, Long, Long>, TimeWindow> operator = new WindowOperator<>(
EventTimeSessionWindows.withGap(Time.seconds(sessionSize)),
new TimeWindow.Serializer(),
- new TupleKeySelector(),
+ new TupleKeySelector<String>(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalIterableWindowFunction<>(new SessionWindowFunction()),
@@ -227,7 +231,7 @@ public class WindowOperatorMigrationTest {
null /* late data output tag */);
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness =
- new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+ new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO);
testHarness.setup();
testHarness.open();
@@ -258,7 +262,7 @@ public class WindowOperatorMigrationTest {
WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple3<String, Long, Long>, TimeWindow> operator = new WindowOperator<>(
EventTimeSessionWindows.withGap(Time.seconds(sessionSize)),
new TimeWindow.Serializer(),
- new TupleKeySelector(),
+ new TupleKeySelector<String>(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalIterableWindowFunction<>(new SessionWindowFunction()),
@@ -269,7 +273,7 @@ public class WindowOperatorMigrationTest {
ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness =
- new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+ new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO);
testHarness.setup();
@@ -322,13 +326,13 @@ public class WindowOperatorMigrationTest {
TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
- new SumReducer(),
+ new SumReducer<>(),
inputType.createSerializer(new ExecutionConfig()));
WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
new TimeWindow.Serializer(),
- new TupleKeySelector(),
+ new TupleKeySelector<>(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
@@ -339,7 +343,7 @@ public class WindowOperatorMigrationTest {
ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
- new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+ new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO);
testHarness.setup();
testHarness.open();
@@ -358,11 +362,11 @@ public class WindowOperatorMigrationTest {
testHarness.processWatermark(new Watermark(999));
expectedOutput.add(new Watermark(999));
- TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());
testHarness.processWatermark(new Watermark(1999));
expectedOutput.add(new Watermark(1999));
- TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());
// do snapshot and save to file
OperatorStateHandles snapshot = testHarness.snapshot(0, 0);
@@ -380,13 +384,13 @@ public class WindowOperatorMigrationTest {
TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
- new SumReducer(),
+ new SumReducer<>(),
inputType.createSerializer(new ExecutionConfig()));
WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
new TimeWindow.Serializer(),
- new TupleKeySelector(),
+ new TupleKeySelector<>(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
@@ -397,7 +401,7 @@ public class WindowOperatorMigrationTest {
ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
- new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+ new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO);
testHarness.setup();
@@ -424,7 +428,7 @@ public class WindowOperatorMigrationTest {
expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 2), 5999));
expectedOutput.add(new Watermark(5999));
- TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());
testHarness.close();
}
@@ -444,7 +448,7 @@ public class WindowOperatorMigrationTest {
WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
new TimeWindow.Serializer(),
- new TupleKeySelector(),
+ new TupleKeySelector<>(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>()),
@@ -455,7 +459,7 @@ public class WindowOperatorMigrationTest {
ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
- new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+ new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO);
testHarness.setup();
testHarness.open();
@@ -474,11 +478,11 @@ public class WindowOperatorMigrationTest {
testHarness.processWatermark(new Watermark(999));
expectedOutput.add(new Watermark(999));
- TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());
testHarness.processWatermark(new Watermark(1999));
expectedOutput.add(new Watermark(1999));
- TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());
// do snapshot and save to file
OperatorStateHandles snapshot = testHarness.snapshot(0, 0);
@@ -501,7 +505,7 @@ public class WindowOperatorMigrationTest {
WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
new TimeWindow.Serializer(),
- new TupleKeySelector(),
+ new TupleKeySelector<>(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>()),
@@ -512,7 +516,7 @@ public class WindowOperatorMigrationTest {
ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
- new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+ new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO);
testHarness.setup();
@@ -539,7 +543,7 @@ public class WindowOperatorMigrationTest {
expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 2), 5999));
expectedOutput.add(new Watermark(5999));
- TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());
testHarness.close();
}
@@ -554,13 +558,13 @@ public class WindowOperatorMigrationTest {
TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
- new SumReducer(),
+ new SumReducer<>(),
inputType.createSerializer(new ExecutionConfig()));
WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
TumblingProcessingTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
new TimeWindow.Serializer(),
- new TupleKeySelector(),
+ new TupleKeySelector<>(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
@@ -571,7 +575,7 @@ public class WindowOperatorMigrationTest {
ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
- new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+ new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO);
testHarness.setup();
testHarness.open();
@@ -587,7 +591,7 @@ public class WindowOperatorMigrationTest {
expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 1), 2999));
expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 1), 2999));
- TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());
// do snapshot and save to file
OperatorStateHandles snapshot = testHarness.snapshot(0, 0);
@@ -606,13 +610,13 @@ public class WindowOperatorMigrationTest {
TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
- new SumReducer(),
+ new SumReducer<>(),
inputType.createSerializer(new ExecutionConfig()));
WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
TumblingProcessingTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
new TimeWindow.Serializer(),
- new TupleKeySelector(),
+ new TupleKeySelector<>(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
@@ -623,7 +627,7 @@ public class WindowOperatorMigrationTest {
ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
- new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+ new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO);
testHarness.setup();
@@ -645,7 +649,7 @@ public class WindowOperatorMigrationTest {
expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 4), 5999));
expectedOutput.add(new StreamRecord<>(new Tuple2<>("key3", 1), 5999));
- TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());
testHarness.close();
}
@@ -665,7 +669,7 @@ public class WindowOperatorMigrationTest {
WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
TumblingProcessingTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
new TimeWindow.Serializer(),
- new TupleKeySelector(),
+ new TupleKeySelector<>(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>()),
@@ -676,7 +680,7 @@ public class WindowOperatorMigrationTest {
ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
- new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+ new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO);
testHarness.setup();
testHarness.open();
@@ -692,7 +696,7 @@ public class WindowOperatorMigrationTest {
expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 1), 2999));
expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 1), 2999));
- TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());
// do snapshot and save to file
OperatorStateHandles snapshot = testHarness.snapshot(0, 0);
@@ -715,7 +719,7 @@ public class WindowOperatorMigrationTest {
WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
TumblingProcessingTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
new TimeWindow.Serializer(),
- new TupleKeySelector(),
+ new TupleKeySelector<>(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>()),
@@ -726,7 +730,7 @@ public class WindowOperatorMigrationTest {
ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
- new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+ new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO);
testHarness.setup();
@@ -748,28 +752,151 @@ public class WindowOperatorMigrationTest {
expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 4), 5999));
expectedOutput.add(new StreamRecord<>(new Tuple2<>("key3", 1), 5999));
- TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());
testHarness.close();
}
- private static class TupleKeySelector implements KeySelector<Tuple2<String, Integer>, String> {
+ /**
+ * Manually run this to write binary snapshot data.
+ */
+ @Ignore
+ @Test
+ public void writeWindowsWithKryoSerializedKeysSnapshot() throws Exception {
+ final int windowSize = 3;
+
+ TypeInformation<Tuple2<NonPojoType, Integer>> inputType = new TypeHint<Tuple2<NonPojoType, Integer>>() {}.getTypeInfo();
+
+ ReducingStateDescriptor<Tuple2<NonPojoType, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
+ new SumReducer<>(),
+ inputType.createSerializer(new ExecutionConfig()));
+
+ TypeSerializer<NonPojoType> keySerializer = TypeInformation.of(NonPojoType.class).createSerializer(new ExecutionConfig());
+ assertTrue(keySerializer instanceof KryoSerializer);
+
+ WindowOperator<NonPojoType, Tuple2<NonPojoType, Integer>, Tuple2<NonPojoType, Integer>, Tuple2<NonPojoType, Integer>, TimeWindow> operator = new WindowOperator<>(
+ TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
+ new TimeWindow.Serializer(),
+ new TupleKeySelector<>(),
+ keySerializer,
+ stateDesc,
+ new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<>()),
+ EventTimeTrigger.create(),
+ 0,
+ null /* late data output tag */);
+
+ ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
+
+ OneInputStreamOperatorTestHarness<Tuple2<NonPojoType, Integer>, Tuple2<NonPojoType, Integer>> testHarness =
+ new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), TypeInformation.of(NonPojoType.class));
+
+ testHarness.setup();
+ testHarness.open();
+
+ // add elements out-of-order
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 1), 3999));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 1), 3000));
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key1"), 1), 20));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key1"), 1), 0));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key1"), 1), 999));
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 1), 1998));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 1), 1999));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 1), 1000));
+
+ testHarness.processWatermark(new Watermark(999));
+ expectedOutput.add(new Watermark(999));
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());
+
+ testHarness.processWatermark(new Watermark(1999));
+ expectedOutput.add(new Watermark(1999));
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());
+
+ // do snapshot and save to file
+ OperatorStateHandles snapshot = testHarness.snapshot(0, 0);
+ OperatorSnapshotUtil.writeStateHandle(
+ snapshot,
+ "src/test/resources/win-op-migration-test-kryo-serialized-key-flink" + flinkGenerateSavepointVersion + "-snapshot");
+
+ testHarness.close();
+ }
+
+ @Test
+ public void testRestoreKryoSerializedKeysWindows() throws Exception {
+ final int windowSize = 3;
+
+ TypeInformation<Tuple2<NonPojoType, Integer>> inputType = new TypeHint<Tuple2<NonPojoType, Integer>>() {}.getTypeInfo();
+
+ ReducingStateDescriptor<Tuple2<NonPojoType, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
+ new SumReducer<>(),
+ inputType.createSerializer(new ExecutionConfig()));
+
+ TypeSerializer<NonPojoType> keySerializer = TypeInformation.of(NonPojoType.class).createSerializer(new ExecutionConfig());
+ assertTrue(keySerializer instanceof KryoSerializer);
+
+ WindowOperator<NonPojoType, Tuple2<NonPojoType, Integer>, Tuple2<NonPojoType, Integer>, Tuple2<NonPojoType, Integer>, TimeWindow> operator = new WindowOperator<>(
+ TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
+ new TimeWindow.Serializer(),
+ new TupleKeySelector<>(),
+ keySerializer,
+ stateDesc,
+ new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<>()),
+ EventTimeTrigger.create(),
+ 0,
+ null /* late data output tag */);
+
+ ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
+
+ OneInputStreamOperatorTestHarness<Tuple2<NonPojoType, Integer>, Tuple2<NonPojoType, Integer>> testHarness =
+ new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), TypeInformation.of(NonPojoType.class));
+
+ testHarness.setup();
+
+ MigrationTestUtil.restoreFromSnapshot(
+ testHarness,
+ OperatorSnapshotUtil.getResourceFilename(
+ "win-op-migration-test-kryo-serialized-key-flink" + testMigrateVersion + "-snapshot"),
+ testMigrateVersion);
+
+ testHarness.open();
+
+ testHarness.processWatermark(new Watermark(2999));
+ expectedOutput.add(new StreamRecord<>(new Tuple2<>(new NonPojoType("key1"), 3), 2999));
+ expectedOutput.add(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 3), 2999));
+ expectedOutput.add(new Watermark(2999));
+
+ testHarness.processWatermark(new Watermark(3999));
+ expectedOutput.add(new Watermark(3999));
+
+ testHarness.processWatermark(new Watermark(4999));
+ expectedOutput.add(new Watermark(4999));
+
+ testHarness.processWatermark(new Watermark(5999));
+ expectedOutput.add(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 2), 5999));
+ expectedOutput.add(new Watermark(5999));
+
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());
+ testHarness.close();
+ }
+
+ private static class TupleKeySelector<K> implements KeySelector<Tuple2<K, Integer>, K> {
private static final long serialVersionUID = 1L;
@Override
- public String getKey(Tuple2<String, Integer> value) throws Exception {
+ public K getKey(Tuple2<K, Integer> value) throws Exception {
return value.f0;
}
}
@SuppressWarnings("unchecked")
- private static class Tuple2ResultSortComparator implements Comparator<Object> {
+ private static class Tuple2ResultSortComparator<K extends Comparable> implements Comparator<Object> {
@Override
public int compare(Object o1, Object o2) {
if (o1 instanceof Watermark || o2 instanceof Watermark) {
return 0;
} else {
- StreamRecord<Tuple2<String, Integer>> sr0 = (StreamRecord<Tuple2<String, Integer>>) o1;
- StreamRecord<Tuple2<String, Integer>> sr1 = (StreamRecord<Tuple2<String, Integer>>) o2;
+ StreamRecord<Tuple2<K, Integer>> sr0 = (StreamRecord<Tuple2<K, Integer>>) o1;
+ StreamRecord<Tuple2<K, Integer>> sr1 = (StreamRecord<Tuple2<K, Integer>>) o2;
if (sr0.getTimestamp() != sr1.getTimestamp()) {
return (int) (sr0.getTimestamp() - sr1.getTimestamp());
}
@@ -809,11 +936,11 @@ public class WindowOperatorMigrationTest {
}
}
- private static class SumReducer implements ReduceFunction<Tuple2<String, Integer>> {
+ private static class SumReducer<K> implements ReduceFunction<Tuple2<K, Integer>> {
private static final long serialVersionUID = 1L;
@Override
- public Tuple2<String, Integer> reduce(Tuple2<String, Integer> value1,
- Tuple2<String, Integer> value2) throws Exception {
+ public Tuple2<K, Integer> reduce(Tuple2<K, Integer> value1,
+ Tuple2<K, Integer> value2) throws Exception {
return new Tuple2<>(value2.f0, value1.f1 + value2.f1);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1f81f6ac/flink-streaming-java/src/test/resources/win-op-migration-test-kryo-serialized-key-flink1.2-snapshot
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-kryo-serialized-key-flink1.2-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-kryo-serialized-key-flink1.2-snapshot
new file mode 100644
index 0000000..3a4cc98
Binary files /dev/null and b/flink-streaming-java/src/test/resources/win-op-migration-test-kryo-serialized-key-flink1.2-snapshot differ
http://git-wip-us.apache.org/repos/asf/flink/blob/1f81f6ac/flink-streaming-java/src/test/resources/win-op-migration-test-kryo-serialized-key-flink1.3-snapshot
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-kryo-serialized-key-flink1.3-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-kryo-serialized-key-flink1.3-snapshot
new file mode 100644
index 0000000..aa81eb4
Binary files /dev/null and b/flink-streaming-java/src/test/resources/win-op-migration-test-kryo-serialized-key-flink1.3-snapshot differ
http://git-wip-us.apache.org/repos/asf/flink/blob/1f81f6ac/flink-streaming-java/src/test/resources/win-op-migration-test-kryo-serialized-key-flink1.4-snapshot
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-kryo-serialized-key-flink1.4-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-kryo-serialized-key-flink1.4-snapshot
new file mode 100644
index 0000000..8449113
Binary files /dev/null and b/flink-streaming-java/src/test/resources/win-op-migration-test-kryo-serialized-key-flink1.4-snapshot differ
[04/28] flink git commit: [FLINK-8472] [DataStream,
test] Refactor StatefulJobSavepointFrom*MigrationITCase to single
ITCase
Posted by tz...@apache.org.
[FLINK-8472] [DataStream, test] Refactor StatefulJobSavepointFrom*MigrationITCase to single ITCase
This commit refactors the StatefulJobSavepointFrom12MigrationITCase and
StatefulJobSavepointFrom13MigrationITCase to a single class,
StatefulJobSavepointMigrationITCase. The new ITCase is parameterized to
ensure that all previous versions and state backend variants are
covered.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/2ada8567
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/2ada8567
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/2ada8567
Branch: refs/heads/release-1.4
Commit: 2ada8567e8013cacd602b3311df66221b16bc7a7
Parents: 4fc14cc
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Thu Jan 25 18:25:45 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 17:31:32 2018 +0100
----------------------------------------------------------------------
...atefulJobSavepointFrom12MigrationITCase.java | 694 -------------------
...atefulJobSavepointFrom13MigrationITCase.java | 58 --
.../StatefulJobSavepointMigrationITCase.java | 661 ++++++++++++++++++
3 files changed, 661 insertions(+), 752 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/2ada8567/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointFrom12MigrationITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointFrom12MigrationITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointFrom12MigrationITCase.java
deleted file mode 100644
index 6859c2d..0000000
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointFrom12MigrationITCase.java
+++ /dev/null
@@ -1,694 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.test.checkpointing.utils;
-
-import org.apache.flink.api.common.accumulators.IntCounter;
-import org.apache.flink.api.common.functions.FlatMapFunction;
-import org.apache.flink.api.common.functions.RichFlatMapFunction;
-import org.apache.flink.api.common.restartstrategy.RestartStrategies;
-import org.apache.flink.api.common.state.ValueState;
-import org.apache.flink.api.common.state.ValueStateDescriptor;
-import org.apache.flink.api.common.typeinfo.TypeHint;
-import org.apache.flink.api.common.typeutils.base.LongSerializer;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.contrib.streaming.state.RocksDBStateBackend;
-import org.apache.flink.runtime.state.memory.MemoryStateBackend;
-import org.apache.flink.streaming.api.TimeCharacteristic;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
-import org.apache.flink.streaming.api.functions.source.RichSourceFunction;
-import org.apache.flink.streaming.api.functions.source.SourceFunction;
-import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
-import org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator;
-import org.apache.flink.streaming.api.operators.InternalTimer;
-import org.apache.flink.streaming.api.operators.InternalTimerService;
-import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
-import org.apache.flink.streaming.api.operators.TimestampedCollector;
-import org.apache.flink.streaming.api.operators.Triggerable;
-import org.apache.flink.streaming.api.watermark.Watermark;
-import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
-import org.apache.flink.util.Collector;
-
-import org.junit.Ignore;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * This verifies that we can restore a complete job from a Flink 1.2 savepoint.
- *
- * <p>The test for checkpointed (legacy state) was removed from this test for Flink 1.4 because compatibility with
- * Flink 1.1 is removed. The legacy state in the binary savepoints is ignored by the tests now.
- *
- * <p>The tests will time out if they don't see the required number of successful checks within
- * a time limit.
- *
- *
- */
-public class StatefulJobSavepointFrom12MigrationITCase extends SavepointMigrationTestBase {
- private static final int NUM_SOURCE_ELEMENTS = 4;
-
- /**
- * This has to be manually executed to create the savepoint on Flink 1.2.
- */
- @Test
- @Ignore
- public void testCreateSavepointOnFlink12() throws Exception {
-
- final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
- env.setStateBackend(new MemoryStateBackend());
- env.enableCheckpointing(500);
- env.setParallelism(4);
- env.setMaxParallelism(4);
-
- env
- .addSource(new LegacyCheckpointedSource(NUM_SOURCE_ELEMENTS)).setMaxParallelism(1).uid("LegacyCheckpointedSource")
- .flatMap(new LegacyCheckpointedFlatMap()).startNewChain().uid("LegacyCheckpointedFlatMap")
- .keyBy(0)
- .flatMap(new LegacyCheckpointedFlatMapWithKeyedState()).startNewChain().uid("LegacyCheckpointedFlatMapWithKeyedState")
- .keyBy(0)
- .flatMap(new KeyedStateSettingFlatMap()).startNewChain().uid("KeyedStateSettingFlatMap")
- .keyBy(0)
- .transform(
- "custom_operator",
- new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
- new CheckpointedUdfOperator(new LegacyCheckpointedFlatMapWithKeyedState())).uid("LegacyCheckpointedOperator")
- .keyBy(0)
- .transform(
- "timely_stateful_operator",
- new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
- new TimelyStatefulOperator()).uid("TimelyStatefulOperator")
- .addSink(new AccumulatorCountingSink<Tuple2<Long, Long>>());
-
- executeAndSavepoint(
- env,
- "src/test/resources/" + getSavepointPath(),
- new Tuple2<>(AccumulatorCountingSink.NUM_ELEMENTS_ACCUMULATOR, NUM_SOURCE_ELEMENTS));
- }
-
- /**
- * This has to be manually executed to create the savepoint on Flink 1.2.
- */
- @Test
- @Ignore
- public void testCreateSavepointOnFlink12WithRocksDB() throws Exception {
-
- final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
- RocksDBStateBackend rocksBackend =
- new RocksDBStateBackend(new MemoryStateBackend());
- env.setStateBackend(rocksBackend);
- env.enableCheckpointing(500);
- env.setParallelism(4);
- env.setMaxParallelism(4);
-
- env
- .addSource(new LegacyCheckpointedSource(NUM_SOURCE_ELEMENTS)).setMaxParallelism(1).uid("LegacyCheckpointedSource")
- .flatMap(new LegacyCheckpointedFlatMap()).startNewChain().uid("LegacyCheckpointedFlatMap")
- .keyBy(0)
- .flatMap(new LegacyCheckpointedFlatMapWithKeyedState()).startNewChain().uid("LegacyCheckpointedFlatMapWithKeyedState")
- .keyBy(0)
- .flatMap(new KeyedStateSettingFlatMap()).startNewChain().uid("KeyedStateSettingFlatMap")
- .keyBy(0)
- .transform(
- "custom_operator",
- new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
- new CheckpointedUdfOperator(new LegacyCheckpointedFlatMapWithKeyedState())).uid("LegacyCheckpointedOperator")
- .keyBy(0)
- .transform(
- "timely_stateful_operator",
- new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
- new TimelyStatefulOperator()).uid("TimelyStatefulOperator")
- .addSink(new AccumulatorCountingSink<Tuple2<Long, Long>>());
-
- executeAndSavepoint(
- env,
- "src/test/resources/" + getRocksDBSavepointPath(),
- new Tuple2<>(AccumulatorCountingSink.NUM_ELEMENTS_ACCUMULATOR, NUM_SOURCE_ELEMENTS));
- }
-
- @Test
- public void testSavepointRestoreFromFlink12() throws Exception {
-
- final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setRestartStrategy(RestartStrategies.noRestart());
- env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
- env.setStateBackend(new MemoryStateBackend());
- env.enableCheckpointing(500);
- env.setParallelism(4);
- env.setMaxParallelism(4);
-
- env
- .addSource(new CheckingRestoringSource(NUM_SOURCE_ELEMENTS)).setMaxParallelism(1).uid("LegacyCheckpointedSource")
- .flatMap(new CheckingRestoringFlatMap()).startNewChain().uid("LegacyCheckpointedFlatMap")
- .keyBy(0)
- .flatMap(new CheckingRestoringFlatMapWithKeyedState()).startNewChain().uid("LegacyCheckpointedFlatMapWithKeyedState")
- .keyBy(0)
- .flatMap(new CheckingKeyedStateFlatMap()).startNewChain().uid("KeyedStateSettingFlatMap")
- .keyBy(0)
- .transform(
- "custom_operator",
- new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
- new CheckingRestoringUdfOperator(new CheckingRestoringFlatMapWithKeyedStateInOperator())).uid("LegacyCheckpointedOperator")
- .keyBy(0)
- .transform(
- "timely_stateful_operator",
- new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
- new CheckingTimelyStatefulOperator()).uid("TimelyStatefulOperator")
- .addSink(new AccumulatorCountingSink<Tuple2<Long, Long>>());
-
- restoreAndExecute(
- env,
- getResourceFilename(getSavepointPath()),
- new Tuple2<>(CheckingRestoringSource.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, 1),
- new Tuple2<>(CheckingRestoringFlatMap.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
- new Tuple2<>(CheckingRestoringFlatMapWithKeyedState.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
- new Tuple2<>(CheckingKeyedStateFlatMap.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
- new Tuple2<>(CheckingRestoringUdfOperator.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
- new Tuple2<>(CheckingRestoringFlatMapWithKeyedStateInOperator.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
- new Tuple2<>(CheckingTimelyStatefulOperator.SUCCESSFUL_PROCESS_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
- new Tuple2<>(CheckingTimelyStatefulOperator.SUCCESSFUL_EVENT_TIME_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
- new Tuple2<>(CheckingTimelyStatefulOperator.SUCCESSFUL_PROCESSING_TIME_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
- new Tuple2<>(AccumulatorCountingSink.NUM_ELEMENTS_ACCUMULATOR, NUM_SOURCE_ELEMENTS));
- }
-
- @Test
- public void testSavepointRestoreFromFlink12FromRocksDB() throws Exception {
-
- final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setRestartStrategy(RestartStrategies.noRestart());
- env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
- env.setStateBackend(new RocksDBStateBackend(new MemoryStateBackend()));
- env.enableCheckpointing(500);
- env.setParallelism(4);
- env.setMaxParallelism(4);
-
- env
- .addSource(new CheckingRestoringSource(NUM_SOURCE_ELEMENTS)).setMaxParallelism(1).uid("LegacyCheckpointedSource")
- .flatMap(new CheckingRestoringFlatMap()).startNewChain().uid("LegacyCheckpointedFlatMap")
- .keyBy(0)
- .flatMap(new CheckingRestoringFlatMapWithKeyedState()).startNewChain().uid("LegacyCheckpointedFlatMapWithKeyedState")
- .keyBy(0)
- .flatMap(new CheckingKeyedStateFlatMap()).startNewChain().uid("KeyedStateSettingFlatMap")
- .keyBy(0)
- .transform(
- "custom_operator",
- new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
- new CheckingRestoringUdfOperator(new CheckingRestoringFlatMapWithKeyedStateInOperator())).uid("LegacyCheckpointedOperator")
- .keyBy(0)
- .transform(
- "timely_stateful_operator",
- new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
- new CheckingTimelyStatefulOperator()).uid("TimelyStatefulOperator")
- .addSink(new AccumulatorCountingSink<Tuple2<Long, Long>>());
-
- restoreAndExecute(
- env,
- getResourceFilename(getRocksDBSavepointPath()),
- new Tuple2<>(CheckingRestoringSource.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, 1),
- new Tuple2<>(CheckingRestoringFlatMap.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
- new Tuple2<>(CheckingRestoringFlatMapWithKeyedState.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
- new Tuple2<>(CheckingKeyedStateFlatMap.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
- new Tuple2<>(CheckingRestoringUdfOperator.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
- new Tuple2<>(CheckingRestoringFlatMapWithKeyedStateInOperator.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
- new Tuple2<>(CheckingTimelyStatefulOperator.SUCCESSFUL_PROCESS_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
- new Tuple2<>(CheckingTimelyStatefulOperator.SUCCESSFUL_EVENT_TIME_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
- new Tuple2<>(CheckingTimelyStatefulOperator.SUCCESSFUL_PROCESSING_TIME_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
- new Tuple2<>(AccumulatorCountingSink.NUM_ELEMENTS_ACCUMULATOR, NUM_SOURCE_ELEMENTS));
- }
-
- protected String getSavepointPath() {
- return "stateful-udf-migration-itcase-flink1.2-savepoint";
- }
-
- protected String getRocksDBSavepointPath() {
- return "stateful-udf-migration-itcase-flink1.2-rocksdb-savepoint";
- }
-
- private static class LegacyCheckpointedSource
- implements SourceFunction<Tuple2<Long, Long>> {
-
- public static String checkpointedString = "Here be dragons!";
-
- private static final long serialVersionUID = 1L;
-
- private volatile boolean isRunning = true;
-
- private final int numElements;
-
- public LegacyCheckpointedSource(int numElements) {
- this.numElements = numElements;
- }
-
- @Override
- public void run(SourceContext<Tuple2<Long, Long>> ctx) throws Exception {
-
- ctx.emitWatermark(new Watermark(0));
-
- synchronized (ctx.getCheckpointLock()) {
- for (long i = 0; i < numElements; i++) {
- ctx.collect(new Tuple2<>(i, i));
- }
- }
-
- // don't emit a final watermark so that we don't trigger the registered event-time
- // timers
- while (isRunning) {
- Thread.sleep(20);
- }
- }
-
- @Override
- public void cancel() {
- isRunning = false;
- }
- }
-
- private static class CheckingRestoringSource
- extends RichSourceFunction<Tuple2<Long, Long>> {
-
- private static final long serialVersionUID = 1L;
-
- public static final String SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR = CheckingRestoringSource.class + "_RESTORE_CHECK";
-
- private volatile boolean isRunning = true;
-
- private final int numElements;
-
- private String restoredState;
-
- public CheckingRestoringSource(int numElements) {
- this.numElements = numElements;
- }
-
- @Override
- public void open(Configuration parameters) throws Exception {
- super.open(parameters);
-
- getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
- }
-
- @Override
- public void run(SourceContext<Tuple2<Long, Long>> ctx) throws Exception {
- getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
-
- // immediately trigger any set timers
- ctx.emitWatermark(new Watermark(1000));
-
- synchronized (ctx.getCheckpointLock()) {
- for (long i = 0; i < numElements; i++) {
- ctx.collect(new Tuple2<>(i, i));
- }
- }
-
- while (isRunning) {
- Thread.sleep(20);
- }
- }
-
- @Override
- public void cancel() {
- isRunning = false;
- }
- }
-
- private static class LegacyCheckpointedFlatMap extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
-
- private static final long serialVersionUID = 1L;
-
- public static Tuple2<String, Long> checkpointedTuple =
- new Tuple2<>("hello", 42L);
-
- @Override
- public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
- out.collect(value);
- }
- }
-
- private static class CheckingRestoringFlatMap extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
-
- private static final long serialVersionUID = 1L;
-
- public static final String SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR = CheckingRestoringFlatMap.class + "_RESTORE_CHECK";
-
- private transient Tuple2<String, Long> restoredState;
-
- @Override
- public void open(Configuration parameters) throws Exception {
- super.open(parameters);
-
- getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
- }
-
- @Override
- public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
- out.collect(value);
-
- getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
-
- }
-
- }
-
- private static class LegacyCheckpointedFlatMapWithKeyedState
- extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
-
- private static final long serialVersionUID = 1L;
-
- public static Tuple2<String, Long> checkpointedTuple =
- new Tuple2<>("hello", 42L);
-
- private final ValueStateDescriptor<Long> stateDescriptor =
- new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
-
- @Override
- public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
- out.collect(value);
-
- getRuntimeContext().getState(stateDescriptor).update(value.f1);
-
- assertEquals(value.f1, getRuntimeContext().getState(stateDescriptor).value());
- }
- }
-
- private static class CheckingRestoringFlatMapWithKeyedState
- extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
-
- private static final long serialVersionUID = 1L;
-
- public static final String SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR = CheckingRestoringFlatMapWithKeyedState.class + "_RESTORE_CHECK";
-
- private transient Tuple2<String, Long> restoredState;
-
- private final ValueStateDescriptor<Long> stateDescriptor =
- new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
-
- @Override
- public void open(Configuration parameters) throws Exception {
- super.open(parameters);
-
- getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
- }
-
- @Override
- public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
- out.collect(value);
-
- ValueState<Long> state = getRuntimeContext().getState(stateDescriptor);
- if (state == null) {
- throw new RuntimeException("Missing key value state for " + value);
- }
-
- assertEquals(value.f1, state.value());
- getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
- }
- }
-
- private static class CheckingRestoringFlatMapWithKeyedStateInOperator
- extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
-
- private static final long serialVersionUID = 1L;
-
- public static final String SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR = CheckingRestoringFlatMapWithKeyedStateInOperator.class + "_RESTORE_CHECK";
-
- private transient Tuple2<String, Long> restoredState;
-
- private final ValueStateDescriptor<Long> stateDescriptor =
- new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
-
- @Override
- public void open(Configuration parameters) throws Exception {
- super.open(parameters);
-
- getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
- }
-
- @Override
- public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
- out.collect(value);
-
- ValueState<Long> state = getRuntimeContext().getState(stateDescriptor);
- if (state == null) {
- throw new RuntimeException("Missing key value state for " + value);
- }
-
- assertEquals(value.f1, state.value());
- getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
- }
- }
-
- private static class KeyedStateSettingFlatMap extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
-
- private static final long serialVersionUID = 1L;
-
- private final ValueStateDescriptor<Long> stateDescriptor =
- new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
-
- @Override
- public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
- out.collect(value);
-
- getRuntimeContext().getState(stateDescriptor).update(value.f1);
- }
- }
-
- private static class CheckingKeyedStateFlatMap extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
-
- private static final long serialVersionUID = 1L;
-
- public static final String SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR = CheckingKeyedStateFlatMap.class + "_RESTORE_CHECK";
-
- private final ValueStateDescriptor<Long> stateDescriptor =
- new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
-
- @Override
- public void open(Configuration parameters) throws Exception {
- super.open(parameters);
-
- getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
- }
-
- @Override
- public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
- out.collect(value);
-
- ValueState<Long> state = getRuntimeContext().getState(stateDescriptor);
- if (state == null) {
- throw new RuntimeException("Missing key value state for " + value);
- }
-
- assertEquals(value.f1, state.value());
- getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
- }
- }
-
- private static class CheckpointedUdfOperator
- extends AbstractUdfStreamOperator<Tuple2<Long, Long>, FlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>>
- implements OneInputStreamOperator<Tuple2<Long, Long>, Tuple2<Long, Long>> {
- private static final long serialVersionUID = 1L;
-
- private static final String CHECKPOINTED_STRING = "Oh my, that's nice!";
-
- public CheckpointedUdfOperator(FlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> userFunction) {
- super(userFunction);
- }
-
- @Override
- public void processElement(StreamRecord<Tuple2<Long, Long>> element) throws Exception {
- userFunction.flatMap(element.getValue(), new TimestampedCollector<>(output));
- }
-
- @Override
- public void processWatermark(Watermark mark) throws Exception {
- output.emitWatermark(mark);
- }
- }
-
- private static class CheckingRestoringUdfOperator
- extends AbstractUdfStreamOperator<Tuple2<Long, Long>, FlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>>
- implements OneInputStreamOperator<Tuple2<Long, Long>, Tuple2<Long, Long>> {
-
- private static final long serialVersionUID = 1L;
-
- public static final String SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR = CheckingRestoringUdfOperator.class + "_RESTORE_CHECK";
-
- private String restoredState;
-
- public CheckingRestoringUdfOperator(FlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> userFunction) {
- super(userFunction);
- }
-
- @Override
- public void open() throws Exception {
- super.open();
-
- getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
- }
-
- @Override
- public void processElement(StreamRecord<Tuple2<Long, Long>> element) throws Exception {
- userFunction.flatMap(element.getValue(), new TimestampedCollector<>(output));
- getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
- }
-
- @Override
- public void processWatermark(Watermark mark) throws Exception {
- output.emitWatermark(mark);
- }
- }
-
- private static class TimelyStatefulOperator
- extends AbstractStreamOperator<Tuple2<Long, Long>>
- implements OneInputStreamOperator<Tuple2<Long, Long>, Tuple2<Long, Long>>, Triggerable<Long, Long> {
- private static final long serialVersionUID = 1L;
-
- private final ValueStateDescriptor<Long> stateDescriptor =
- new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
-
- private transient InternalTimerService<Long> timerService;
-
- @Override
- public void open() throws Exception {
- super.open();
-
- timerService = getInternalTimerService(
- "timer",
- LongSerializer.INSTANCE,
- this);
-
- }
-
- @Override
- public void processElement(StreamRecord<Tuple2<Long, Long>> element) throws Exception {
- ValueState<Long> state = getKeyedStateBackend().getPartitionedState(
- element.getValue().f0,
- LongSerializer.INSTANCE,
- stateDescriptor);
-
- state.update(element.getValue().f1);
-
- timerService.registerEventTimeTimer(element.getValue().f0, timerService.currentWatermark() + 10);
- timerService.registerProcessingTimeTimer(element.getValue().f0, timerService.currentProcessingTime() + 30_000);
-
- output.collect(element);
- }
-
- @Override
- public void onEventTime(InternalTimer<Long, Long> timer) throws Exception {
-
- }
-
- @Override
- public void onProcessingTime(InternalTimer<Long, Long> timer) throws Exception {
-
- }
-
- @Override
- public void processWatermark(Watermark mark) throws Exception {
- output.emitWatermark(mark);
- }
- }
-
- private static class CheckingTimelyStatefulOperator
- extends AbstractStreamOperator<Tuple2<Long, Long>>
- implements OneInputStreamOperator<Tuple2<Long, Long>, Tuple2<Long, Long>>, Triggerable<Long, Long> {
- private static final long serialVersionUID = 1L;
-
- public static final String SUCCESSFUL_PROCESS_CHECK_ACCUMULATOR = CheckingTimelyStatefulOperator.class + "_PROCESS_CHECKS";
- public static final String SUCCESSFUL_EVENT_TIME_CHECK_ACCUMULATOR = CheckingTimelyStatefulOperator.class + "_ET_CHECKS";
- public static final String SUCCESSFUL_PROCESSING_TIME_CHECK_ACCUMULATOR = CheckingTimelyStatefulOperator.class + "_PT_CHECKS";
-
- private final ValueStateDescriptor<Long> stateDescriptor =
- new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
-
- private transient InternalTimerService<Long> timerService;
-
- @Override
- public void open() throws Exception {
- super.open();
-
- timerService = getInternalTimerService(
- "timer",
- LongSerializer.INSTANCE,
- this);
-
- getRuntimeContext().addAccumulator(SUCCESSFUL_PROCESS_CHECK_ACCUMULATOR, new IntCounter());
- getRuntimeContext().addAccumulator(SUCCESSFUL_EVENT_TIME_CHECK_ACCUMULATOR, new IntCounter());
- getRuntimeContext().addAccumulator(SUCCESSFUL_PROCESSING_TIME_CHECK_ACCUMULATOR, new IntCounter());
- }
-
- @Override
- public void processElement(StreamRecord<Tuple2<Long, Long>> element) throws Exception {
- ValueState<Long> state = getKeyedStateBackend().getPartitionedState(
- element.getValue().f0,
- LongSerializer.INSTANCE,
- stateDescriptor);
-
- assertEquals(state.value(), element.getValue().f1);
- getRuntimeContext().getAccumulator(SUCCESSFUL_PROCESS_CHECK_ACCUMULATOR).add(1);
-
- output.collect(element);
- }
-
- @Override
- public void onEventTime(InternalTimer<Long, Long> timer) throws Exception {
- ValueState<Long> state = getKeyedStateBackend().getPartitionedState(
- timer.getNamespace(),
- LongSerializer.INSTANCE,
- stateDescriptor);
-
- assertEquals(state.value(), timer.getNamespace());
- getRuntimeContext().getAccumulator(SUCCESSFUL_EVENT_TIME_CHECK_ACCUMULATOR).add(1);
- }
-
- @Override
- public void onProcessingTime(InternalTimer<Long, Long> timer) throws Exception {
- ValueState<Long> state = getKeyedStateBackend().getPartitionedState(
- timer.getNamespace(),
- LongSerializer.INSTANCE,
- stateDescriptor);
-
- assertEquals(state.value(), timer.getNamespace());
- getRuntimeContext().getAccumulator(SUCCESSFUL_PROCESSING_TIME_CHECK_ACCUMULATOR).add(1);
- }
- }
-
- private static class AccumulatorCountingSink<T> extends RichSinkFunction<T> {
- private static final long serialVersionUID = 1L;
-
- public static final String NUM_ELEMENTS_ACCUMULATOR = AccumulatorCountingSink.class + "_NUM_ELEMENTS";
-
- int count = 0;
-
- @Override
- public void open(Configuration parameters) throws Exception {
- super.open(parameters);
-
- getRuntimeContext().addAccumulator(NUM_ELEMENTS_ACCUMULATOR, new IntCounter());
- }
-
- @Override
- public void invoke(T value) throws Exception {
- count++;
- getRuntimeContext().getAccumulator(NUM_ELEMENTS_ACCUMULATOR).add(1);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/2ada8567/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointFrom13MigrationITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointFrom13MigrationITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointFrom13MigrationITCase.java
deleted file mode 100644
index a2d3201..0000000
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointFrom13MigrationITCase.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.test.checkpointing.utils;
-
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * This verifies that we can restore a complete job from a Flink 1.3 savepoint.
- *
- * <p>The test pipeline contains both "Checkpointed" state and keyed user state.
- */
-public class StatefulJobSavepointFrom13MigrationITCase extends StatefulJobSavepointFrom12MigrationITCase {
-
- /**
- * This has to be manually executed to create the savepoint on Flink 1.3.
- */
- @Test
- @Ignore
- public void testCreateSavepointOnFlink13() throws Exception {
- testCreateSavepointOnFlink12();
- }
-
- /**
- * This has to be manually executed to create the savepoint on Flink 1.3.
- */
- @Test
- @Ignore
- public void testCreateSavepointOnFlink13WithRocksDB() throws Exception {
- testCreateSavepointOnFlink12WithRocksDB();
- }
-
- @Override
- protected String getSavepointPath() {
- return "stateful-udf-migration-itcase-flink1.3-savepoint";
- }
-
- @Override
- protected String getRocksDBSavepointPath() {
- return "stateful-udf-migration-itcase-flink1.3-rocksdb-savepoint";
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/2ada8567/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointMigrationITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointMigrationITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointMigrationITCase.java
new file mode 100644
index 0000000..28f7e4b
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointMigrationITCase.java
@@ -0,0 +1,661 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.checkpointing.utils;
+
+import org.apache.flink.api.common.accumulators.IntCounter;
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.api.common.functions.RichFlatMapFunction;
+import org.apache.flink.api.common.restartstrategy.RestartStrategies;
+import org.apache.flink.api.common.state.ValueState;
+import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.typeinfo.TypeHint;
+import org.apache.flink.api.common.typeutils.base.LongSerializer;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.contrib.streaming.state.RocksDBStateBackend;
+import org.apache.flink.runtime.state.AbstractStateBackend;
+import org.apache.flink.runtime.state.memory.MemoryStateBackend;
+import org.apache.flink.streaming.api.TimeCharacteristic;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
+import org.apache.flink.streaming.api.functions.source.RichSourceFunction;
+import org.apache.flink.streaming.api.functions.source.SourceFunction;
+import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
+import org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator;
+import org.apache.flink.streaming.api.operators.InternalTimer;
+import org.apache.flink.streaming.api.operators.InternalTimerService;
+import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
+import org.apache.flink.streaming.api.operators.TimestampedCollector;
+import org.apache.flink.streaming.api.operators.Triggerable;
+import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.streaming.util.migration.MigrationVersion;
+import org.apache.flink.util.Collector;
+
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Migration ITCases for a stateful job. The tests are parameterized to cover
+ * migrating for multiple previous Flink versions, as well as for different state backends.
+ */
+@RunWith(Parameterized.class)
+public class StatefulJobSavepointMigrationITCase extends SavepointMigrationTestBase {
+
+ private static final int NUM_SOURCE_ELEMENTS = 4;
+
+ @Parameterized.Parameters(name = "Migrate Savepoint / Backend: {0}")
+ public static Collection<Tuple2<MigrationVersion, String>> parameters () {
+ return Arrays.asList(
+ Tuple2.of(MigrationVersion.v1_2, AbstractStateBackend.MEMORY_STATE_BACKEND_NAME),
+ Tuple2.of(MigrationVersion.v1_2, AbstractStateBackend.ROCKSDB_STATE_BACKEND_NAME),
+ Tuple2.of(MigrationVersion.v1_3, AbstractStateBackend.MEMORY_STATE_BACKEND_NAME),
+ Tuple2.of(MigrationVersion.v1_3, AbstractStateBackend.ROCKSDB_STATE_BACKEND_NAME));
+ }
+
+ /**
+ * TODO to generate savepoints for a specific Flink version / backend type,
+ * TODO change these values accordingly, e.g. to generate for 1.3 with RocksDB,
+ * TODO set as (MigrationVersion.v1_3, AbstractStateBackend.ROCKSDB_STATE_BACKEND_NAME)
+ */
+ private final MigrationVersion flinkGenerateSavepointVersion = MigrationVersion.v1_3;
+ private final String flinkGenerateSavepointBackendType = AbstractStateBackend.ROCKSDB_STATE_BACKEND_NAME;
+
+ private final MigrationVersion testMigrateVersion;
+ private final String testStateBackend;
+
+ public StatefulJobSavepointMigrationITCase(Tuple2<MigrationVersion, String> testMigrateVersionAndBackend) {
+ this.testMigrateVersion = testMigrateVersionAndBackend.f0;
+ this.testStateBackend = testMigrateVersionAndBackend.f1;
+ }
+
+ /**
+ * Manually run this to write binary snapshot data.
+ */
+ @Test
+ @Ignore
+ public void writeSavepoint() throws Exception {
+
+ final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
+
+ switch (flinkGenerateSavepointBackendType) {
+ case AbstractStateBackend.ROCKSDB_STATE_BACKEND_NAME:
+ env.setStateBackend(new RocksDBStateBackend(new MemoryStateBackend()));
+ break;
+ case AbstractStateBackend.MEMORY_STATE_BACKEND_NAME:
+ env.setStateBackend(new MemoryStateBackend());
+ break;
+ default:
+ throw new UnsupportedOperationException();
+ }
+
+ env.enableCheckpointing(500);
+ env.setParallelism(4);
+ env.setMaxParallelism(4);
+
+ env
+ .addSource(new LegacyCheckpointedSource(NUM_SOURCE_ELEMENTS)).setMaxParallelism(1).uid("LegacyCheckpointedSource")
+ .flatMap(new LegacyCheckpointedFlatMap()).startNewChain().uid("LegacyCheckpointedFlatMap")
+ .keyBy(0)
+ .flatMap(new LegacyCheckpointedFlatMapWithKeyedState()).startNewChain().uid("LegacyCheckpointedFlatMapWithKeyedState")
+ .keyBy(0)
+ .flatMap(new KeyedStateSettingFlatMap()).startNewChain().uid("KeyedStateSettingFlatMap")
+ .keyBy(0)
+ .transform(
+ "custom_operator",
+ new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
+ new CheckpointedUdfOperator(new LegacyCheckpointedFlatMapWithKeyedState())).uid("LegacyCheckpointedOperator")
+ .keyBy(0)
+ .transform(
+ "timely_stateful_operator",
+ new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
+ new TimelyStatefulOperator()).uid("TimelyStatefulOperator")
+ .addSink(new AccumulatorCountingSink<Tuple2<Long, Long>>());
+
+ executeAndSavepoint(
+ env,
+ "src/test/resources/" + getSavepointPath(flinkGenerateSavepointVersion, flinkGenerateSavepointBackendType),
+ new Tuple2<>(AccumulatorCountingSink.NUM_ELEMENTS_ACCUMULATOR, NUM_SOURCE_ELEMENTS));
+ }
+
+ @Test
+ public void testSavepointRestore() throws Exception {
+
+ final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setRestartStrategy(RestartStrategies.noRestart());
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
+
+ switch (testStateBackend) {
+ case AbstractStateBackend.ROCKSDB_STATE_BACKEND_NAME:
+ env.setStateBackend(new RocksDBStateBackend(new MemoryStateBackend()));
+ break;
+ case AbstractStateBackend.MEMORY_STATE_BACKEND_NAME:
+ env.setStateBackend(new MemoryStateBackend());
+ break;
+ default:
+ throw new UnsupportedOperationException();
+ }
+
+ env.enableCheckpointing(500);
+ env.setParallelism(4);
+ env.setMaxParallelism(4);
+
+ env
+ .addSource(new CheckingRestoringSource(NUM_SOURCE_ELEMENTS)).setMaxParallelism(1).uid("LegacyCheckpointedSource")
+ .flatMap(new CheckingRestoringFlatMap()).startNewChain().uid("LegacyCheckpointedFlatMap")
+ .keyBy(0)
+ .flatMap(new CheckingRestoringFlatMapWithKeyedState()).startNewChain().uid("LegacyCheckpointedFlatMapWithKeyedState")
+ .keyBy(0)
+ .flatMap(new CheckingKeyedStateFlatMap()).startNewChain().uid("KeyedStateSettingFlatMap")
+ .keyBy(0)
+ .transform(
+ "custom_operator",
+ new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
+ new CheckingRestoringUdfOperator(new CheckingRestoringFlatMapWithKeyedStateInOperator())).uid("LegacyCheckpointedOperator")
+ .keyBy(0)
+ .transform(
+ "timely_stateful_operator",
+ new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
+ new CheckingTimelyStatefulOperator()).uid("TimelyStatefulOperator")
+ .addSink(new AccumulatorCountingSink<Tuple2<Long, Long>>());
+
+ restoreAndExecute(
+ env,
+ getResourceFilename(getSavepointPath(testMigrateVersion, testStateBackend)),
+ new Tuple2<>(CheckingRestoringSource.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, 1),
+ new Tuple2<>(CheckingRestoringFlatMap.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
+ new Tuple2<>(CheckingRestoringFlatMapWithKeyedState.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
+ new Tuple2<>(CheckingKeyedStateFlatMap.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
+ new Tuple2<>(CheckingRestoringUdfOperator.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
+ new Tuple2<>(CheckingRestoringFlatMapWithKeyedStateInOperator.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
+ new Tuple2<>(CheckingTimelyStatefulOperator.SUCCESSFUL_PROCESS_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
+ new Tuple2<>(CheckingTimelyStatefulOperator.SUCCESSFUL_EVENT_TIME_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
+ new Tuple2<>(CheckingTimelyStatefulOperator.SUCCESSFUL_PROCESSING_TIME_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
+ new Tuple2<>(AccumulatorCountingSink.NUM_ELEMENTS_ACCUMULATOR, NUM_SOURCE_ELEMENTS));
+ }
+
+ private String getSavepointPath(MigrationVersion savepointVersion, String backendType) {
+ switch (backendType) {
+ case AbstractStateBackend.ROCKSDB_STATE_BACKEND_NAME:
+ return "stateful-udf-migration-itcase-flink" + savepointVersion + "-rocksdb-savepoint";
+ case AbstractStateBackend.MEMORY_STATE_BACKEND_NAME:
+ return "stateful-udf-migration-itcase-flink" + savepointVersion + "-savepoint";
+ default:
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ private static class LegacyCheckpointedSource
+ implements SourceFunction<Tuple2<Long, Long>> {
+
+ public static String checkpointedString = "Here be dragons!";
+
+ private static final long serialVersionUID = 1L;
+
+ private volatile boolean isRunning = true;
+
+ private final int numElements;
+
+ public LegacyCheckpointedSource(int numElements) {
+ this.numElements = numElements;
+ }
+
+ @Override
+ public void run(SourceContext<Tuple2<Long, Long>> ctx) throws Exception {
+
+ ctx.emitWatermark(new Watermark(0));
+
+ synchronized (ctx.getCheckpointLock()) {
+ for (long i = 0; i < numElements; i++) {
+ ctx.collect(new Tuple2<>(i, i));
+ }
+ }
+
+ // don't emit a final watermark so that we don't trigger the registered event-time
+ // timers
+ while (isRunning) {
+ Thread.sleep(20);
+ }
+ }
+
+ @Override
+ public void cancel() {
+ isRunning = false;
+ }
+ }
+
+ private static class CheckingRestoringSource
+ extends RichSourceFunction<Tuple2<Long, Long>> {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR = CheckingRestoringSource.class + "_RESTORE_CHECK";
+
+ private volatile boolean isRunning = true;
+
+ private final int numElements;
+
+ private String restoredState;
+
+ public CheckingRestoringSource(int numElements) {
+ this.numElements = numElements;
+ }
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ super.open(parameters);
+
+ getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
+ }
+
+ @Override
+ public void run(SourceContext<Tuple2<Long, Long>> ctx) throws Exception {
+ getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
+
+ // immediately trigger any set timers
+ ctx.emitWatermark(new Watermark(1000));
+
+ synchronized (ctx.getCheckpointLock()) {
+ for (long i = 0; i < numElements; i++) {
+ ctx.collect(new Tuple2<>(i, i));
+ }
+ }
+
+ while (isRunning) {
+ Thread.sleep(20);
+ }
+ }
+
+ @Override
+ public void cancel() {
+ isRunning = false;
+ }
+ }
+
+ private static class LegacyCheckpointedFlatMap extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
+
+ private static final long serialVersionUID = 1L;
+
+ public static Tuple2<String, Long> checkpointedTuple =
+ new Tuple2<>("hello", 42L);
+
+ @Override
+ public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
+ out.collect(value);
+ }
+ }
+
+ private static class CheckingRestoringFlatMap extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR = CheckingRestoringFlatMap.class + "_RESTORE_CHECK";
+
+ private transient Tuple2<String, Long> restoredState;
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ super.open(parameters);
+
+ getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
+ }
+
+ @Override
+ public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
+ out.collect(value);
+
+ getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
+
+ }
+
+ }
+
+ private static class LegacyCheckpointedFlatMapWithKeyedState
+ extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
+
+ private static final long serialVersionUID = 1L;
+
+ public static Tuple2<String, Long> checkpointedTuple =
+ new Tuple2<>("hello", 42L);
+
+ private final ValueStateDescriptor<Long> stateDescriptor =
+ new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
+
+ @Override
+ public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
+ out.collect(value);
+
+ getRuntimeContext().getState(stateDescriptor).update(value.f1);
+
+ assertEquals(value.f1, getRuntimeContext().getState(stateDescriptor).value());
+ }
+ }
+
+ private static class CheckingRestoringFlatMapWithKeyedState
+ extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR = CheckingRestoringFlatMapWithKeyedState.class + "_RESTORE_CHECK";
+
+ private transient Tuple2<String, Long> restoredState;
+
+ private final ValueStateDescriptor<Long> stateDescriptor =
+ new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ super.open(parameters);
+
+ getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
+ }
+
+ @Override
+ public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
+ out.collect(value);
+
+ ValueState<Long> state = getRuntimeContext().getState(stateDescriptor);
+ if (state == null) {
+ throw new RuntimeException("Missing key value state for " + value);
+ }
+
+ assertEquals(value.f1, state.value());
+ getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
+ }
+ }
+
+ private static class CheckingRestoringFlatMapWithKeyedStateInOperator
+ extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR = CheckingRestoringFlatMapWithKeyedStateInOperator.class + "_RESTORE_CHECK";
+
+ private transient Tuple2<String, Long> restoredState;
+
+ private final ValueStateDescriptor<Long> stateDescriptor =
+ new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ super.open(parameters);
+
+ getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
+ }
+
+ @Override
+ public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
+ out.collect(value);
+
+ ValueState<Long> state = getRuntimeContext().getState(stateDescriptor);
+ if (state == null) {
+ throw new RuntimeException("Missing key value state for " + value);
+ }
+
+ assertEquals(value.f1, state.value());
+ getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
+ }
+ }
+
+ private static class KeyedStateSettingFlatMap extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
+
+ private static final long serialVersionUID = 1L;
+
+ private final ValueStateDescriptor<Long> stateDescriptor =
+ new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
+
+ @Override
+ public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
+ out.collect(value);
+
+ getRuntimeContext().getState(stateDescriptor).update(value.f1);
+ }
+ }
+
+ private static class CheckingKeyedStateFlatMap extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR = CheckingKeyedStateFlatMap.class + "_RESTORE_CHECK";
+
+ private final ValueStateDescriptor<Long> stateDescriptor =
+ new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ super.open(parameters);
+
+ getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
+ }
+
+ @Override
+ public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
+ out.collect(value);
+
+ ValueState<Long> state = getRuntimeContext().getState(stateDescriptor);
+ if (state == null) {
+ throw new RuntimeException("Missing key value state for " + value);
+ }
+
+ assertEquals(value.f1, state.value());
+ getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
+ }
+ }
+
+ private static class CheckpointedUdfOperator
+ extends AbstractUdfStreamOperator<Tuple2<Long, Long>, FlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>>
+ implements OneInputStreamOperator<Tuple2<Long, Long>, Tuple2<Long, Long>> {
+ private static final long serialVersionUID = 1L;
+
+ private static final String CHECKPOINTED_STRING = "Oh my, that's nice!";
+
+ public CheckpointedUdfOperator(FlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> userFunction) {
+ super(userFunction);
+ }
+
+ @Override
+ public void processElement(StreamRecord<Tuple2<Long, Long>> element) throws Exception {
+ userFunction.flatMap(element.getValue(), new TimestampedCollector<>(output));
+ }
+
+ @Override
+ public void processWatermark(Watermark mark) throws Exception {
+ output.emitWatermark(mark);
+ }
+ }
+
+ private static class CheckingRestoringUdfOperator
+ extends AbstractUdfStreamOperator<Tuple2<Long, Long>, FlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>>
+ implements OneInputStreamOperator<Tuple2<Long, Long>, Tuple2<Long, Long>> {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR = CheckingRestoringUdfOperator.class + "_RESTORE_CHECK";
+
+ private String restoredState;
+
+ public CheckingRestoringUdfOperator(FlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> userFunction) {
+ super(userFunction);
+ }
+
+ @Override
+ public void open() throws Exception {
+ super.open();
+
+ getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
+ }
+
+ @Override
+ public void processElement(StreamRecord<Tuple2<Long, Long>> element) throws Exception {
+ userFunction.flatMap(element.getValue(), new TimestampedCollector<>(output));
+ getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
+ }
+
+ @Override
+ public void processWatermark(Watermark mark) throws Exception {
+ output.emitWatermark(mark);
+ }
+ }
+
+ private static class TimelyStatefulOperator
+ extends AbstractStreamOperator<Tuple2<Long, Long>>
+ implements OneInputStreamOperator<Tuple2<Long, Long>, Tuple2<Long, Long>>, Triggerable<Long, Long> {
+ private static final long serialVersionUID = 1L;
+
+ private final ValueStateDescriptor<Long> stateDescriptor =
+ new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
+
+ private transient InternalTimerService<Long> timerService;
+
+ @Override
+ public void open() throws Exception {
+ super.open();
+
+ timerService = getInternalTimerService(
+ "timer",
+ LongSerializer.INSTANCE,
+ this);
+
+ }
+
+ @Override
+ public void processElement(StreamRecord<Tuple2<Long, Long>> element) throws Exception {
+ ValueState<Long> state = getKeyedStateBackend().getPartitionedState(
+ element.getValue().f0,
+ LongSerializer.INSTANCE,
+ stateDescriptor);
+
+ state.update(element.getValue().f1);
+
+ timerService.registerEventTimeTimer(element.getValue().f0, timerService.currentWatermark() + 10);
+ timerService.registerProcessingTimeTimer(element.getValue().f0, timerService.currentProcessingTime() + 30_000);
+
+ output.collect(element);
+ }
+
+ @Override
+ public void onEventTime(InternalTimer<Long, Long> timer) throws Exception {
+
+ }
+
+ @Override
+ public void onProcessingTime(InternalTimer<Long, Long> timer) throws Exception {
+
+ }
+
+ @Override
+ public void processWatermark(Watermark mark) throws Exception {
+ output.emitWatermark(mark);
+ }
+ }
+
+ private static class CheckingTimelyStatefulOperator
+ extends AbstractStreamOperator<Tuple2<Long, Long>>
+ implements OneInputStreamOperator<Tuple2<Long, Long>, Tuple2<Long, Long>>, Triggerable<Long, Long> {
+ private static final long serialVersionUID = 1L;
+
+ public static final String SUCCESSFUL_PROCESS_CHECK_ACCUMULATOR = CheckingTimelyStatefulOperator.class + "_PROCESS_CHECKS";
+ public static final String SUCCESSFUL_EVENT_TIME_CHECK_ACCUMULATOR = CheckingTimelyStatefulOperator.class + "_ET_CHECKS";
+ public static final String SUCCESSFUL_PROCESSING_TIME_CHECK_ACCUMULATOR = CheckingTimelyStatefulOperator.class + "_PT_CHECKS";
+
+ private final ValueStateDescriptor<Long> stateDescriptor =
+ new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
+
+ private transient InternalTimerService<Long> timerService;
+
+ @Override
+ public void open() throws Exception {
+ super.open();
+
+ timerService = getInternalTimerService(
+ "timer",
+ LongSerializer.INSTANCE,
+ this);
+
+ getRuntimeContext().addAccumulator(SUCCESSFUL_PROCESS_CHECK_ACCUMULATOR, new IntCounter());
+ getRuntimeContext().addAccumulator(SUCCESSFUL_EVENT_TIME_CHECK_ACCUMULATOR, new IntCounter());
+ getRuntimeContext().addAccumulator(SUCCESSFUL_PROCESSING_TIME_CHECK_ACCUMULATOR, new IntCounter());
+ }
+
+ @Override
+ public void processElement(StreamRecord<Tuple2<Long, Long>> element) throws Exception {
+ ValueState<Long> state = getKeyedStateBackend().getPartitionedState(
+ element.getValue().f0,
+ LongSerializer.INSTANCE,
+ stateDescriptor);
+
+ assertEquals(state.value(), element.getValue().f1);
+ getRuntimeContext().getAccumulator(SUCCESSFUL_PROCESS_CHECK_ACCUMULATOR).add(1);
+
+ output.collect(element);
+ }
+
+ @Override
+ public void onEventTime(InternalTimer<Long, Long> timer) throws Exception {
+ ValueState<Long> state = getKeyedStateBackend().getPartitionedState(
+ timer.getNamespace(),
+ LongSerializer.INSTANCE,
+ stateDescriptor);
+
+ assertEquals(state.value(), timer.getNamespace());
+ getRuntimeContext().getAccumulator(SUCCESSFUL_EVENT_TIME_CHECK_ACCUMULATOR).add(1);
+ }
+
+ @Override
+ public void onProcessingTime(InternalTimer<Long, Long> timer) throws Exception {
+ ValueState<Long> state = getKeyedStateBackend().getPartitionedState(
+ timer.getNamespace(),
+ LongSerializer.INSTANCE,
+ stateDescriptor);
+
+ assertEquals(state.value(), timer.getNamespace());
+ getRuntimeContext().getAccumulator(SUCCESSFUL_PROCESSING_TIME_CHECK_ACCUMULATOR).add(1);
+ }
+ }
+
+ private static class AccumulatorCountingSink<T> extends RichSinkFunction<T> {
+ private static final long serialVersionUID = 1L;
+
+ public static final String NUM_ELEMENTS_ACCUMULATOR = AccumulatorCountingSink.class + "_NUM_ELEMENTS";
+
+ int count = 0;
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ super.open(parameters);
+
+ getRuntimeContext().addAccumulator(NUM_ELEMENTS_ACCUMULATOR, new IntCounter());
+ }
+
+ @Override
+ public void invoke(T value) throws Exception {
+ count++;
+ getRuntimeContext().getAccumulator(NUM_ELEMENTS_ACCUMULATOR).add(1);
+ }
+ }
+
+}
[03/28] flink git commit: [FLINK-8472] [DataStream,
test] Extend StatefulJobSavepointMigrationITCase for Flink 1.4
Posted by tz...@apache.org.
[FLINK-8472] [DataStream, test] Extend StatefulJobSavepointMigrationITCase for Flink 1.4
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/83b9727d
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/83b9727d
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/83b9727d
Branch: refs/heads/release-1.4
Commit: 83b9727dcc9e3c60b2caee147562003e48821908
Parents: 2ada856
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Thu Jan 25 18:37:05 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 17:31:32 2018 +0100
----------------------------------------------------------------------
.../utils/StatefulJobSavepointMigrationITCase.java | 6 ++++--
.../_metadata | Bin 0 -> 32810 bytes
.../_metadata | Bin 0 -> 32738 bytes
3 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/83b9727d/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointMigrationITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointMigrationITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointMigrationITCase.java
index 28f7e4b..641e450b 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointMigrationITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointMigrationITCase.java
@@ -73,7 +73,9 @@ public class StatefulJobSavepointMigrationITCase extends SavepointMigrationTestB
Tuple2.of(MigrationVersion.v1_2, AbstractStateBackend.MEMORY_STATE_BACKEND_NAME),
Tuple2.of(MigrationVersion.v1_2, AbstractStateBackend.ROCKSDB_STATE_BACKEND_NAME),
Tuple2.of(MigrationVersion.v1_3, AbstractStateBackend.MEMORY_STATE_BACKEND_NAME),
- Tuple2.of(MigrationVersion.v1_3, AbstractStateBackend.ROCKSDB_STATE_BACKEND_NAME));
+ Tuple2.of(MigrationVersion.v1_3, AbstractStateBackend.ROCKSDB_STATE_BACKEND_NAME),
+ Tuple2.of(MigrationVersion.v1_4, AbstractStateBackend.MEMORY_STATE_BACKEND_NAME),
+ Tuple2.of(MigrationVersion.v1_4, AbstractStateBackend.ROCKSDB_STATE_BACKEND_NAME));
}
/**
@@ -81,7 +83,7 @@ public class StatefulJobSavepointMigrationITCase extends SavepointMigrationTestB
* TODO change these values accordingly, e.g. to generate for 1.3 with RocksDB,
* TODO set as (MigrationVersion.v1_3, AbstractStateBackend.ROCKSDB_STATE_BACKEND_NAME)
*/
- private final MigrationVersion flinkGenerateSavepointVersion = MigrationVersion.v1_3;
+ private final MigrationVersion flinkGenerateSavepointVersion = MigrationVersion.v1_4;
private final String flinkGenerateSavepointBackendType = AbstractStateBackend.ROCKSDB_STATE_BACKEND_NAME;
private final MigrationVersion testMigrateVersion;
http://git-wip-us.apache.org/repos/asf/flink/blob/83b9727d/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.4-rocksdb-savepoint/_metadata
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.4-rocksdb-savepoint/_metadata b/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.4-rocksdb-savepoint/_metadata
new file mode 100644
index 0000000..61a2549
Binary files /dev/null and b/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.4-rocksdb-savepoint/_metadata differ
http://git-wip-us.apache.org/repos/asf/flink/blob/83b9727d/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.4-savepoint/_metadata
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.4-savepoint/_metadata b/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.4-savepoint/_metadata
new file mode 100644
index 0000000..5e760c2
Binary files /dev/null and b/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.4-savepoint/_metadata differ
[15/28] flink git commit: [hotfix][connectors] Fix log format strings
Posted by tz...@apache.org.
[hotfix][connectors] Fix log format strings
Fixes three occurrences of using '%s' instead of '{}' in logging
statements.
This closes #5380.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/0ad82b39
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/0ad82b39
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/0ad82b39
Branch: refs/heads/release-1.4
Commit: 0ad82b39ad8a4e22a0e4db8a712a1e617e4025fa
Parents: 432273a
Author: Patrick Lucas <me...@patricklucas.com>
Authored: Mon Jan 29 13:58:09 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 17:31:32 2018 +0100
----------------------------------------------------------------------
.../flink/streaming/connectors/kafka/FlinkKafkaProducer011.java | 2 +-
.../java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormat.java | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/0ad82b39/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer011.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer011.java b/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer011.java
index 58355c9..b662de2 100644
--- a/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer011.java
+++ b/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer011.java
@@ -512,7 +512,7 @@ public class FlinkKafkaProducer011<IN>
long timeout = DEFAULT_KAFKA_TRANSACTION_TIMEOUT.toMilliseconds();
checkState(timeout < Integer.MAX_VALUE && timeout > 0, "timeout does not fit into 32 bit integer");
this.producerConfig.put(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG, (int) timeout);
- LOG.warn("Property [%s] not specified. Setting it to %s", ProducerConfig.TRANSACTION_TIMEOUT_CONFIG, DEFAULT_KAFKA_TRANSACTION_TIMEOUT);
+ LOG.warn("Property [{}] not specified. Setting it to {}", ProducerConfig.TRANSACTION_TIMEOUT_CONFIG, DEFAULT_KAFKA_TRANSACTION_TIMEOUT);
}
// Enable transactionTimeoutWarnings to avoid silent data loss
http://git-wip-us.apache.org/repos/asf/flink/blob/0ad82b39/flink-connectors/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormat.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormat.java b/flink-connectors/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormat.java
index 2497712..67c7a0e 100644
--- a/flink-connectors/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormat.java
+++ b/flink-connectors/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormat.java
@@ -116,7 +116,7 @@ public class JDBCOutputFormat extends RichOutputFormat<Row> {
if (typesArray == null) {
// no types provided
for (int index = 0; index < row.getArity(); index++) {
- LOG.warn("Unknown column type for column %s. Best effort approach to set its value: %s.", index + 1, row.getField(index));
+ LOG.warn("Unknown column type for column {}. Best effort approach to set its value: {}.", index + 1, row.getField(index));
upload.setObject(index + 1, row.getField(index));
}
} else {
@@ -181,7 +181,7 @@ public class JDBCOutputFormat extends RichOutputFormat<Row> {
break;
default:
upload.setObject(index + 1, row.getField(index));
- LOG.warn("Unmanaged sql type (%s) for column %s. Best effort approach to set its value: %s.",
+ LOG.warn("Unmanaged sql type ({}) for column {}. Best effort approach to set its value: {}.",
typesArray[index], index + 1, row.getField(index));
// case java.sql.Types.SQLXML
// case java.sql.Types.ARRAY:
[19/28] flink git commit: [FLINK-8472] [fs,
test] Extend ContinuousFileProcessingMigrationTest for Flink 1.4
Posted by tz...@apache.org.
[FLINK-8472] [fs, test] Extend ContinuousFileProcessingMigrationTest for Flink 1.4
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/8d910963
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/8d910963
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/8d910963
Branch: refs/heads/release-1.4
Commit: 8d910963fcaa77993b9825cb5b3b915a20618419
Parents: da47bd2
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Thu Jan 25 17:28:17 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 17:31:32 2018 +0100
----------------------------------------------------------------------
.../ContinuousFileProcessingMigrationTest.java | 3 ++-
...n-migration-test-1516897628000-flink1.4-snapshot | Bin 0 -> 537 bytes
.../reader-migration-test-flink1.4-snapshot | Bin 0 -> 2823 bytes
3 files changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/8d910963/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingMigrationTest.java
----------------------------------------------------------------------
diff --git a/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingMigrationTest.java b/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingMigrationTest.java
index 602ad3e..1c2ea6f 100644
--- a/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingMigrationTest.java
+++ b/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingMigrationTest.java
@@ -77,7 +77,8 @@ public class ContinuousFileProcessingMigrationTest {
public static Collection<Tuple2<MigrationVersion, Long>> parameters () {
return Arrays.asList(
Tuple2.of(MigrationVersion.v1_2, 1493116191000L),
- Tuple2.of(MigrationVersion.v1_3, 1496532000000L));
+ Tuple2.of(MigrationVersion.v1_3, 1496532000000L),
+ Tuple2.of(MigrationVersion.v1_4, 1516897628000L));
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/8d910963/flink-fs-tests/src/test/resources/monitoring-function-migration-test-1516897628000-flink1.4-snapshot
----------------------------------------------------------------------
diff --git a/flink-fs-tests/src/test/resources/monitoring-function-migration-test-1516897628000-flink1.4-snapshot b/flink-fs-tests/src/test/resources/monitoring-function-migration-test-1516897628000-flink1.4-snapshot
new file mode 100644
index 0000000..7b98325
Binary files /dev/null and b/flink-fs-tests/src/test/resources/monitoring-function-migration-test-1516897628000-flink1.4-snapshot differ
http://git-wip-us.apache.org/repos/asf/flink/blob/8d910963/flink-fs-tests/src/test/resources/reader-migration-test-flink1.4-snapshot
----------------------------------------------------------------------
diff --git a/flink-fs-tests/src/test/resources/reader-migration-test-flink1.4-snapshot b/flink-fs-tests/src/test/resources/reader-migration-test-flink1.4-snapshot
new file mode 100644
index 0000000..c5d0ecf
Binary files /dev/null and b/flink-fs-tests/src/test/resources/reader-migration-test-flink1.4-snapshot differ
[11/28] flink git commit: [FLINK-8484] [kinesis] Synchronize local
shard metainfo's on restore
Posted by tz...@apache.org.
[FLINK-8484] [kinesis] Synchronize local shard metainfo's on restore
Ensure that a metainfo change in the StreamShardMetadata other than
`StreamShardMetadata.shardId` or `StreamShardMetadata.streamName` does
not result in the shard not being able to be restored. This handles the
corner case where a shard might have been closed (ending sequence number
set to not-null) since the last savepoint or checkpoint when a job is
restarted from a snapshot state.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/426cf469
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/426cf469
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/426cf469
Branch: refs/heads/release-1.4
Commit: 426cf46970630d6606938ecb9c19fcecbb2c80f5
Parents: ca1e525
Author: Philip Luppens <ph...@gmail.com>
Authored: Tue Jan 23 10:00:23 2018 +0200
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 17:31:32 2018 +0100
----------------------------------------------------------------------
.../kinesis/FlinkKinesisConsumer.java | 49 ++++++++
.../kinesis/FlinkKinesisConsumerTest.java | 125 +++++++++++++++++++
2 files changed, 174 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/426cf469/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumer.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumer.java b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumer.java
index a3681ec..de66f93 100644
--- a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumer.java
+++ b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumer.java
@@ -208,6 +208,16 @@ public class FlinkKinesisConsumer<T> extends RichParallelSourceFunction<T> imple
for (StreamShardHandle shard : allShards) {
StreamShardMetadata kinesisStreamShard = KinesisDataFetcher.convertToStreamShardMetadata(shard);
if (sequenceNumsToRestore != null) {
+
+ // We need to do this to make sure that a shard that was closed after this restored state was taken will be properly
+ // detected and have its sequence numbers restored. A shard will be closed when re-sharding, which can happen when
+ // scaling up & down the Kinesis stream, and if the state is not synchronized, then the equality check of the current
+ // Kinesis shard will not match the stored state, which will cause us to re-read the entire shard from the event horizon.
+ if (updateKinesisShardStateWithMissingEndingSequenceNumber(kinesisStreamShard, sequenceNumsToRestore)) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Updated local stored state for shard {} with a new ending number: {}", kinesisStreamShard.getShardId(), sequenceNumsToRestore.get(kinesisStreamShard));
+ }
+ }
if (sequenceNumsToRestore.containsKey(kinesisStreamShard)) {
// if the shard was already seen and is contained in the state,
// just use the sequence number stored in the state
@@ -265,6 +275,45 @@ public class FlinkKinesisConsumer<T> extends RichParallelSourceFunction<T> imple
sourceContext.close();
}
+ /**
+ * Synchronizes the Kinesis shard information from the current Kinesis shard with the restored state, if we find
+ * a shard that match the shardId and streamName. If we find one, and its ending key is different that what we
+ * have in our stored state, then we update the stored's shard's metadata's ending number.
+ *
+ * @param current the current Kinesis shard we're trying to synchronize.
+ * @param sequenceNumsToRestore the (re)stored shard metadata and their sequence numbers.
+ * @return {@code true} if the local state was updated with the current Kinesis shard's ending number.
+ */
+ @VisibleForTesting
+ boolean updateKinesisShardStateWithMissingEndingSequenceNumber(StreamShardMetadata current, HashMap<StreamShardMetadata, SequenceNumber> sequenceNumsToRestore) {
+ checkNotNull(current.getStreamName(), "Stream name not set on the current metadata shard");
+ checkNotNull(current.getShardId(), "Shard id not set on the current metadata shard");
+
+ // short-circuit: if the current shard doesn't have an ending sequence number, then there's no point in trying to update the local state
+ // since that's the only property that can change.
+ if (current.getEndingSequenceNumber() == null) {
+ return false;
+ }
+
+ // try to find the matching shard based on the id & stream name
+ for (Map.Entry<StreamShardMetadata, SequenceNumber> entry : sequenceNumsToRestore.entrySet()) {
+ if (current.getStreamName().equals(entry.getKey().getStreamName())
+ && current.getShardId().equals(entry.getKey().getShardId())) {
+ // synchronize the local state if the ending sequence number is different
+ if (!current.getEndingSequenceNumber().equals(entry.getKey().getEndingSequenceNumber())) {
+ // ugly, but since the hashcode will change, we'll need to remove it and add it back
+ sequenceNumsToRestore.remove(entry.getKey());
+ entry.getKey().setEndingSequenceNumber(current.getEndingSequenceNumber());
+ sequenceNumsToRestore.put(entry.getKey(), entry.getValue());
+ return true;
+ }
+ // we already found the matching shard
+ break;
+ }
+ }
+ return false;
+ }
+
@Override
public void cancel() {
running = false;
http://git-wip-us.apache.org/repos/asf/flink/blob/426cf469/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerTest.java b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerTest.java
index 2b1fcf4..a142b26 100644
--- a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerTest.java
+++ b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerTest.java
@@ -63,12 +63,14 @@ import org.powermock.modules.junit4.PowerMockRunner;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.UUID;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
@@ -522,6 +524,129 @@ public class FlinkKinesisConsumerTest {
assertTrue(typeInformation.createSerializer(new ExecutionConfig()) instanceof PojoSerializer);
}
+ /**
+ * FLINK-8484: ensure that a state change in the StreamShardMetadata other than {@link StreamShardMetadata#shardId} or
+ * {@link StreamShardMetadata#streamName} does not result in the shard not being able to be restored.
+ * This handles the corner case where the stored shard metadata is open (no ending sequence number), but after the
+ * job restore, the shard has been closed (ending number set) due to re-sharding, and we can no longer rely on
+ * {@link StreamShardMetadata#equals(Object)} to find back the sequence number in the collection of restored shard metadata.
+ * <p></p>
+ * Therefore, we will rely on synchronizing the snapshot's state with the Kinesis shard before attempting to find back
+ * the sequence number to restore.
+ */
+ @Test
+ public void testFindSequenceNumberToRestoreFromIfTheShardHasBeenClosedSinceTheStateWasStored() throws Exception {
+ // ----------------------------------------------------------------------
+ // setup initial state
+ // ----------------------------------------------------------------------
+
+ HashMap<StreamShardHandle, SequenceNumber> fakeRestoredState = getFakeRestoredStore("all");
+
+ // ----------------------------------------------------------------------
+ // mock operator state backend and initial state for initializeState()
+ // ----------------------------------------------------------------------
+
+ TestingListState<Tuple2<StreamShardMetadata, SequenceNumber>> listState = new TestingListState<>();
+ for (Map.Entry<StreamShardHandle, SequenceNumber> state : fakeRestoredState.entrySet()) {
+ listState.add(Tuple2.of(KinesisDataFetcher.convertToStreamShardMetadata(state.getKey()), state.getValue()));
+ }
+
+ OperatorStateStore operatorStateStore = mock(OperatorStateStore.class);
+ when(operatorStateStore.getUnionListState(Matchers.any(ListStateDescriptor.class))).thenReturn(listState);
+
+ StateInitializationContext initializationContext = mock(StateInitializationContext.class);
+ when(initializationContext.getOperatorStateStore()).thenReturn(operatorStateStore);
+ when(initializationContext.isRestored()).thenReturn(true);
+
+ // ----------------------------------------------------------------------
+ // mock fetcher
+ // ----------------------------------------------------------------------
+
+ KinesisDataFetcher mockedFetcher = Mockito.mock(KinesisDataFetcher.class);
+ List<StreamShardHandle> shards = new ArrayList<>();
+
+ // create a fake stream shard handle based on the first entry in the restored state
+ final StreamShardHandle originalStreamShardHandle = fakeRestoredState.keySet().iterator().next();
+ final StreamShardHandle closedStreamShardHandle = new StreamShardHandle(originalStreamShardHandle.getStreamName(), originalStreamShardHandle.getShard());
+ // close the shard handle by setting an ending sequence number
+ final SequenceNumberRange sequenceNumberRange = new SequenceNumberRange();
+ sequenceNumberRange.setEndingSequenceNumber("1293844");
+ closedStreamShardHandle.getShard().setSequenceNumberRange(sequenceNumberRange);
+
+ shards.add(closedStreamShardHandle);
+
+ when(mockedFetcher.discoverNewShardsToSubscribe()).thenReturn(shards);
+ PowerMockito.whenNew(KinesisDataFetcher.class).withAnyArguments().thenReturn(mockedFetcher);
+
+ // assume the given config is correct
+ PowerMockito.mockStatic(KinesisConfigUtil.class);
+ PowerMockito.doNothing().when(KinesisConfigUtil.class);
+
+ // ----------------------------------------------------------------------
+ // start to test fetcher's initial state seeding
+ // ----------------------------------------------------------------------
+
+ TestableFlinkKinesisConsumer consumer = new TestableFlinkKinesisConsumer(
+ "fakeStream", new Properties(), 10, 2);
+ consumer.initializeState(initializationContext);
+ consumer.open(new Configuration());
+ consumer.run(Mockito.mock(SourceFunction.SourceContext.class));
+
+ Mockito.verify(mockedFetcher).registerNewSubscribedShardState(
+ new KinesisStreamShardState(KinesisDataFetcher.convertToStreamShardMetadata(closedStreamShardHandle),
+ closedStreamShardHandle, fakeRestoredState.get(closedStreamShardHandle)));
+ }
+
+ @Test
+ public void testUpdateKinesisShardStateWithMissingEndingSequenceNumber() {
+ final String streamName = "fakeStream1";
+ final String shardId = "shard-000001";
+
+ Properties config = TestUtils.getStandardProperties();
+ FlinkKinesisConsumer<String> consumer = new FlinkKinesisConsumer<>(streamName, new SimpleStringSchema(), config);
+
+ // having the current Kinesis shard we're trying to find the restored state for
+ final StreamShardMetadata current = new StreamShardMetadata();
+ current.setShardId(shardId);
+ current.setStreamName(streamName);
+ current.setEndingSequenceNumber(null);
+
+ final HashMap<StreamShardMetadata, SequenceNumber> sequenceNumsToRestore = new LinkedHashMap<>();
+ assertFalse("Current shard is open, expecting a short-circuit", consumer.updateKinesisShardStateWithMissingEndingSequenceNumber(current, sequenceNumsToRestore));
+
+ // create some non-matching shards
+ final StreamShardMetadata differentStreamName = new StreamShardMetadata();
+ differentStreamName.setStreamName("fakeStream2");
+ differentStreamName.setShardId(shardId);
+
+ final StreamShardMetadata differentShardId = new StreamShardMetadata();
+ differentShardId.setStreamName(streamName);
+ differentShardId.setShardId("shard-000002");
+
+ // create the matching shard
+ final StreamShardMetadata match = new StreamShardMetadata();
+ match.setStreamName(streamName);
+ match.setShardId(shardId);
+ // ensure the sequence number isn't set (shard is considered in open state)
+ match.setEndingSequenceNumber(null);
+
+ sequenceNumsToRestore.put(differentStreamName, new SequenceNumber("123"));
+ sequenceNumsToRestore.put(differentShardId, new SequenceNumber("456"));
+ sequenceNumsToRestore.put(match, new SequenceNumber("789"));
+
+ assertFalse("No ending sequence number was set, so no synchronisation was done.", consumer.updateKinesisShardStateWithMissingEndingSequenceNumber(current, sequenceNumsToRestore));
+
+ // alter the ending sequence number (indicating the shard is now closed)
+ final String endingSequenceNumber = "99999";
+ current.setEndingSequenceNumber(endingSequenceNumber);
+ assertTrue("Shard was closed (ending number set).", consumer.updateKinesisShardStateWithMissingEndingSequenceNumber(current, sequenceNumsToRestore));
+ assertEquals(endingSequenceNumber, match.getEndingSequenceNumber());
+ assertEquals(current, match);
+ assertTrue("Make sure we can still find back our match in the restored sequences ", sequenceNumsToRestore.containsKey(match));
+
+ assertFalse("Ending number was already set, no more need for synchronisation", consumer.updateKinesisShardStateWithMissingEndingSequenceNumber(current, sequenceNumsToRestore));
+ }
+
private static final class TestingListState<T> implements ListState<T> {
private final List<T> list = new ArrayList<>();
[13/28] flink git commit: [FLINK-8472] [kinesis,
test] Extend FlinkKinesisConsumerMigrationTest for Flink 1.4
Posted by tz...@apache.org.
[FLINK-8472] [kinesis, test] Extend FlinkKinesisConsumerMigrationTest for Flink 1.4
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/8cef7887
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/8cef7887
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/8cef7887
Branch: refs/heads/release-1.4
Commit: 8cef78878dfcf155a182bafbfacc1d6354c4db9a
Parents: 83b9727
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Thu Jan 25 16:23:17 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 17:31:32 2018 +0100
----------------------------------------------------------------------
.../kinesis/FlinkKinesisConsumerMigrationTest.java | 2 +-
...consumer-migration-test-flink1.4-empty-snapshot | Bin 0 -> 13147 bytes
...nesis-consumer-migration-test-flink1.4-snapshot | Bin 0 -> 13215 bytes
3 files changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/8cef7887/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java
index eddbab0..541d464 100644
--- a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java
+++ b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java
@@ -97,7 +97,7 @@ public class FlinkKinesisConsumerMigrationTest {
@Parameterized.Parameters(name = "Migration Savepoint: {0}")
public static Collection<MigrationVersion> parameters () {
- return Arrays.asList(MigrationVersion.v1_3);
+ return Arrays.asList(MigrationVersion.v1_3, MigrationVersion.v1_4);
}
public FlinkKinesisConsumerMigrationTest(MigrationVersion testMigrateVersion) {
http://git-wip-us.apache.org/repos/asf/flink/blob/8cef7887/flink-connectors/flink-connector-kinesis/src/test/resources/kinesis-consumer-migration-test-flink1.4-empty-snapshot
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/test/resources/kinesis-consumer-migration-test-flink1.4-empty-snapshot b/flink-connectors/flink-connector-kinesis/src/test/resources/kinesis-consumer-migration-test-flink1.4-empty-snapshot
new file mode 100644
index 0000000..fba6258
Binary files /dev/null and b/flink-connectors/flink-connector-kinesis/src/test/resources/kinesis-consumer-migration-test-flink1.4-empty-snapshot differ
http://git-wip-us.apache.org/repos/asf/flink/blob/8cef7887/flink-connectors/flink-connector-kinesis/src/test/resources/kinesis-consumer-migration-test-flink1.4-snapshot
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/test/resources/kinesis-consumer-migration-test-flink1.4-snapshot b/flink-connectors/flink-connector-kinesis/src/test/resources/kinesis-consumer-migration-test-flink1.4-snapshot
new file mode 100644
index 0000000..5ea4009
Binary files /dev/null and b/flink-connectors/flink-connector-kinesis/src/test/resources/kinesis-consumer-migration-test-flink1.4-snapshot differ
[18/28] flink git commit: [FLINK-8472] [fs,
test] Extend BucketingSinkMigrationTest for Flink 1.4
Posted by tz...@apache.org.
[FLINK-8472] [fs, test] Extend BucketingSinkMigrationTest for Flink 1.4
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/6f4116fe
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/6f4116fe
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/6f4116fe
Branch: refs/heads/release-1.4
Commit: 6f4116fe53a36ceea38f18efb7c75c866ab249b0
Parents: 8d91096
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Thu Jan 25 17:32:28 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 17:31:32 2018 +0100
----------------------------------------------------------------------
.../fs/bucketing/BucketingSinkMigrationTest.java | 3 ++-
.../bucketing-sink-migration-test-flink1.4-snapshot | Bin 0 -> 1862 bytes
2 files changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/6f4116fe/flink-connectors/flink-connector-filesystem/src/test/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSinkMigrationTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-filesystem/src/test/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSinkMigrationTest.java b/flink-connectors/flink-connector-filesystem/src/test/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSinkMigrationTest.java
index bfb326c..3b7c7d4 100644
--- a/flink-connectors/flink-connector-filesystem/src/test/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSinkMigrationTest.java
+++ b/flink-connectors/flink-connector-filesystem/src/test/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSinkMigrationTest.java
@@ -86,7 +86,8 @@ public class BucketingSinkMigrationTest {
public static Collection<Tuple2<MigrationVersion, String>> parameters () {
return Arrays.asList(
Tuple2.of(MigrationVersion.v1_2, "/var/folders/v_/ry2wp5fx0y7c1rvr41xy9_700000gn/T/junit9160378385359106772/junit479663758539998903/1970-01-01--01/part-0-"),
- Tuple2.of(MigrationVersion.v1_3, "/var/folders/tv/b_1d8fvx23dgk1_xs8db_95h0000gn/T/junit4273542175898623023/junit3801102997056424640/1970-01-01--01/part-0-"));
+ Tuple2.of(MigrationVersion.v1_3, "/var/folders/tv/b_1d8fvx23dgk1_xs8db_95h0000gn/T/junit4273542175898623023/junit3801102997056424640/1970-01-01--01/part-0-"),
+ Tuple2.of(MigrationVersion.v1_4, "/var/folders/tv/b_1d8fvx23dgk1_xs8db_95h0000gn/T/junit3198043255809479705/junit8947526563966405708/1970-01-01--01/part-0-"));
}
private final MigrationVersion testMigrateVersion;
http://git-wip-us.apache.org/repos/asf/flink/blob/6f4116fe/flink-connectors/flink-connector-filesystem/src/test/resources/bucketing-sink-migration-test-flink1.4-snapshot
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-filesystem/src/test/resources/bucketing-sink-migration-test-flink1.4-snapshot b/flink-connectors/flink-connector-filesystem/src/test/resources/bucketing-sink-migration-test-flink1.4-snapshot
new file mode 100644
index 0000000..45e7bcc
Binary files /dev/null and b/flink-connectors/flink-connector-filesystem/src/test/resources/bucketing-sink-migration-test-flink1.4-snapshot differ
[06/28] flink git commit: [hotfix] [kafka] Make
AbortedReassignmentException a static class
Posted by tz...@apache.org.
[hotfix] [kafka] Make AbortedReassignmentException a static class
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/a951852f
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/a951852f
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/a951852f
Branch: refs/heads/release-1.4
Commit: a951852fb4a2059cbd2c23659e7b456691639873
Parents: 0816300
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Sat Jan 13 20:35:03 2018 +0800
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 17:31:32 2018 +0100
----------------------------------------------------------------------
.../streaming/connectors/kafka/internal/KafkaConsumerThread.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/a951852f/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java
index c462b26..ee0a63b 100644
--- a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java
+++ b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java
@@ -505,7 +505,7 @@ public class KafkaConsumerThread extends Thread {
* Utility exception that serves as a signal for the main loop to continue through the loop
* if a reassignment attempt was aborted due to an pre-reassignment wakeup call on the consumer.
*/
- private class AbortedReassignmentException extends Exception {
+ private static class AbortedReassignmentException extends Exception {
private static final long serialVersionUID = 1L;
}
}
[27/28] flink git commit: [FLINK-8275] [security,
yarn] Fix keytab local path in YarnTaskManagerRunner
Posted by tz...@apache.org.
[FLINK-8275] [security, yarn] Fix keytab local path in YarnTaskManagerRunner
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/20146652
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/20146652
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/20146652
Branch: refs/heads/release-1.4
Commit: 20146652baff31c3824f7ae31506b8c481cdf56c
Parents: fb1e24e
Author: Shuyi Chen <sh...@uber.com>
Authored: Sun Dec 17 23:50:07 2017 -0800
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 18:30:32 2018 +0100
----------------------------------------------------------------------
.../flink/runtime/security/SecurityUtils.java | 4 +-
.../runtime/security/modules/HadoopModule.java | 6 ++
.../yarn/TestingYarnTaskManagerRunner.java | 3 +-
.../flink/yarn/TestingYarnTaskManager.scala | 3 +-
.../flink/yarn/YarnApplicationMasterRunner.java | 31 ++------
.../flink/yarn/YarnTaskManagerRunner.java | 64 +++++++++--------
.../org/apache/flink/yarn/YarnTaskManager.scala | 3 +-
.../flink/yarn/YarnTaskManagerRunnerTest.java | 74 ++++++++++++++++++++
flink-yarn/src/test/resources/flink-conf.yaml | 23 ++++++
flink-yarn/src/test/resources/krb5.keytab | 0
pom.xml | 1 +
11 files changed, 151 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/20146652/flink-runtime/src/main/java/org/apache/flink/runtime/security/SecurityUtils.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/security/SecurityUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/security/SecurityUtils.java
index c73e966..cb68da4 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/security/SecurityUtils.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/security/SecurityUtils.java
@@ -18,7 +18,6 @@
package org.apache.flink.runtime.security;
-import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.runtime.security.modules.SecurityModule;
import org.apache.flink.runtime.security.modules.SecurityModuleFactory;
@@ -47,8 +46,7 @@ public class SecurityUtils {
return installedContext;
}
- @VisibleForTesting
- static List<SecurityModule> getInstalledModules() {
+ public static List<SecurityModule> getInstalledModules() {
return installedModules;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/20146652/flink-runtime/src/main/java/org/apache/flink/runtime/security/modules/HadoopModule.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/security/modules/HadoopModule.java b/flink-runtime/src/main/java/org/apache/flink/runtime/security/modules/HadoopModule.java
index a39d879..0738708 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/security/modules/HadoopModule.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/security/modules/HadoopModule.java
@@ -18,6 +18,7 @@
package org.apache.flink.runtime.security.modules;
+import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.runtime.security.SecurityConfiguration;
import org.apache.flink.runtime.util.HadoopUtils;
@@ -58,6 +59,11 @@ public class HadoopModule implements SecurityModule {
this.hadoopConfiguration = checkNotNull(hadoopConfiguration);
}
+ @VisibleForTesting
+ public SecurityConfiguration getSecurityConfig() {
+ return securityConfig;
+ }
+
@Override
public void install() throws SecurityInstallException {
http://git-wip-us.apache.org/repos/asf/flink/blob/20146652/flink-yarn-tests/src/test/java/org/apache/flink/yarn/TestingYarnTaskManagerRunner.java
----------------------------------------------------------------------
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/TestingYarnTaskManagerRunner.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/TestingYarnTaskManagerRunner.java
index 8586a77..90cdf9f 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/TestingYarnTaskManagerRunner.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/TestingYarnTaskManagerRunner.java
@@ -25,6 +25,7 @@ import java.io.IOException;
*/
public class TestingYarnTaskManagerRunner {
public static void main(String[] args) throws IOException {
- YarnTaskManagerRunner.runYarnTaskManager(args, TestingYarnTaskManager.class);
+ YarnTaskManagerRunner.runYarnTaskManager(
+ args, TestingYarnTaskManager.class, System.getenv(), null);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/20146652/flink-yarn-tests/src/test/scala/org/apache/flink/yarn/TestingYarnTaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-yarn-tests/src/test/scala/org/apache/flink/yarn/TestingYarnTaskManager.scala b/flink-yarn-tests/src/test/scala/org/apache/flink/yarn/TestingYarnTaskManager.scala
index 228eaaa..a194e79 100644
--- a/flink-yarn-tests/src/test/scala/org/apache/flink/yarn/TestingYarnTaskManager.scala
+++ b/flink-yarn-tests/src/test/scala/org/apache/flink/yarn/TestingYarnTaskManager.scala
@@ -72,7 +72,8 @@ class TestingYarnTaskManager(
* @param args The command line arguments.
*/
def main(args: Array[String]): Unit = {
- YarnTaskManagerRunner.runYarnTaskManager(args, classOf[TestingYarnTaskManager])
+ YarnTaskManagerRunner.runYarnTaskManager(
+ args, classOf[TestingYarnTaskManager], System.getenv(), null)
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/20146652/flink-yarn/src/main/java/org/apache/flink/yarn/YarnApplicationMasterRunner.java
----------------------------------------------------------------------
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnApplicationMasterRunner.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnApplicationMasterRunner.java
index 279981a..1c18e90 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnApplicationMasterRunner.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnApplicationMasterRunner.java
@@ -145,19 +145,9 @@ public class YarnApplicationMasterRunner {
require(currDir != null, "Current working directory variable (%s) not set", Environment.PWD.key());
LOG.debug("Current working Directory: {}", currDir);
- final String remoteKeytabPath = ENV.get(YarnConfigKeys.KEYTAB_PATH);
- LOG.debug("remoteKeytabPath obtained {}", remoteKeytabPath);
-
final String remoteKeytabPrincipal = ENV.get(YarnConfigKeys.KEYTAB_PRINCIPAL);
LOG.info("remoteKeytabPrincipal obtained {}", remoteKeytabPrincipal);
- String keytabPath = null;
- if (remoteKeytabPath != null) {
- File f = new File(currDir, Utils.KEYTAB_FILE_NAME);
- keytabPath = f.getAbsolutePath();
- LOG.debug("keytabPath: {}", keytabPath);
- }
-
UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
LOG.info("YARN daemon is running as: {} Yarn client user obtainer: {}",
@@ -170,9 +160,10 @@ public class YarnApplicationMasterRunner {
final Configuration flinkConfig = createConfiguration(currDir, dynamicProperties);
- // set keytab principal and replace path with the local path of the shipped keytab file in NodeManager
- if (keytabPath != null && remoteKeytabPrincipal != null) {
- flinkConfig.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, keytabPath);
+ File f = new File(currDir, Utils.KEYTAB_FILE_NAME);
+ if (remoteKeytabPrincipal != null && f.exists()) {
+ // set keytab principal and replace path with the local path of the shipped keytab file in NodeManager
+ flinkConfig.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, f.getAbsolutePath());
flinkConfig.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL, remoteKeytabPrincipal);
}
@@ -252,19 +243,11 @@ public class YarnApplicationMasterRunner {
LOG.info("YARN assigned hostname for application master: {}", appMasterHostname);
- //Update keytab and principal path to reflect YARN container path location
- final String remoteKeytabPath = ENV.get(YarnConfigKeys.KEYTAB_PATH);
-
final String remoteKeytabPrincipal = ENV.get(YarnConfigKeys.KEYTAB_PRINCIPAL);
- String keytabPath = null;
- if (remoteKeytabPath != null) {
- File f = new File(currDir, Utils.KEYTAB_FILE_NAME);
- keytabPath = f.getAbsolutePath();
- LOG.info("keytabPath: {}", keytabPath);
- }
- if (keytabPath != null && remoteKeytabPrincipal != null) {
- config.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, keytabPath);
+ File f = new File(currDir, Utils.KEYTAB_FILE_NAME);
+ if (remoteKeytabPrincipal != null && f.exists()) {
+ config.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, f.getAbsolutePath());
config.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL, remoteKeytabPrincipal);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/20146652/flink-yarn/src/main/java/org/apache/flink/yarn/YarnTaskManagerRunner.java
----------------------------------------------------------------------
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnTaskManagerRunner.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnTaskManagerRunner.java
index a3b964f..d1eb1ed 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnTaskManagerRunner.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnTaskManagerRunner.java
@@ -51,7 +51,29 @@ public class YarnTaskManagerRunner {
private static final Logger LOG = LoggerFactory.getLogger(YarnTaskManagerRunner.class);
- public static void runYarnTaskManager(String[] args, final Class<? extends YarnTaskManager> taskManager) throws IOException {
+ private static Callable<Object> createMainRunner(
+ Configuration configuration,
+ ResourceID resourceId,
+ final Class<? extends YarnTaskManager> taskManager) {
+ return new Callable<Object>() {
+ @Override
+ public Integer call() {
+ try {
+ TaskManager.selectNetworkInterfaceAndRunTaskManager(
+ configuration, resourceId, taskManager);
+ } catch (Throwable t) {
+ LOG.error("Error while starting the TaskManager", t);
+ System.exit(TaskManager.STARTUP_FAILURE_RETURN_CODE());
+ }
+ return null;
+ }
+ };
+ }
+
+ public static void runYarnTaskManager(String[] args,
+ final Class<? extends YarnTaskManager> taskManager,
+ Map<String, String> envs,
+ Callable<Object> mainRunner) throws IOException {
EnvironmentInformation.logEnvironmentInfo(LOG, "YARN TaskManager", args);
SignalHandler.register(LOG);
JvmShutdownSafeguard.installAsShutdownHook(LOG);
@@ -68,7 +90,6 @@ public class YarnTaskManagerRunner {
}
// read the environment variables for YARN
- final Map<String, String> envs = System.getenv();
final String yarnClientUsername = envs.get(YarnConfigKeys.ENV_HADOOP_USER_NAME);
final String localDirs = envs.get(Environment.LOCAL_DIRS.key());
LOG.info("Current working/local Directory: {}", localDirs);
@@ -76,9 +97,6 @@ public class YarnTaskManagerRunner {
final String currDir = envs.get(Environment.PWD.key());
LOG.info("Current working Directory: {}", currDir);
- final String remoteKeytabPath = envs.get(YarnConfigKeys.KEYTAB_PATH);
- LOG.info("TM: remoteKeytabPath obtained {}", remoteKeytabPath);
-
final String remoteKeytabPrincipal = envs.get(YarnConfigKeys.KEYTAB_PRINCIPAL);
LOG.info("TM: remoteKeytabPrincipal obtained {}", remoteKeytabPrincipal);
@@ -96,13 +114,6 @@ public class YarnTaskManagerRunner {
// tell akka to die in case of an error
configuration.setBoolean(AkkaOptions.JVM_EXIT_ON_FATAL_ERROR, true);
- String localKeytabPath = null;
- if (remoteKeytabPath != null) {
- File f = new File(currDir, Utils.KEYTAB_FILE_NAME);
- localKeytabPath = f.getAbsolutePath();
- LOG.info("localKeytabPath: {}", localKeytabPath);
- }
-
UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
LOG.info("YARN daemon is running as: {} Yarn client user obtainer: {}",
@@ -113,6 +124,12 @@ public class YarnTaskManagerRunner {
final ResourceID resourceId = new ResourceID(containerID);
LOG.info("ResourceID assigned for this container: {}", resourceId);
+ File f = new File(currDir, Utils.KEYTAB_FILE_NAME);
+ if (remoteKeytabPrincipal != null && f.exists()) {
+ // set keytab principal and replace path with the local path of the shipped keytab file in NodeManager
+ configuration.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, f.getAbsolutePath());
+ configuration.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL, remoteKeytabPrincipal);
+ }
try {
SecurityConfiguration sc;
@@ -126,12 +143,6 @@ public class YarnTaskManagerRunner {
hadoopConfiguration.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
hadoopConfiguration.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, "true");
- // set keytab principal and replace path with the local path of the shipped keytab file in NodeManager
- if (localKeytabPath != null && remoteKeytabPrincipal != null) {
- configuration.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, localKeytabPath);
- configuration.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL, remoteKeytabPrincipal);
- }
-
sc = new SecurityConfiguration(configuration,
Collections.singletonList(securityConfig -> new HadoopModule(securityConfig, hadoopConfiguration)));
@@ -142,19 +153,10 @@ public class YarnTaskManagerRunner {
SecurityUtils.install(sc);
- SecurityUtils.getInstalledContext().runSecured(new Callable<Object>() {
- @Override
- public Integer call() {
- try {
- TaskManager.selectNetworkInterfaceAndRunTaskManager(configuration, resourceId, taskManager);
- }
- catch (Throwable t) {
- LOG.error("Error while starting the TaskManager", t);
- System.exit(TaskManager.STARTUP_FAILURE_RETURN_CODE());
- }
- return null;
- }
- });
+ if (mainRunner == null) {
+ mainRunner = createMainRunner(configuration, resourceId, taskManager);
+ }
+ SecurityUtils.getInstalledContext().runSecured(mainRunner);
} catch (Exception e) {
LOG.error("Exception occurred while launching Task Manager", e);
throw new RuntimeException(e);
http://git-wip-us.apache.org/repos/asf/flink/blob/20146652/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnTaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnTaskManager.scala b/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnTaskManager.scala
index 615466d..5b8f4ca 100644
--- a/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnTaskManager.scala
+++ b/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnTaskManager.scala
@@ -62,7 +62,8 @@ class YarnTaskManager(
* @param args The command line arguments.
*/
def main(args: Array[String]): Unit = {
- YarnTaskManagerRunner.runYarnTaskManager(args, classOf[YarnTaskManager])
+ YarnTaskManagerRunner.runYarnTaskManager(
+ args, classOf[YarnTaskManager], System.getenv(), null)
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/20146652/flink-yarn/src/test/java/org/apache/flink/yarn/YarnTaskManagerRunnerTest.java
----------------------------------------------------------------------
diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnTaskManagerRunnerTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnTaskManagerRunnerTest.java
new file mode 100644
index 0000000..cdacfc0
--- /dev/null
+++ b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnTaskManagerRunnerTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.yarn;
+
+import org.apache.flink.runtime.security.SecurityUtils;
+import org.apache.flink.runtime.security.modules.HadoopModule;
+import org.apache.flink.runtime.security.modules.SecurityModule;
+
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.Callable;
+
+/**
+ * Tests for {@link YarnTaskManagerRunner}.
+ */
+public class YarnTaskManagerRunnerTest {
+
+ @Test
+ public void testKerberosKeytabConfiguration() throws IOException {
+ final String resourceDirPath =
+ Paths.get("src", "test", "resources").toAbsolutePath().toString();
+ final Map<String, String> envs = new HashMap<>();
+ envs.put(YarnFlinkResourceManager.ENV_FLINK_CONTAINER_ID, "test_container_00001");
+ envs.put(YarnConfigKeys.KEYTAB_PRINCIPAL, "testuser1@domain");
+ envs.put(ApplicationConstants.Environment.PWD.key(), resourceDirPath);
+ YarnTaskManagerRunner.runYarnTaskManager(
+ new String[]{"--configDir", resourceDirPath},
+ YarnTaskManager.class,
+ envs,
+ new Callable<Object>() {
+ @Override
+ public Integer call() {
+ final List<SecurityModule> modules = SecurityUtils.getInstalledModules();
+ Optional<SecurityModule> moduleOpt =
+ modules.stream().filter(s -> s instanceof HadoopModule).findFirst();
+ if (moduleOpt.isPresent()) {
+ HadoopModule hadoopModule = (HadoopModule) moduleOpt.get();
+ assertEquals("testuser1@domain", hadoopModule.getSecurityConfig().getPrincipal());
+ assertEquals(resourceDirPath + "/krb5.keytab",
+ hadoopModule.getSecurityConfig().getKeytab());
+ } else {
+ fail("Can not find HadoopModule!");
+ }
+ return null;
+ }
+ });
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/20146652/flink-yarn/src/test/resources/flink-conf.yaml
----------------------------------------------------------------------
diff --git a/flink-yarn/src/test/resources/flink-conf.yaml b/flink-yarn/src/test/resources/flink-conf.yaml
new file mode 100644
index 0000000..2c95194
--- /dev/null
+++ b/flink-yarn/src/test/resources/flink-conf.yaml
@@ -0,0 +1,23 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+#
+# This is a test configuration for validation of YarnTaskManagerRunner.
+#
+
+taskmanager.tmp.dirs: /tmp
http://git-wip-us.apache.org/repos/asf/flink/blob/20146652/flink-yarn/src/test/resources/krb5.keytab
----------------------------------------------------------------------
diff --git a/flink-yarn/src/test/resources/krb5.keytab b/flink-yarn/src/test/resources/krb5.keytab
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/flink/blob/20146652/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index f85f335..dd9faec 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1015,6 +1015,7 @@ under the License.
<exclude>flink-formats/flink-avro/src/test/resources/avro/*.avsc</exclude>
<exclude>out/test/flink-avro/avro/user.avsc</exclude>
<exclude>flink-libraries/flink-table/src/test/scala/resources/*.out</exclude>
+ <exclude>flink-yarn/src/test/resources/krb5.keytab</exclude>
<exclude>test-infra/end-to-end-test/test-data/*</exclude>
<!-- snapshots -->
[14/28] flink git commit: [FLINK-8484] [kinesis] Use an equivalence
wrapper around restored StreamShardMetadatas to bypass out-of-sync shard
infos
Posted by tz...@apache.org.
[FLINK-8484] [kinesis] Use an equivalence wrapper around restored StreamShardMetadatas to bypass out-of-sync shard infos
This closes #5337.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/7884a4f7
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/7884a4f7
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/7884a4f7
Branch: refs/heads/release-1.4
Commit: 7884a4f749b8e7ba7b831963cf2a3be12df877b7
Parents: f887575
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Thu Feb 1 12:53:42 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 17:31:32 2018 +0100
----------------------------------------------------------------------
.../kinesis/FlinkKinesisConsumer.java | 76 +++-------
.../kinesis/internals/KinesisDataFetcher.java | 5 +
.../kinesis/model/StreamShardMetadata.java | 44 ++++++
.../FlinkKinesisConsumerMigrationTest.java | 145 ++++++++++++++++++-
.../kinesis/FlinkKinesisConsumerTest.java | 52 -------
5 files changed, 209 insertions(+), 113 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/7884a4f7/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumer.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumer.java b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumer.java
index de66f93..bf5f779 100644
--- a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumer.java
+++ b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumer.java
@@ -99,7 +99,7 @@ public class FlinkKinesisConsumer<T> extends RichParallelSourceFunction<T> imple
private transient KinesisDataFetcher<T> fetcher;
/** The sequence numbers to restore to upon restore from failure. */
- private transient HashMap<StreamShardMetadata, SequenceNumber> sequenceNumsToRestore;
+ private transient HashMap<StreamShardMetadata.EquivalenceWrapper, SequenceNumber> sequenceNumsToRestore;
private volatile boolean running = true;
@@ -206,23 +206,16 @@ public class FlinkKinesisConsumer<T> extends RichParallelSourceFunction<T> imple
List<StreamShardHandle> allShards = fetcher.discoverNewShardsToSubscribe();
for (StreamShardHandle shard : allShards) {
- StreamShardMetadata kinesisStreamShard = KinesisDataFetcher.convertToStreamShardMetadata(shard);
+ StreamShardMetadata.EquivalenceWrapper kinesisStreamShard =
+ new StreamShardMetadata.EquivalenceWrapper(KinesisDataFetcher.convertToStreamShardMetadata(shard));
+
if (sequenceNumsToRestore != null) {
- // We need to do this to make sure that a shard that was closed after this restored state was taken will be properly
- // detected and have its sequence numbers restored. A shard will be closed when re-sharding, which can happen when
- // scaling up & down the Kinesis stream, and if the state is not synchronized, then the equality check of the current
- // Kinesis shard will not match the stored state, which will cause us to re-read the entire shard from the event horizon.
- if (updateKinesisShardStateWithMissingEndingSequenceNumber(kinesisStreamShard, sequenceNumsToRestore)) {
- if (LOG.isInfoEnabled()) {
- LOG.info("Updated local stored state for shard {} with a new ending number: {}", kinesisStreamShard.getShardId(), sequenceNumsToRestore.get(kinesisStreamShard));
- }
- }
if (sequenceNumsToRestore.containsKey(kinesisStreamShard)) {
// if the shard was already seen and is contained in the state,
// just use the sequence number stored in the state
fetcher.registerNewSubscribedShardState(
- new KinesisStreamShardState(kinesisStreamShard, shard, sequenceNumsToRestore.get(kinesisStreamShard)));
+ new KinesisStreamShardState(kinesisStreamShard.getShardMetadata(), shard, sequenceNumsToRestore.get(kinesisStreamShard)));
if (LOG.isInfoEnabled()) {
LOG.info("Subtask {} is seeding the fetcher with restored shard {}," +
@@ -232,7 +225,7 @@ public class FlinkKinesisConsumer<T> extends RichParallelSourceFunction<T> imple
} else {
// the shard wasn't discovered in the previous run, therefore should be consumed from the beginning
fetcher.registerNewSubscribedShardState(
- new KinesisStreamShardState(kinesisStreamShard, shard, SentinelSequenceNumber.SENTINEL_EARLIEST_SEQUENCE_NUM.get()));
+ new KinesisStreamShardState(kinesisStreamShard.getShardMetadata(), shard, SentinelSequenceNumber.SENTINEL_EARLIEST_SEQUENCE_NUM.get()));
if (LOG.isInfoEnabled()) {
LOG.info("Subtask {} is seeding the fetcher with new discovered shard {}," +
@@ -248,7 +241,7 @@ public class FlinkKinesisConsumer<T> extends RichParallelSourceFunction<T> imple
ConsumerConfigConstants.DEFAULT_STREAM_INITIAL_POSITION)).toSentinelSequenceNumber();
fetcher.registerNewSubscribedShardState(
- new KinesisStreamShardState(kinesisStreamShard, shard, startingSeqNum.get()));
+ new KinesisStreamShardState(kinesisStreamShard.getShardMetadata(), shard, startingSeqNum.get()));
if (LOG.isInfoEnabled()) {
LOG.info("Subtask {} will be seeded with initial shard {}, starting state set as sequence number {}",
@@ -275,45 +268,6 @@ public class FlinkKinesisConsumer<T> extends RichParallelSourceFunction<T> imple
sourceContext.close();
}
- /**
- * Synchronizes the Kinesis shard information from the current Kinesis shard with the restored state, if we find
- * a shard that match the shardId and streamName. If we find one, and its ending key is different that what we
- * have in our stored state, then we update the stored's shard's metadata's ending number.
- *
- * @param current the current Kinesis shard we're trying to synchronize.
- * @param sequenceNumsToRestore the (re)stored shard metadata and their sequence numbers.
- * @return {@code true} if the local state was updated with the current Kinesis shard's ending number.
- */
- @VisibleForTesting
- boolean updateKinesisShardStateWithMissingEndingSequenceNumber(StreamShardMetadata current, HashMap<StreamShardMetadata, SequenceNumber> sequenceNumsToRestore) {
- checkNotNull(current.getStreamName(), "Stream name not set on the current metadata shard");
- checkNotNull(current.getShardId(), "Shard id not set on the current metadata shard");
-
- // short-circuit: if the current shard doesn't have an ending sequence number, then there's no point in trying to update the local state
- // since that's the only property that can change.
- if (current.getEndingSequenceNumber() == null) {
- return false;
- }
-
- // try to find the matching shard based on the id & stream name
- for (Map.Entry<StreamShardMetadata, SequenceNumber> entry : sequenceNumsToRestore.entrySet()) {
- if (current.getStreamName().equals(entry.getKey().getStreamName())
- && current.getShardId().equals(entry.getKey().getShardId())) {
- // synchronize the local state if the ending sequence number is different
- if (!current.getEndingSequenceNumber().equals(entry.getKey().getEndingSequenceNumber())) {
- // ugly, but since the hashcode will change, we'll need to remove it and add it back
- sequenceNumsToRestore.remove(entry.getKey());
- entry.getKey().setEndingSequenceNumber(current.getEndingSequenceNumber());
- sequenceNumsToRestore.put(entry.getKey(), entry.getValue());
- return true;
- }
- // we already found the matching shard
- break;
- }
- }
- return false;
- }
-
@Override
public void cancel() {
running = false;
@@ -362,7 +316,13 @@ public class FlinkKinesisConsumer<T> extends RichParallelSourceFunction<T> imple
if (sequenceNumsToRestore == null) {
sequenceNumsToRestore = new HashMap<>();
for (Tuple2<StreamShardMetadata, SequenceNumber> kinesisSequenceNumber : sequenceNumsStateForCheckpoint.get()) {
- sequenceNumsToRestore.put(kinesisSequenceNumber.f0, kinesisSequenceNumber.f1);
+ sequenceNumsToRestore.put(
+ // we wrap the restored metadata inside an equivalence wrapper that checks only stream name and shard id,
+ // so that if a shard had been closed (due to a Kinesis reshard operation, for example) since
+ // the savepoint and has a different metadata than what we last stored,
+ // we will still be able to match it in sequenceNumsToRestore. Please see FLINK-8484 for details.
+ new StreamShardMetadata.EquivalenceWrapper(kinesisSequenceNumber.f0),
+ kinesisSequenceNumber.f1);
}
LOG.info("Setting restore state in the FlinkKinesisConsumer. Using the following offsets: {}",
@@ -386,16 +346,16 @@ public class FlinkKinesisConsumer<T> extends RichParallelSourceFunction<T> imple
if (fetcher == null) {
if (sequenceNumsToRestore != null) {
- for (Map.Entry<StreamShardMetadata, SequenceNumber> entry : sequenceNumsToRestore.entrySet()) {
+ for (Map.Entry<StreamShardMetadata.EquivalenceWrapper, SequenceNumber> entry : sequenceNumsToRestore.entrySet()) {
// sequenceNumsToRestore is the restored global union state;
// should only snapshot shards that actually belong to us
if (KinesisDataFetcher.isThisSubtaskShouldSubscribeTo(
- KinesisDataFetcher.convertToStreamShardHandle(entry.getKey()),
+ KinesisDataFetcher.convertToStreamShardHandle(entry.getKey().getShardMetadata()),
getRuntimeContext().getNumberOfParallelSubtasks(),
getRuntimeContext().getIndexOfThisSubtask())) {
- sequenceNumsStateForCheckpoint.add(Tuple2.of(entry.getKey(), entry.getValue()));
+ sequenceNumsStateForCheckpoint.add(Tuple2.of(entry.getKey().getShardMetadata(), entry.getValue()));
}
}
}
@@ -426,7 +386,7 @@ public class FlinkKinesisConsumer<T> extends RichParallelSourceFunction<T> imple
}
@VisibleForTesting
- HashMap<StreamShardMetadata, SequenceNumber> getRestoredState() {
+ HashMap<StreamShardMetadata.EquivalenceWrapper, SequenceNumber> getRestoredState() {
return sequenceNumsToRestore;
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/7884a4f7/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
index a8f37a5..79e4bfd 100644
--- a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
+++ b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
@@ -574,6 +574,11 @@ public class KinesisDataFetcher<T> {
});
}
+ @VisibleForTesting
+ public List<KinesisStreamShardState> getSubscribedShardsState() {
+ return subscribedShardsState;
+ }
+
/**
* Utility function to create an initial map of the last discovered shard id of each subscribed stream, set to null;
* This is called in the constructor; correct values will be set later on by calling advanceLastDiscoveredShardOfStream().
http://git-wip-us.apache.org/repos/asf/flink/blob/7884a4f7/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/model/StreamShardMetadata.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/model/StreamShardMetadata.java b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/model/StreamShardMetadata.java
index a158a8b..707822a 100644
--- a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/model/StreamShardMetadata.java
+++ b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/model/StreamShardMetadata.java
@@ -20,6 +20,8 @@ package org.apache.flink.streaming.connectors.kinesis.model;
import java.io.Serializable;
import java.util.Objects;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
/**
* A serializable representation of a AWS Kinesis Stream shard. It is basically a wrapper class around the information
* disintegrated from {@link com.amazonaws.services.kinesis.model.Shard} and its nested classes. The disintegration
@@ -170,4 +172,46 @@ public class StreamShardMetadata implements Serializable {
return hash;
}
+ /** An equivalence wrapper that only checks for the stream name and shard id for equality. */
+ public static class EquivalenceWrapper {
+
+ private final StreamShardMetadata shardMetadata;
+
+ public EquivalenceWrapper(StreamShardMetadata shardMetadata) {
+ this.shardMetadata = checkNotNull(shardMetadata);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof EquivalenceWrapper)) {
+ return false;
+ }
+
+ if (obj == this) {
+ return true;
+ }
+
+ EquivalenceWrapper other = (EquivalenceWrapper) obj;
+
+ return shardMetadata.getStreamName().equals(other.shardMetadata.getStreamName())
+ && shardMetadata.getShardId().equals(other.shardMetadata.getShardId());
+ }
+
+ @Override
+ public int hashCode() {
+ int hash = 17;
+
+ if (shardMetadata.getStreamName() != null) {
+ hash = 37 * hash + shardMetadata.getStreamName().hashCode();
+ }
+ if (shardMetadata.getShardId() != null) {
+ hash = 37 * hash + shardMetadata.getShardId().hashCode();
+ }
+ return hash;
+ }
+
+ public StreamShardMetadata getShardMetadata() {
+ return shardMetadata;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/7884a4f7/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java
index 46010d1..eddbab0 100644
--- a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java
+++ b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java
@@ -24,6 +24,8 @@ import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.flink.streaming.api.operators.StreamSource;
import org.apache.flink.streaming.connectors.kinesis.config.AWSConfigConstants;
import org.apache.flink.streaming.connectors.kinesis.internals.KinesisDataFetcher;
+import org.apache.flink.streaming.connectors.kinesis.model.KinesisStreamShardState;
+import org.apache.flink.streaming.connectors.kinesis.model.SentinelSequenceNumber;
import org.apache.flink.streaming.connectors.kinesis.model.SequenceNumber;
import org.apache.flink.streaming.connectors.kinesis.model.StreamShardHandle;
import org.apache.flink.streaming.connectors.kinesis.model.StreamShardMetadata;
@@ -38,6 +40,7 @@ import org.apache.flink.streaming.util.migration.MigrationTestUtil;
import org.apache.flink.streaming.util.migration.MigrationVersion;
import org.apache.flink.streaming.util.serialization.SimpleStringSchema;
+import com.amazonaws.services.kinesis.model.SequenceNumberRange;
import com.amazonaws.services.kinesis.model.Shard;
import org.junit.Ignore;
@@ -52,10 +55,12 @@ import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicReference;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
@@ -76,14 +81,16 @@ public class FlinkKinesisConsumerMigrationTest {
private final MigrationVersion flinkGenerateSavepointVersion = null;
private static final String TEST_STREAM_NAME = "fakeStream1";
+ private static final SequenceNumber TEST_SEQUENCE_NUMBER = new SequenceNumber("987654321");
+ private static final String TEST_SHARD_ID = KinesisShardIdGenerator.generateFromShardOrder(0);
private static final HashMap<StreamShardMetadata, SequenceNumber> TEST_STATE = new HashMap<>();
static {
StreamShardMetadata shardMetadata = new StreamShardMetadata();
shardMetadata.setStreamName(TEST_STREAM_NAME);
- shardMetadata.setShardId(KinesisShardIdGenerator.generateFromShardOrder(0));
+ shardMetadata.setShardId(TEST_SHARD_ID);
- TEST_STATE.put(shardMetadata, new SequenceNumber("987654321"));
+ TEST_STATE.put(shardMetadata, TEST_SEQUENCE_NUMBER);
}
private final MigrationVersion testMigrateVersion;
@@ -116,6 +123,10 @@ public class FlinkKinesisConsumerMigrationTest {
Shard shard = new Shard();
shard.setShardId(shardMetadata.getShardId());
+ SequenceNumberRange sequenceNumberRange = new SequenceNumberRange();
+ sequenceNumberRange.withStartingSequenceNumber("1");
+ shard.setSequenceNumberRange(sequenceNumberRange);
+
initialDiscoveryShards.add(new StreamShardHandle(shardMetadata.getStreamName(), shard));
}
@@ -147,6 +158,15 @@ public class FlinkKinesisConsumerMigrationTest {
// assert that no state was restored
assertTrue(consumerFunction.getRestoredState().isEmpty());
+ // although the restore state is empty, the fetcher should still have been registered the initial discovered shard;
+ // furthermore, the discovered shard should be considered a newly created shard while the job wasn't running,
+ // and therefore should be consumed from the earliest sequence number
+ KinesisStreamShardState restoredShardState = fetcher.getSubscribedShardsState().get(0);
+ assertEquals(TEST_STREAM_NAME, restoredShardState.getStreamShardHandle().getStreamName());
+ assertEquals(TEST_SHARD_ID, restoredShardState.getStreamShardHandle().getShard().getShardId());
+ assertFalse(restoredShardState.getStreamShardHandle().isClosed());
+ assertEquals(SentinelSequenceNumber.SENTINEL_EARLIEST_SEQUENCE_NUM.get(), restoredShardState.getLastProcessedSequenceNum());
+
consumerOperator.close();
consumerOperator.cancel();
}
@@ -158,6 +178,10 @@ public class FlinkKinesisConsumerMigrationTest {
Shard shard = new Shard();
shard.setShardId(shardMetadata.getShardId());
+ SequenceNumberRange sequenceNumberRange = new SequenceNumberRange();
+ sequenceNumberRange.withStartingSequenceNumber("1");
+ shard.setSequenceNumberRange(sequenceNumberRange);
+
initialDiscoveryShards.add(new StreamShardHandle(shardMetadata.getStreamName(), shard));
}
@@ -190,7 +214,111 @@ public class FlinkKinesisConsumerMigrationTest {
// assert that state is correctly restored
assertNotEquals(null, consumerFunction.getRestoredState());
assertEquals(1, consumerFunction.getRestoredState().size());
- assertEquals(TEST_STATE, consumerFunction.getRestoredState());
+ assertEquals(TEST_STATE, removeEquivalenceWrappers(consumerFunction.getRestoredState()));
+ assertEquals(1, fetcher.getSubscribedShardsState().size());
+ assertEquals(TEST_SEQUENCE_NUMBER, fetcher.getSubscribedShardsState().get(0).getLastProcessedSequenceNum());
+
+ KinesisStreamShardState restoredShardState = fetcher.getSubscribedShardsState().get(0);
+ assertEquals(TEST_STREAM_NAME, restoredShardState.getStreamShardHandle().getStreamName());
+ assertEquals(TEST_SHARD_ID, restoredShardState.getStreamShardHandle().getShard().getShardId());
+ assertFalse(restoredShardState.getStreamShardHandle().isClosed());
+ assertEquals(TEST_SEQUENCE_NUMBER, restoredShardState.getLastProcessedSequenceNum());
+
+ consumerOperator.close();
+ consumerOperator.cancel();
+ }
+
+ @Test
+ public void testRestoreWithReshardedStream() throws Exception {
+ final List<StreamShardHandle> initialDiscoveryShards = new ArrayList<>(TEST_STATE.size());
+ for (StreamShardMetadata shardMetadata : TEST_STATE.keySet()) {
+ // setup the closed shard
+ Shard closedShard = new Shard();
+ closedShard.setShardId(shardMetadata.getShardId());
+
+ SequenceNumberRange closedSequenceNumberRange = new SequenceNumberRange();
+ closedSequenceNumberRange.withStartingSequenceNumber("1");
+ closedSequenceNumberRange.withEndingSequenceNumber("1087654321"); // this represents a closed shard
+ closedShard.setSequenceNumberRange(closedSequenceNumberRange);
+
+ initialDiscoveryShards.add(new StreamShardHandle(shardMetadata.getStreamName(), closedShard));
+
+ // setup the new shards
+ Shard newSplitShard1 = new Shard();
+ newSplitShard1.setShardId(KinesisShardIdGenerator.generateFromShardOrder(1));
+
+ SequenceNumberRange newSequenceNumberRange1 = new SequenceNumberRange();
+ newSequenceNumberRange1.withStartingSequenceNumber("1087654322");
+ newSplitShard1.setSequenceNumberRange(newSequenceNumberRange1);
+
+ newSplitShard1.setParentShardId(TEST_SHARD_ID);
+
+ Shard newSplitShard2 = new Shard();
+ newSplitShard2.setShardId(KinesisShardIdGenerator.generateFromShardOrder(2));
+
+ SequenceNumberRange newSequenceNumberRange2 = new SequenceNumberRange();
+ newSequenceNumberRange2.withStartingSequenceNumber("2087654322");
+ newSplitShard2.setSequenceNumberRange(newSequenceNumberRange2);
+
+ newSplitShard2.setParentShardId(TEST_SHARD_ID);
+
+ initialDiscoveryShards.add(new StreamShardHandle(shardMetadata.getStreamName(), newSplitShard1));
+ initialDiscoveryShards.add(new StreamShardHandle(shardMetadata.getStreamName(), newSplitShard2));
+ }
+
+ final TestFetcher<String> fetcher = new TestFetcher<>(
+ Collections.singletonList(TEST_STREAM_NAME),
+ new TestSourceContext<>(),
+ getMockRuntimeContext(1, 0),
+ getStandardProperties(),
+ new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
+ null,
+ initialDiscoveryShards);
+
+ final DummyFlinkKinesisConsumer<String> consumerFunction = new DummyFlinkKinesisConsumer<>(
+ fetcher, new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()));
+
+ StreamSource<String, DummyFlinkKinesisConsumer<String>> consumerOperator =
+ new StreamSource<>(consumerFunction);
+
+ final AbstractStreamOperatorTestHarness<String> testHarness =
+ new AbstractStreamOperatorTestHarness<>(consumerOperator, 1, 1, 0);
+
+ testHarness.setup();
+ MigrationTestUtil.restoreFromSnapshot(
+ testHarness,
+ "src/test/resources/kinesis-consumer-migration-test-flink" + testMigrateVersion + "-snapshot", testMigrateVersion);
+ testHarness.open();
+
+ consumerFunction.run(new TestSourceContext<>());
+
+ // assert that state is correctly restored
+ assertNotEquals(null, consumerFunction.getRestoredState());
+ assertEquals(1, consumerFunction.getRestoredState().size());
+ assertEquals(TEST_STATE, removeEquivalenceWrappers(consumerFunction.getRestoredState()));
+
+ // assert that the fetcher is registered with all shards, including new shards
+ assertEquals(3, fetcher.getSubscribedShardsState().size());
+
+ KinesisStreamShardState restoredClosedShardState = fetcher.getSubscribedShardsState().get(0);
+ assertEquals(TEST_STREAM_NAME, restoredClosedShardState.getStreamShardHandle().getStreamName());
+ assertEquals(TEST_SHARD_ID, restoredClosedShardState.getStreamShardHandle().getShard().getShardId());
+ assertTrue(restoredClosedShardState.getStreamShardHandle().isClosed());
+ assertEquals(TEST_SEQUENCE_NUMBER, restoredClosedShardState.getLastProcessedSequenceNum());
+
+ KinesisStreamShardState restoredNewSplitShard1 = fetcher.getSubscribedShardsState().get(1);
+ assertEquals(TEST_STREAM_NAME, restoredNewSplitShard1.getStreamShardHandle().getStreamName());
+ assertEquals(KinesisShardIdGenerator.generateFromShardOrder(1), restoredNewSplitShard1.getStreamShardHandle().getShard().getShardId());
+ assertFalse(restoredNewSplitShard1.getStreamShardHandle().isClosed());
+ // new shards should be consumed from the beginning
+ assertEquals(SentinelSequenceNumber.SENTINEL_EARLIEST_SEQUENCE_NUM.get(), restoredNewSplitShard1.getLastProcessedSequenceNum());
+
+ KinesisStreamShardState restoredNewSplitShard2 = fetcher.getSubscribedShardsState().get(2);
+ assertEquals(TEST_STREAM_NAME, restoredNewSplitShard2.getStreamShardHandle().getStreamName());
+ assertEquals(KinesisShardIdGenerator.generateFromShardOrder(2), restoredNewSplitShard2.getStreamShardHandle().getShard().getShardId());
+ assertFalse(restoredNewSplitShard2.getStreamShardHandle().isClosed());
+ // new shards should be consumed from the beginning
+ assertEquals(SentinelSequenceNumber.SENTINEL_EARLIEST_SEQUENCE_NUM.get(), restoredNewSplitShard2.getLastProcessedSequenceNum());
consumerOperator.close();
consumerOperator.cancel();
@@ -344,4 +472,15 @@ public class FlinkKinesisConsumerMigrationTest {
return runtimeContext;
}
+
+ private static Map<StreamShardMetadata, SequenceNumber> removeEquivalenceWrappers(
+ Map<StreamShardMetadata.EquivalenceWrapper, SequenceNumber> equivalenceWrappedMap) {
+
+ Map<StreamShardMetadata, SequenceNumber> unwrapped = new HashMap<>();
+ for (Map.Entry<StreamShardMetadata.EquivalenceWrapper, SequenceNumber> wrapped : equivalenceWrappedMap.entrySet()) {
+ unwrapped.put(wrapped.getKey().getShardMetadata(), wrapped.getValue());
+ }
+
+ return unwrapped;
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/7884a4f7/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerTest.java b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerTest.java
index a142b26..11b951b 100644
--- a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerTest.java
+++ b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerTest.java
@@ -63,14 +63,12 @@ import org.powermock.modules.junit4.PowerMockRunner;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
-import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.UUID;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
@@ -597,56 +595,6 @@ public class FlinkKinesisConsumerTest {
closedStreamShardHandle, fakeRestoredState.get(closedStreamShardHandle)));
}
- @Test
- public void testUpdateKinesisShardStateWithMissingEndingSequenceNumber() {
- final String streamName = "fakeStream1";
- final String shardId = "shard-000001";
-
- Properties config = TestUtils.getStandardProperties();
- FlinkKinesisConsumer<String> consumer = new FlinkKinesisConsumer<>(streamName, new SimpleStringSchema(), config);
-
- // having the current Kinesis shard we're trying to find the restored state for
- final StreamShardMetadata current = new StreamShardMetadata();
- current.setShardId(shardId);
- current.setStreamName(streamName);
- current.setEndingSequenceNumber(null);
-
- final HashMap<StreamShardMetadata, SequenceNumber> sequenceNumsToRestore = new LinkedHashMap<>();
- assertFalse("Current shard is open, expecting a short-circuit", consumer.updateKinesisShardStateWithMissingEndingSequenceNumber(current, sequenceNumsToRestore));
-
- // create some non-matching shards
- final StreamShardMetadata differentStreamName = new StreamShardMetadata();
- differentStreamName.setStreamName("fakeStream2");
- differentStreamName.setShardId(shardId);
-
- final StreamShardMetadata differentShardId = new StreamShardMetadata();
- differentShardId.setStreamName(streamName);
- differentShardId.setShardId("shard-000002");
-
- // create the matching shard
- final StreamShardMetadata match = new StreamShardMetadata();
- match.setStreamName(streamName);
- match.setShardId(shardId);
- // ensure the sequence number isn't set (shard is considered in open state)
- match.setEndingSequenceNumber(null);
-
- sequenceNumsToRestore.put(differentStreamName, new SequenceNumber("123"));
- sequenceNumsToRestore.put(differentShardId, new SequenceNumber("456"));
- sequenceNumsToRestore.put(match, new SequenceNumber("789"));
-
- assertFalse("No ending sequence number was set, so no synchronisation was done.", consumer.updateKinesisShardStateWithMissingEndingSequenceNumber(current, sequenceNumsToRestore));
-
- // alter the ending sequence number (indicating the shard is now closed)
- final String endingSequenceNumber = "99999";
- current.setEndingSequenceNumber(endingSequenceNumber);
- assertTrue("Shard was closed (ending number set).", consumer.updateKinesisShardStateWithMissingEndingSequenceNumber(current, sequenceNumsToRestore));
- assertEquals(endingSequenceNumber, match.getEndingSequenceNumber());
- assertEquals(current, match);
- assertTrue("Make sure we can still find back our match in the restored sequences ", sequenceNumsToRestore.containsKey(match));
-
- assertFalse("Ending number was already set, no more need for synchronisation", consumer.updateKinesisShardStateWithMissingEndingSequenceNumber(current, sequenceNumsToRestore));
- }
-
private static final class TestingListState<T> implements ListState<T> {
private final List<T> list = new ArrayList<>();
[26/28] flink git commit: [FLINK-8421] [test] Increase
MAX_MEM_STATE_SIZE and Akka framesize in
AbstractEventTimeWindowCheckpointingITCase
Posted by tz...@apache.org.
[FLINK-8421] [test] Increase MAX_MEM_STATE_SIZE and Akka framesize in AbstractEventTimeWindowCheckpointingITCase
After adding the TypeSerializerConfigSnapshots of timer serializers to
the timers snapshots, the size of the timer snapshots have potentially
doubled. This caused the AbstractEventTimeWindowCheckpointingITCase to
be failing, because the configured max memory state size and Akka
framesize were too small. This commit doubles those sizes.
This closes #5362.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/fb1e24e2
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/fb1e24e2
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/fb1e24e2
Branch: refs/heads/release-1.4
Commit: fb1e24e25eddaded9668041ba4f24e35384962c3
Parents: 71d682e
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Wed Jan 31 11:13:00 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 18:30:32 2018 +0100
----------------------------------------------------------------------
.../AbstractEventTimeWindowCheckpointingITCase.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/fb1e24e2/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractEventTimeWindowCheckpointingITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractEventTimeWindowCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractEventTimeWindowCheckpointingITCase.java
index 829ac93..53d329b 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractEventTimeWindowCheckpointingITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractEventTimeWindowCheckpointingITCase.java
@@ -25,6 +25,7 @@ import org.apache.flink.api.common.state.ValueStateDescriptor;
import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple4;
+import org.apache.flink.configuration.AkkaOptions;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.HighAvailabilityOptions;
@@ -87,7 +88,7 @@ import static org.junit.Assert.fail;
@SuppressWarnings("serial")
public abstract class AbstractEventTimeWindowCheckpointingITCase extends TestLogger {
- private static final int MAX_MEM_STATE_SIZE = 10 * 1024 * 1024;
+ private static final int MAX_MEM_STATE_SIZE = 20 * 1024 * 1024;
private static final int PARALLELISM = 4;
private static LocalFlinkMiniCluster cluster;
@@ -137,6 +138,7 @@ public abstract class AbstractEventTimeWindowCheckpointingITCase extends TestLog
config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 48L);
// the default network buffers size (10% of heap max =~ 150MB) seems to much for this test case
config.setLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX, 80L << 20); // 80 MB
+ config.setString(AkkaOptions.FRAMESIZE, String.valueOf(MAX_MEM_STATE_SIZE) + "b");
if (zkServer != null) {
config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
[21/28] flink git commit: [FLINK-8472] [tests] Extend
AbstractOperatorRestoreTestBases for Flink 1.4
Posted by tz...@apache.org.
[FLINK-8472] [tests] Extend AbstractOperatorRestoreTestBases for Flink 1.4
This closes #5364.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/f68fe3ee
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/f68fe3ee
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/f68fe3ee
Branch: refs/heads/release-1.4
Commit: f68fe3ee652820ea29653c17c7515bd466389ce9
Parents: 6f4116f
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Thu Feb 1 15:27:46 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 17:31:32 2018 +0100
----------------------------------------------------------------------
.../AbstractKeyedOperatorRestoreTestBase.java | 15 +++++++--------
.../restore/keyed/KeyedComplexChainTest.java | 5 +++--
.../state/operator/restore/keyed/KeyedJob.java | 14 ++++----------
...AbstractNonKeyedOperatorRestoreTestBase.java | 19 +++++++++----------
.../restore/unkeyed/ChainBreakTest.java | 5 +++--
.../unkeyed/ChainLengthDecreaseTest.java | 5 +++--
.../unkeyed/ChainLengthIncreaseTest.java | 5 +++--
.../ChainLengthStatelessDecreaseTest.java | 5 +++--
.../restore/unkeyed/ChainOrderTest.java | 5 +++--
.../restore/unkeyed/ChainUnionTest.java | 5 +++--
.../operator/restore/unkeyed/NonKeyedJob.java | 8 ++------
.../complexKeyed-flink1.4/_metadata | Bin 0 -> 163530 bytes
.../operatorstate/nonKeyed-flink1.4/_metadata | Bin 0 -> 6248 bytes
13 files changed, 43 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/f68fe3ee/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/AbstractKeyedOperatorRestoreTestBase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/AbstractKeyedOperatorRestoreTestBase.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/AbstractKeyedOperatorRestoreTestBase.java
index 1b66c21..36301df 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/AbstractKeyedOperatorRestoreTestBase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/AbstractKeyedOperatorRestoreTestBase.java
@@ -21,6 +21,7 @@ package org.apache.flink.test.state.operator.restore.keyed;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.util.migration.MigrationVersion;
import org.apache.flink.test.state.operator.restore.AbstractOperatorRestoreTestBase;
import org.apache.flink.test.state.operator.restore.ExecutionMode;
@@ -36,17 +37,15 @@ import java.util.Collection;
@RunWith(Parameterized.class)
public abstract class AbstractKeyedOperatorRestoreTestBase extends AbstractOperatorRestoreTestBase {
- private final String savepointPath;
+ private final MigrationVersion migrationVersion;
@Parameterized.Parameters(name = "Migrate Savepoint: {0}")
- public static Collection<String> parameters () {
- return Arrays.asList(
- "complexKeyed-flink1.2",
- "complexKeyed-flink1.3");
+ public static Collection<MigrationVersion> parameters () {
+ return Arrays.asList(MigrationVersion.v1_2, MigrationVersion.v1_3, MigrationVersion.v1_4);
}
- public AbstractKeyedOperatorRestoreTestBase(String savepointPath) {
- this.savepointPath = savepointPath;
+ public AbstractKeyedOperatorRestoreTestBase(MigrationVersion migrationVersion) {
+ this.migrationVersion = migrationVersion;
}
@Override
@@ -65,6 +64,6 @@ public abstract class AbstractKeyedOperatorRestoreTestBase extends AbstractOpera
@Override
protected String getMigrationSavepointName() {
- return savepointPath;
+ return "complexKeyed-flink" + migrationVersion;
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/f68fe3ee/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/KeyedComplexChainTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/KeyedComplexChainTest.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/KeyedComplexChainTest.java
index 45dbc78..04cd956 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/KeyedComplexChainTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/KeyedComplexChainTest.java
@@ -21,6 +21,7 @@ package org.apache.flink.test.state.operator.restore.keyed;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.util.migration.MigrationVersion;
import org.apache.flink.test.state.operator.restore.ExecutionMode;
/**
@@ -28,8 +29,8 @@ import org.apache.flink.test.state.operator.restore.ExecutionMode;
*/
public class KeyedComplexChainTest extends AbstractKeyedOperatorRestoreTestBase {
- public KeyedComplexChainTest(String savepointPath) {
- super(savepointPath);
+ public KeyedComplexChainTest(MigrationVersion migrationVersion) {
+ super(migrationVersion);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/f68fe3ee/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/KeyedJob.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/KeyedJob.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/KeyedJob.java
index 076feda..1994852 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/KeyedJob.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/KeyedJob.java
@@ -98,11 +98,8 @@ public class KeyedJob {
public static SingleOutputStreamOperator<Integer> createFirstStatefulMap(ExecutionMode mode, DataStream<Integer> input) {
SingleOutputStreamOperator<Integer> map = input
.map(new StatefulStringStoringMap(mode, "first"))
- .setParallelism(4);
-
- if (mode == ExecutionMode.MIGRATE || mode == ExecutionMode.RESTORE) {
- map.uid("first");
- }
+ .setParallelism(4)
+ .uid("first");
return map;
}
@@ -110,11 +107,8 @@ public class KeyedJob {
public static SingleOutputStreamOperator<Integer> createSecondStatefulMap(ExecutionMode mode, DataStream<Integer> input) {
SingleOutputStreamOperator<Integer> map = input
.map(new StatefulStringStoringMap(mode, "second"))
- .setParallelism(4);
-
- if (mode == ExecutionMode.MIGRATE || mode == ExecutionMode.RESTORE) {
- map.uid("second");
- }
+ .setParallelism(4)
+ .uid("second");
return map;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/f68fe3ee/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/AbstractNonKeyedOperatorRestoreTestBase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/AbstractNonKeyedOperatorRestoreTestBase.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/AbstractNonKeyedOperatorRestoreTestBase.java
index c100dc9..6c9767d 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/AbstractNonKeyedOperatorRestoreTestBase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/AbstractNonKeyedOperatorRestoreTestBase.java
@@ -21,6 +21,7 @@ package org.apache.flink.test.state.operator.restore.unkeyed;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.util.migration.MigrationVersion;
import org.apache.flink.test.state.operator.restore.AbstractOperatorRestoreTestBase;
import org.apache.flink.test.state.operator.restore.ExecutionMode;
@@ -42,22 +43,20 @@ import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.c
@RunWith(Parameterized.class)
public abstract class AbstractNonKeyedOperatorRestoreTestBase extends AbstractOperatorRestoreTestBase {
- private final String savepointPath;
+ private final MigrationVersion migrationVersion;
@Parameterized.Parameters(name = "Migrate Savepoint: {0}")
- public static Collection<String> parameters () {
- return Arrays.asList(
- "nonKeyed-flink1.2",
- "nonKeyed-flink1.3");
+ public static Collection<MigrationVersion> parameters () {
+ return Arrays.asList(MigrationVersion.v1_2, MigrationVersion.v1_3, MigrationVersion.v1_4);
}
- protected AbstractNonKeyedOperatorRestoreTestBase(String savepointPath) {
- this.savepointPath = savepointPath;
+ protected AbstractNonKeyedOperatorRestoreTestBase(MigrationVersion migrationVersion) {
+ this.migrationVersion = migrationVersion;
}
- protected AbstractNonKeyedOperatorRestoreTestBase(String savepointPath, boolean allowNonRestoredState) {
+ protected AbstractNonKeyedOperatorRestoreTestBase(MigrationVersion migrationVersion, boolean allowNonRestoredState) {
super(allowNonRestoredState);
- this.savepointPath = savepointPath;
+ this.migrationVersion = migrationVersion;
}
@Override
@@ -80,6 +79,6 @@ public abstract class AbstractNonKeyedOperatorRestoreTestBase extends AbstractOp
@Override
protected String getMigrationSavepointName() {
- return savepointPath;
+ return "nonKeyed-flink" + migrationVersion;
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/f68fe3ee/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainBreakTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainBreakTest.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainBreakTest.java
index 4e074ea..b061847 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainBreakTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainBreakTest.java
@@ -21,6 +21,7 @@ package org.apache.flink.test.state.operator.restore.unkeyed;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.util.migration.MigrationVersion;
import org.apache.flink.test.state.operator.restore.ExecutionMode;
import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createFirstStatefulMap;
@@ -34,8 +35,8 @@ import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.c
*/
public class ChainBreakTest extends AbstractNonKeyedOperatorRestoreTestBase {
- public ChainBreakTest(String savepointPath) {
- super(savepointPath);
+ public ChainBreakTest(MigrationVersion migrationVersion) {
+ super(migrationVersion);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/f68fe3ee/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthDecreaseTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthDecreaseTest.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthDecreaseTest.java
index af81c0e..0190ca0 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthDecreaseTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthDecreaseTest.java
@@ -21,6 +21,7 @@ package org.apache.flink.test.state.operator.restore.unkeyed;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.util.migration.MigrationVersion;
import org.apache.flink.test.state.operator.restore.ExecutionMode;
import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createFirstStatefulMap;
@@ -33,8 +34,8 @@ import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.c
*/
public class ChainLengthDecreaseTest extends AbstractNonKeyedOperatorRestoreTestBase {
- public ChainLengthDecreaseTest(String savepointPath) {
- super(savepointPath);
+ public ChainLengthDecreaseTest(MigrationVersion migrationVersion) {
+ super(migrationVersion);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/f68fe3ee/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthIncreaseTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthIncreaseTest.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthIncreaseTest.java
index 1f07e0b..d0648ad 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthIncreaseTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthIncreaseTest.java
@@ -21,6 +21,7 @@ package org.apache.flink.test.state.operator.restore.unkeyed;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.util.migration.MigrationVersion;
import org.apache.flink.test.state.operator.restore.ExecutionMode;
import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createFirstStatefulMap;
@@ -34,8 +35,8 @@ import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.c
*/
public class ChainLengthIncreaseTest extends AbstractNonKeyedOperatorRestoreTestBase {
- public ChainLengthIncreaseTest(String savepointPath) {
- super(savepointPath);
+ public ChainLengthIncreaseTest(MigrationVersion migrationVersion) {
+ super(migrationVersion);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/f68fe3ee/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthStatelessDecreaseTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthStatelessDecreaseTest.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthStatelessDecreaseTest.java
index fb4dcf5..b0f0945 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthStatelessDecreaseTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthStatelessDecreaseTest.java
@@ -21,6 +21,7 @@ package org.apache.flink.test.state.operator.restore.unkeyed;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.util.migration.MigrationVersion;
import org.apache.flink.test.state.operator.restore.ExecutionMode;
import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createFirstStatefulMap;
@@ -36,8 +37,8 @@ import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.c
*/
public class ChainLengthStatelessDecreaseTest extends AbstractNonKeyedOperatorRestoreTestBase {
- public ChainLengthStatelessDecreaseTest(String savepointPath) {
- super(savepointPath, false);
+ public ChainLengthStatelessDecreaseTest(MigrationVersion migrationVersion) {
+ super(migrationVersion);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/f68fe3ee/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainOrderTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainOrderTest.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainOrderTest.java
index 61e366d..cb2252e 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainOrderTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainOrderTest.java
@@ -21,6 +21,7 @@ package org.apache.flink.test.state.operator.restore.unkeyed;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.util.migration.MigrationVersion;
import org.apache.flink.test.state.operator.restore.ExecutionMode;
import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createFirstStatefulMap;
@@ -34,8 +35,8 @@ import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.c
*/
public class ChainOrderTest extends AbstractNonKeyedOperatorRestoreTestBase {
- public ChainOrderTest(String savepointPath) {
- super(savepointPath);
+ public ChainOrderTest(MigrationVersion migrationVersion) {
+ super(migrationVersion);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/f68fe3ee/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainUnionTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainUnionTest.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainUnionTest.java
index 456e746..d8134d8 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainUnionTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainUnionTest.java
@@ -21,6 +21,7 @@ package org.apache.flink.test.state.operator.restore.unkeyed;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.util.migration.MigrationVersion;
import org.apache.flink.test.state.operator.restore.ExecutionMode;
import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createFirstStatefulMap;
@@ -34,8 +35,8 @@ import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.c
*/
public class ChainUnionTest extends AbstractNonKeyedOperatorRestoreTestBase {
- public ChainUnionTest(String savepointPath) {
- super(savepointPath);
+ public ChainUnionTest(MigrationVersion migrationVersion) {
+ super(migrationVersion);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/f68fe3ee/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/NonKeyedJob.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/NonKeyedJob.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/NonKeyedJob.java
index 1b0ed45..d6df74b 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/NonKeyedJob.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/NonKeyedJob.java
@@ -100,12 +100,8 @@ public class NonKeyedJob {
public static SingleOutputStreamOperator<Integer> createThirdStatefulMap(ExecutionMode mode, DataStream<Integer> input) {
SingleOutputStreamOperator<Integer> map = input
.map(new StatefulStringStoringMap(mode, "third"))
- .setParallelism(4);
-
- // we cannot set the uid on a chained operator in 1.2
- if (mode == ExecutionMode.MIGRATE || mode == ExecutionMode.RESTORE) {
- map.uid("third");
- }
+ .setParallelism(4)
+ .uid("third");
return map;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/f68fe3ee/flink-tests/src/test/resources/operatorstate/complexKeyed-flink1.4/_metadata
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/resources/operatorstate/complexKeyed-flink1.4/_metadata b/flink-tests/src/test/resources/operatorstate/complexKeyed-flink1.4/_metadata
new file mode 100644
index 0000000..8153044
Binary files /dev/null and b/flink-tests/src/test/resources/operatorstate/complexKeyed-flink1.4/_metadata differ
http://git-wip-us.apache.org/repos/asf/flink/blob/f68fe3ee/flink-tests/src/test/resources/operatorstate/nonKeyed-flink1.4/_metadata
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/resources/operatorstate/nonKeyed-flink1.4/_metadata b/flink-tests/src/test/resources/operatorstate/nonKeyed-flink1.4/_metadata
new file mode 100644
index 0000000..0b2a6a7
Binary files /dev/null and b/flink-tests/src/test/resources/operatorstate/nonKeyed-flink1.4/_metadata differ
[10/28] flink git commit: [FLINK-8398] [kinesis,
tests] Stabilize flaky KinesisDataFetcherTests
Posted by tz...@apache.org.
[FLINK-8398] [kinesis, tests] Stabilize flaky KinesisDataFetcherTests
Prior to this commit, several unit tests in KinesisDataFetcherTest
relied on sleeps to wait until a certain operation happens, in order for
the test to pass.
This commit removes those sleeps and replaces the test behaviours with
OneShotLatches.
This closes #5268.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/ca1e525c
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/ca1e525c
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/ca1e525c
Branch: refs/heads/release-1.4
Commit: ca1e525c5c2ddb2b36fcdd7fe8d2e7b053a063cf
Parents: 6922e4c
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Wed Jan 10 13:41:49 2018 +0800
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 17:31:32 2018 +0100
----------------------------------------------------------------------
.../internals/KinesisDataFetcherTest.java | 42 +++++++++++++-------
.../testutils/TestableKinesisDataFetcher.java | 16 ++++++++
2 files changed, 44 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/ca1e525c/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcherTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcherTest.java b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcherTest.java
index 56566c0..47d4653 100644
--- a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcherTest.java
+++ b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcherTest.java
@@ -18,10 +18,10 @@
package org.apache.flink.streaming.connectors.kinesis.internals;
import org.apache.flink.api.common.functions.RuntimeContext;
-import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.core.testutils.CheckedThread;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.flink.streaming.connectors.kinesis.FlinkKinesisConsumer;
+import org.apache.flink.streaming.connectors.kinesis.config.AWSConfigConstants;
import org.apache.flink.streaming.connectors.kinesis.model.KinesisStreamShardState;
import org.apache.flink.streaming.connectors.kinesis.model.SequenceNumber;
import org.apache.flink.streaming.connectors.kinesis.model.StreamShardHandle;
@@ -31,8 +31,9 @@ import org.apache.flink.streaming.connectors.kinesis.serialization.KinesisDeseri
import org.apache.flink.streaming.connectors.kinesis.testutils.FakeKinesisBehavioursFactory;
import org.apache.flink.streaming.connectors.kinesis.testutils.KinesisShardIdGenerator;
import org.apache.flink.streaming.connectors.kinesis.testutils.TestSourceContext;
-import org.apache.flink.streaming.connectors.kinesis.testutils.TestUtils;
import org.apache.flink.streaming.connectors.kinesis.testutils.TestableKinesisDataFetcher;
+import org.apache.flink.streaming.util.serialization.SimpleStringSchema;
+import org.apache.flink.util.TestLogger;
import com.amazonaws.services.kinesis.model.HashKeyRange;
import com.amazonaws.services.kinesis.model.SequenceNumberRange;
@@ -58,7 +59,7 @@ import static org.mockito.Mockito.when;
/**
* Tests for the {@link KinesisDataFetcher}.
*/
-public class KinesisDataFetcherTest {
+public class KinesisDataFetcherTest extends TestLogger {
@Test(expected = RuntimeException.class)
public void testIfNoShardsAreFoundShouldThrowException() throws Exception {
@@ -73,7 +74,7 @@ public class KinesisDataFetcherTest {
new TestableKinesisDataFetcher<>(
fakeStreams,
new TestSourceContext<>(),
- TestUtils.getStandardProperties(),
+ getStandardProperties(),
new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
10,
2,
@@ -106,7 +107,7 @@ public class KinesisDataFetcherTest {
new TestableKinesisDataFetcher<>(
fakeStreams,
new TestSourceContext<>(),
- TestUtils.getStandardProperties(),
+ getStandardProperties(),
new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
10,
2,
@@ -116,7 +117,7 @@ public class KinesisDataFetcherTest {
FakeKinesisBehavioursFactory.nonReshardedStreamsBehaviour(streamToShardCount));
final DummyFlinkKafkaConsumer<String> consumer = new DummyFlinkKafkaConsumer<>(
- TestUtils.getStandardProperties(), fetcher, 1, 0);
+ getStandardProperties(), fetcher, 1, 0);
CheckedThread consumerThread = new CheckedThread() {
@Override
@@ -191,7 +192,7 @@ public class KinesisDataFetcherTest {
new TestableKinesisDataFetcher<>(
fakeStreams,
new TestSourceContext<>(),
- TestUtils.getStandardProperties(),
+ getStandardProperties(),
new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
10,
2,
@@ -214,7 +215,8 @@ public class KinesisDataFetcherTest {
}
};
runFetcherThread.start();
- Thread.sleep(1000); // sleep a while before closing
+
+ fetcher.waitUntilInitialDiscovery();
fetcher.shutdownFetcher();
runFetcherThread.sync();
@@ -280,7 +282,7 @@ public class KinesisDataFetcherTest {
new TestableKinesisDataFetcher<>(
fakeStreams,
new TestSourceContext<>(),
- TestUtils.getStandardProperties(),
+ getStandardProperties(),
new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
10,
2,
@@ -303,7 +305,8 @@ public class KinesisDataFetcherTest {
}
};
runFetcherThread.start();
- Thread.sleep(1000); // sleep a while before closing
+
+ fetcher.waitUntilInitialDiscovery();
fetcher.shutdownFetcher();
runFetcherThread.sync();
@@ -373,7 +376,7 @@ public class KinesisDataFetcherTest {
new TestableKinesisDataFetcher<>(
fakeStreams,
new TestSourceContext<>(),
- TestUtils.getStandardProperties(),
+ getStandardProperties(),
new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
10,
2,
@@ -396,7 +399,8 @@ public class KinesisDataFetcherTest {
}
};
runFetcherThread.start();
- Thread.sleep(1000); // sleep a while before closing
+
+ fetcher.waitUntilInitialDiscovery();
fetcher.shutdownFetcher();
runFetcherThread.sync();
@@ -469,7 +473,7 @@ public class KinesisDataFetcherTest {
new TestableKinesisDataFetcher<>(
fakeStreams,
new TestSourceContext<>(),
- TestUtils.getStandardProperties(),
+ getStandardProperties(),
new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
10,
2,
@@ -492,7 +496,8 @@ public class KinesisDataFetcherTest {
}
};
runFetcherThread.start();
- Thread.sleep(1000); // sleep a while before closing
+
+ fetcher.waitUntilInitialDiscovery();
fetcher.shutdownFetcher();
runFetcherThread.sync();
@@ -587,4 +592,13 @@ public class KinesisDataFetcherTest {
return context;
}
}
+
+ private static Properties getStandardProperties() {
+ Properties config = new Properties();
+ config.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
+ config.setProperty(AWSConfigConstants.AWS_ACCESS_KEY_ID, "accessKeyId");
+ config.setProperty(AWSConfigConstants.AWS_SECRET_ACCESS_KEY, "secretKey");
+
+ return config;
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/ca1e525c/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestableKinesisDataFetcher.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestableKinesisDataFetcher.java b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestableKinesisDataFetcher.java
index 65ae6cc..83af7a0 100644
--- a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestableKinesisDataFetcher.java
+++ b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestableKinesisDataFetcher.java
@@ -22,6 +22,7 @@ import org.apache.flink.core.testutils.OneShotLatch;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.flink.streaming.connectors.kinesis.internals.KinesisDataFetcher;
import org.apache.flink.streaming.connectors.kinesis.model.KinesisStreamShardState;
+import org.apache.flink.streaming.connectors.kinesis.model.StreamShardHandle;
import org.apache.flink.streaming.connectors.kinesis.proxy.KinesisProxyInterface;
import org.apache.flink.streaming.connectors.kinesis.serialization.KinesisDeserializationSchema;
@@ -44,6 +45,8 @@ import static org.mockito.Mockito.when;
public class TestableKinesisDataFetcher<T> extends KinesisDataFetcher<T> {
private OneShotLatch runWaiter;
+ private OneShotLatch initialDiscoveryWaiter;
+
private volatile boolean running;
public TestableKinesisDataFetcher(
@@ -70,6 +73,8 @@ public class TestableKinesisDataFetcher<T> extends KinesisDataFetcher<T> {
fakeKinesis);
this.runWaiter = new OneShotLatch();
+ this.initialDiscoveryWaiter = new OneShotLatch();
+
this.running = true;
}
@@ -97,6 +102,17 @@ public class TestableKinesisDataFetcher<T> extends KinesisDataFetcher<T> {
super.awaitTermination();
}
+ @Override
+ public List<StreamShardHandle> discoverNewShardsToSubscribe() throws InterruptedException {
+ List<StreamShardHandle> newShards = super.discoverNewShardsToSubscribe();
+ initialDiscoveryWaiter.trigger();
+ return newShards;
+ }
+
+ public void waitUntilInitialDiscovery() throws InterruptedException {
+ initialDiscoveryWaiter.await();
+ }
+
private static RuntimeContext getMockedRuntimeContext(final int fakeTotalCountOfSubtasks, final int fakeIndexOfThisSubtask) {
RuntimeContext mockedRuntimeContext = mock(RuntimeContext.class);