You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by sr...@apache.org on 2018/02/09 17:23:51 UTC

[1/2] flink git commit: [FLINK-8571] [DataStream] Introduce utility function that reinterprets a data stream as keyed stream

Repository: flink
Updated Branches:
  refs/heads/master dea417260 -> 91eea376e


[FLINK-8571] [DataStream] Introduce utility function that reinterprets a data stream as keyed stream

This closes #5424.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/91eea376
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/91eea376
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/91eea376

Branch: refs/heads/master
Commit: 91eea376eee1ca2612b0f0603511b5eb52f11195
Parents: bfe6f84
Author: Stefan Richter <s....@data-artisans.com>
Authored: Fri Feb 2 18:39:51 2018 +0100
Committer: Stefan Richter <s....@data-artisans.com>
Committed: Fri Feb 9 18:23:37 2018 +0100

----------------------------------------------------------------------
 docs/dev/api_concepts.md                        |   1 -
 docs/dev/stream/experimental.md                 |  79 +++++++
 .../api/datastream/DataStreamUtils.java         |  62 ++++-
 .../streaming/api/datastream/KeyedStream.java   |  33 ++-
 ...einterpretDataStreamAsKeyedStreamITCase.java | 231 +++++++++++++++++++
 .../streaming/api/scala/DataStreamUtils.scala   |  25 ++
 .../streaming/api/scala/DataStreamTest.scala    |   4 +-
 ...interpretDataStreamAsKeyedStreamITCase.scala |  40 ++++
 8 files changed, 468 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/91eea376/docs/dev/api_concepts.md
----------------------------------------------------------------------
diff --git a/docs/dev/api_concepts.md b/docs/dev/api_concepts.md
index c675225..b336fce 100644
--- a/docs/dev/api_concepts.md
+++ b/docs/dev/api_concepts.md
@@ -895,4 +895,3 @@ result type ```R``` for the final result. E.g. for a histogram, ```V``` is a num
  a histogram. ```SimpleAccumulator``` is for the cases where both types are the same, e.g. for counters.
 
 {% top %}
-

http://git-wip-us.apache.org/repos/asf/flink/blob/91eea376/docs/dev/stream/experimental.md
----------------------------------------------------------------------
diff --git a/docs/dev/stream/experimental.md b/docs/dev/stream/experimental.md
new file mode 100644
index 0000000..db029be
--- /dev/null
+++ b/docs/dev/stream/experimental.md
@@ -0,0 +1,79 @@
+---
+title: "Experimental Features"
+nav-id: experimental_features
+nav-show_overview: true
+nav-parent_id: streaming
+nav-pos: 100
+---
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+This section describes experimental features in the DataStream API. Experimental features are still evolving and can be either unstable,
+incomplete, or subject to heavy change in future versions.
+
+Reinterpreting a pre-partitioned data stream as keyed stream
+------------------------------------------------------------
+
+We can re-interpret a pre-partitioned data stream as a keyed stream to avoid shuffling.
+
+**WARNING**: The re-interpreted data stream **MUST** already be pre-partitioned in **EXACTLY** the same way Flink's keyBy would partition
+the data in a shuffle w.r.t. key-group assignment.
+
+One use-case for this could be a materialized shuffle between two jobs: the first job performs a keyBy shuffle and materializes
+each output into a partition. A second job has sources that, for each parallel instance, reads from the corresponding partitions
+created by the first job. Those sources can now be re-interpreted as keyed streams, e.g. to apply windowing. Notice that this trick
+makes the second job embarrassingly parallel, which can be helpful for a fine-grained recovery scheme.
+
+This re-interpretation functionality is exposed through `DataStreamUtils`:
+
+{% highlight java %}
+	static <T, K> KeyedStream<T, K> reinterpretAsKeyedStream(
+		DataStream<T> stream,
+		KeySelector<T, K> keySelector,
+		TypeInformation<K> typeInfo)
+{% endhighlight %}
+
+Given a base stream, a key selector, and type information,
+the method creates a keyed stream from the base stream.
+
+Code example:
+
+<div class="codetabs" markdown="1">
+<div data-lang="java" markdown="1">
+{% highlight java %}
+        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+        DataStreamSource<Integer> source = ...
+        DataStreamUtils.reinterpretAsKeyedStream(source, (in) -> in, TypeInformation.of(Integer.class))
+            .timeWindow(Time.seconds(1))
+            .reduce((a, b) -> a + b)
+            .addSink(new DiscardingSink<>());
+        env.execute();
+{% endhighlight %}
+</div>
+<div data-lang="scala" markdown="1">
+{% highlight scala %}
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setParallelism(1)
+    val source = ...
+    new DataStreamUtils(source).reinterpretAsKeyedStream((in) => in)
+      .timeWindow(Time.seconds(1))
+      .reduce((a, b) => a + b)
+      .addSink(new DiscardingSink[Int])
+    env.execute()
+{% endhighlight %}

http://git-wip-us.apache.org/repos/asf/flink/blob/91eea376/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamUtils.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamUtils.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamUtils.java
index d145d6f..351c456 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamUtils.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamUtils.java
@@ -18,13 +18,18 @@
 package org.apache.flink.streaming.api.datastream;
 
 import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
 import org.apache.flink.runtime.net.ConnectionUtils;
 import org.apache.flink.streaming.api.environment.LocalStreamEnvironment;
 import org.apache.flink.streaming.api.environment.RemoteStreamEnvironment;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.transformations.PartitionTransformation;
 import org.apache.flink.streaming.experimental.CollectSink;
 import org.apache.flink.streaming.experimental.SocketStreamIterator;
+import org.apache.flink.streaming.runtime.partitioner.ForwardPartitioner;
 
 import java.io.IOException;
 import java.net.InetAddress;
@@ -33,7 +38,7 @@ import java.net.UnknownHostException;
 import java.util.Iterator;
 
 /**
- * A collection of experimental utilities for {@link DataStream DataStreams}.
+ * A collection of utilities for {@link DataStream DataStreams}.
  *
  * <p>This experimental class is relocated from flink-streaming-contrib. Please see package-info.java
  * for more information.
@@ -85,6 +90,61 @@ public final class DataStreamUtils {
 		return iter;
 	}
 
+	/**
+	 * Reinterprets the given {@link DataStream} as a {@link KeyedStream}, which extracts keys with the given
+	 * {@link KeySelector}.
+	 *
+	 * <p>IMPORTANT: For every partition of the base stream, the keys of events in the base stream must be
+	 * partitioned exactly in the same way as if it was created through a {@link DataStream#keyBy(KeySelector)}.
+	 *
+	 * @param stream      The data stream to reinterpret. For every partition, this stream must be partitioned exactly
+	 *                    in the same way as if it was created through a {@link DataStream#keyBy(KeySelector)}.
+	 * @param keySelector Function that defines how keys are extracted from the data stream.
+	 * @param <T>         Type of events in the data stream.
+	 * @param <K>         Type of the extracted keys.
+	 * @return The reinterpretation of the {@link DataStream} as a {@link KeyedStream}.
+	 */
+	public static <T, K> KeyedStream<T, K> reinterpretAsKeyedStream(
+		DataStream<T> stream,
+		KeySelector<T, K> keySelector) {
+
+		return reinterpretAsKeyedStream(
+			stream,
+			keySelector,
+			TypeExtractor.getKeySelectorTypes(keySelector, stream.getType()));
+	}
+
+	/**
+	 * Reinterprets the given {@link DataStream} as a {@link KeyedStream}, which extracts keys with the given
+	 * {@link KeySelector}.
+	 *
+	 * <p>IMPORTANT: For every partition of the base stream, the keys of events in the base stream must be
+	 * partitioned exactly in the same way as if it was created through a {@link DataStream#keyBy(KeySelector)}.
+	 *
+	 * @param stream      The data stream to reinterpret. For every partition, this stream must be partitioned exactly
+	 *                    in the same way as if it was created through a {@link DataStream#keyBy(KeySelector)}.
+	 * @param keySelector Function that defines how keys are extracted from the data stream.
+	 * @param typeInfo    Explicit type information about the key type.
+	 * @param <T>         Type of events in the data stream.
+	 * @param <K>         Type of the extracted keys.
+	 * @return The reinterpretation of the {@link DataStream} as a {@link KeyedStream}.
+	 */
+	public static <T, K> KeyedStream<T, K> reinterpretAsKeyedStream(
+		DataStream<T> stream,
+		KeySelector<T, K> keySelector,
+		TypeInformation<K> typeInfo) {
+
+		PartitionTransformation<T> partitionTransformation = new PartitionTransformation<>(
+			stream.getTransformation(),
+			new ForwardPartitioner<>());
+
+		return new KeyedStream<>(
+			stream,
+			partitionTransformation,
+			keySelector,
+			typeInfo);
+	}
+
 	private static class CallExecute extends Thread {
 
 		private final StreamExecutionEnvironment toTrigger;

http://git-wip-us.apache.org/repos/asf/flink/blob/91eea376/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/KeyedStream.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/KeyedStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/KeyedStream.java
index ebcd7d5..7beaa03 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/KeyedStream.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/KeyedStream.java
@@ -120,12 +120,37 @@ public class KeyedStream<T, KEY> extends DataStream<T> {
 	 *            Function for determining state partitions
 	 */
 	public KeyedStream(DataStream<T> dataStream, KeySelector<T, KEY> keySelector, TypeInformation<KEY> keyType) {
-		super(
-			dataStream.getExecutionEnvironment(),
+		this(
+			dataStream,
 			new PartitionTransformation<>(
 				dataStream.getTransformation(),
-				new KeyGroupStreamPartitioner<>(keySelector, StreamGraphGenerator.DEFAULT_LOWER_BOUND_MAX_PARALLELISM)));
-		this.keySelector = keySelector;
+				new KeyGroupStreamPartitioner<>(keySelector, StreamGraphGenerator.DEFAULT_LOWER_BOUND_MAX_PARALLELISM)),
+			keySelector,
+			keyType);
+	}
+
+	/**
+	 * Creates a new {@link KeyedStream} using the given {@link KeySelector} and {@link TypeInformation}
+	 * to partition operator state by key, where the partitioning is defined by a {@link PartitionTransformation}.
+	 *
+	 * @param stream
+	 *            Base stream of data
+	 * @param partitionTransformation
+	 *            Function that determines how the keys are distributed to downstream operator(s)
+	 * @param keySelector
+	 *            Function to extract keys from the base stream
+	 * @param keyType
+	 *            Defines the type of the extracted keys
+	 */
+	@Internal
+	KeyedStream(
+		DataStream<T> stream,
+		PartitionTransformation<T> partitionTransformation,
+		KeySelector<T, KEY> keySelector,
+		TypeInformation<KEY> keyType) {
+
+		super(stream.getExecutionEnvironment(), partitionTransformation);
+		this.keySelector = clean(keySelector);
 		this.keyType = validateKeyType(keyType);
 	}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/91eea376/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/datastream/ReinterpretDataStreamAsKeyedStreamITCase.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/datastream/ReinterpretDataStreamAsKeyedStreamITCase.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/datastream/ReinterpretDataStreamAsKeyedStreamITCase.java
new file mode 100644
index 0000000..fc8e997
--- /dev/null
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/datastream/ReinterpretDataStreamAsKeyedStreamITCase.java
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.datastream;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.TimeCharacteristic;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
+import org.apache.flink.streaming.api.functions.source.ParallelSourceFunction;
+import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
+import org.apache.flink.streaming.api.windowing.time.Time;
+import org.apache.flink.util.Preconditions;
+
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+/**
+ * Integration test for {@link DataStreamUtils#reinterpretAsKeyedStream(DataStream, KeySelector, TypeInformation)}.
+ */
+public class ReinterpretDataStreamAsKeyedStreamITCase {
+
+	@Rule
+	public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+	/**
+	 * This test checks that reinterpreting a data stream to a keyed stream works as expected. This test consists of
+	 * two jobs. The first job materializes a keyBy into files, one files per partition. The second job opens the
+	 * files created by the first jobs as sources (doing the correct assignment of files to partitions) and
+	 * reinterprets the sources as keyed, because we know they have been partitioned in a keyBy from the first job.
+	 */
+	@Test
+	public void testReinterpretAsKeyedStream() throws Exception {
+
+		final int numEventsPerInstance = 100;
+		final int maxParallelism = 8;
+		final int parallelism = 3;
+		final int numUniqueKeys = 12;
+
+		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+		env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
+		env.setMaxParallelism(maxParallelism);
+		env.setParallelism(parallelism);
+
+		final List<File> partitionFiles = new ArrayList<>(parallelism);
+		for (int i = 0; i < parallelism; ++i) {
+			File partitionFile = temporaryFolder.newFile();
+			partitionFiles.add(i, partitionFile);
+		}
+
+		env.addSource(new RandomTupleSource(numEventsPerInstance, numUniqueKeys))
+			.keyBy(0)
+			.addSink(new ToPartitionFileSink(partitionFiles));
+
+		env.execute();
+
+		DataStreamUtils.reinterpretAsKeyedStream(
+			env.addSource(new FromPartitionFileSource(partitionFiles)),
+			(KeySelector<Tuple2<Integer, Integer>, Integer>) value -> value.f0,
+			TypeInformation.of(Integer.class))
+			.timeWindow(Time.seconds(1)) // test that also timers and aggregated state work as expected
+			.reduce((ReduceFunction<Tuple2<Integer, Integer>>) (value1, value2) ->
+				new Tuple2<>(value1.f0, value1.f1 + value2.f1))
+			.addSink(new ValidatingSink(numEventsPerInstance * parallelism)).setParallelism(1);
+
+		env.execute();
+	}
+
+	private static class RandomTupleSource implements ParallelSourceFunction<Tuple2<Integer, Integer>> {
+		private static final long serialVersionUID = 1L;
+
+		private int numKeys;
+		private int remainingEvents;
+
+		public RandomTupleSource(int numEvents, int numKeys) {
+			this.numKeys = numKeys;
+			this.remainingEvents = numEvents;
+		}
+
+		@Override
+		public void run(SourceContext<Tuple2<Integer, Integer>> out) throws Exception {
+			Random random = new Random(42);
+			while (--remainingEvents >= 0) {
+				out.collect(new Tuple2<>(random.nextInt(numKeys), 1));
+			}
+		}
+
+		@Override
+		public void cancel() {
+			this.remainingEvents = 0;
+		}
+	}
+
+	private static class ToPartitionFileSink extends RichSinkFunction<Tuple2<Integer, Integer>> {
+
+		private static final long serialVersionUID = 1L;
+
+		private final List<File> allPartitions;
+		private DataOutputStream dos;
+
+		public ToPartitionFileSink(List<File> allPartitions) {
+			this.allPartitions = allPartitions;
+		}
+
+		@Override
+		public void open(Configuration parameters) throws Exception {
+			super.open(parameters);
+			int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask();
+			dos = new DataOutputStream(
+				new BufferedOutputStream(
+					new FileOutputStream(allPartitions.get(subtaskIdx))));
+		}
+
+		@Override
+		public void close() throws Exception {
+			super.close();
+			dos.close();
+		}
+
+		@Override
+		public void invoke(Tuple2<Integer, Integer> value, Context context) throws Exception {
+			dos.writeInt(value.f0);
+			dos.writeInt(value.f1);
+		}
+	}
+
+	private static class FromPartitionFileSource extends RichParallelSourceFunction<Tuple2<Integer, Integer>> {
+		private static final long serialVersionUID = 1L;
+
+		private List<File> allPartitions;
+		private DataInputStream din;
+		private volatile boolean running;
+
+		public FromPartitionFileSource(List<File> allPartitons) {
+			this.allPartitions = allPartitons;
+		}
+
+		@Override
+		public void open(Configuration parameters) throws Exception {
+			super.open(parameters);
+			int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask();
+			din = new DataInputStream(
+				new BufferedInputStream(
+					new FileInputStream(allPartitions.get(subtaskIdx))));
+		}
+
+		@Override
+		public void close() throws Exception {
+			super.close();
+			din.close();
+		}
+
+		@Override
+		public void run(SourceContext<Tuple2<Integer, Integer>> out) throws Exception {
+			this.running = true;
+			try {
+				while (running) {
+					Integer key = din.readInt();
+					Integer val = din.readInt();
+					out.collect(new Tuple2<>(key, val));
+				}
+			} catch (EOFException ignore) {
+			}
+		}
+
+		@Override
+		public void cancel() {
+			this.running = false;
+		}
+	}
+
+	private static class ValidatingSink extends RichSinkFunction<Tuple2<Integer, Integer>> {
+
+		private static final long serialVersionUID = 1L;
+		private final int expectedSum;
+		private int runningSum = 0;
+
+		private ValidatingSink(int expectedSum) {
+			this.expectedSum = expectedSum;
+		}
+
+		@Override
+		public void open(Configuration parameters) throws Exception {
+			super.open(parameters);
+			Preconditions.checkState(getRuntimeContext().getNumberOfParallelSubtasks() == 1);
+		}
+
+		@Override
+		public void invoke(Tuple2<Integer, Integer> value, Context context) throws Exception {
+			runningSum += value.f1;
+		}
+
+		@Override
+		public void close() throws Exception {
+			Assert.assertEquals(expectedSum, runningSum);
+			super.close();
+		}
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/91eea376/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStreamUtils.scala
----------------------------------------------------------------------
diff --git a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStreamUtils.scala b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStreamUtils.scala
index 74dd66a..0f42aa2 100644
--- a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStreamUtils.scala
+++ b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStreamUtils.scala
@@ -43,5 +43,30 @@ class DataStreamUtils[T: TypeInformation : ClassTag](val self: DataStream[T]) {
   def collect() : Iterator[T] = {
     JavaStreamUtils.collect(self.javaStream).asScala
   }
+
+  /**
+    * Reinterprets the given [[DataStream]] as a [[KeyedStream]], which extracts keys with the
+    * given [[KeySelectorWithType]].
+    *
+    * IMPORTANT: For every partition of the base stream, the keys of events in the base stream
+    * must be partitioned exactly in the same way as if it was created through a
+    * [[DataStream#keyBy(KeySelectorWithType)]].
+    *
+    * @param keySelector Function that defines how keys are extracted from the data stream.
+    * @return The reinterpretation of the [[DataStream]] as a [[KeyedStream]].
+    */
+  def reinterpretAsKeyedStream[K: TypeInformation](
+        keySelector: T => K): KeyedStream[T, K] = {
+
+    val keySelectorWithType =
+      new KeySelectorWithType[T, K](clean(keySelector), implicitly[TypeInformation[K]])
+
+    asScalaStream(
+      JavaStreamUtils.reinterpretAsKeyedStream(self.javaStream, keySelectorWithType))
+  }
+
+  private[flink] def clean[F <: AnyRef](f: F): F = {
+    new StreamExecutionEnvironment(self.javaStream.getExecutionEnvironment).scalaClean(f)
+  }
 }
 

http://git-wip-us.apache.org/repos/asf/flink/blob/91eea376/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/DataStreamTest.scala
----------------------------------------------------------------------
diff --git a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/DataStreamTest.scala b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/DataStreamTest.scala
index 805eb41..e2c5b41 100644
--- a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/DataStreamTest.scala
+++ b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/DataStreamTest.scala
@@ -25,17 +25,19 @@ import org.apache.flink.api.java.typeutils.TypeExtractor
 import org.apache.flink.streaming.api.collector.selector.OutputSelector
 import org.apache.flink.streaming.api.functions.ProcessFunction
 import org.apache.flink.streaming.api.functions.co.CoMapFunction
+import org.apache.flink.streaming.api.functions.sink.DiscardingSink
 import org.apache.flink.streaming.api.graph.{StreamEdge, StreamGraph}
 import org.apache.flink.streaming.api.operators.{AbstractUdfStreamOperator, KeyedProcessOperator, ProcessOperator, StreamOperator}
 import org.apache.flink.streaming.api.windowing.assigners.GlobalWindows
+import org.apache.flink.streaming.api.windowing.time.Time
 import org.apache.flink.streaming.api.windowing.triggers.{CountTrigger, PurgingTrigger}
 import org.apache.flink.streaming.api.windowing.windows.GlobalWindow
 import org.apache.flink.streaming.runtime.partitioner._
 import org.apache.flink.test.util.AbstractTestBase
 import org.apache.flink.util.Collector
 import org.junit.Assert._
-import org.junit.{Rule, Test}
 import org.junit.rules.ExpectedException
+import org.junit.{Rule, Test}
 
 class DataStreamTest extends AbstractTestBase {
 

http://git-wip-us.apache.org/repos/asf/flink/blob/91eea376/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/ReinterpretDataStreamAsKeyedStreamITCase.scala
----------------------------------------------------------------------
diff --git a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/ReinterpretDataStreamAsKeyedStreamITCase.scala b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/ReinterpretDataStreamAsKeyedStreamITCase.scala
new file mode 100644
index 0000000..0957424
--- /dev/null
+++ b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/ReinterpretDataStreamAsKeyedStreamITCase.scala
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.streaming.api.scala
+
+import org.apache.flink.streaming.api.functions.sink.DiscardingSink
+import org.apache.flink.streaming.api.windowing.time.Time
+import org.junit.Test
+
+/**
+  * Integration test for [[DataStreamUtils.reinterpretAsKeyedStream()]].
+  */
+class ReinterpretDataStreamAsKeyedStreamITCase {
+
+  @Test
+  def testReinterpretAsKeyedStream(): Unit = {
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setParallelism(1)
+    val source = env.fromElements(1, 2, 3)
+    new DataStreamUtils(source).reinterpretAsKeyedStream((in) => in)
+      .timeWindow(Time.seconds(1))
+      .reduce((a, b) => a + b)
+      .addSink(new DiscardingSink[Int])
+    env.execute()
+  }
+}


[2/2] flink git commit: [hotfix] Move DataStreamUtils to the datastream API package so that we can actually use it to expose package-private constructors or methods for experimental features.

Posted by sr...@apache.org.
[hotfix] Move DataStreamUtils to the datastream API package so that we can actually use it to expose package-private constructors or methods for experimental features.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/bfe6f84c
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/bfe6f84c
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/bfe6f84c

Branch: refs/heads/master
Commit: bfe6f84cf1b99f1dfa801a1818fa51ccc7817c9b
Parents: dea4172
Author: Stefan Richter <s....@data-artisans.com>
Authored: Wed Feb 7 11:04:14 2018 +0100
Committer: Stefan Richter <s....@data-artisans.com>
Committed: Fri Feb 9 18:23:37 2018 +0100

----------------------------------------------------------------------
 .../api/datastream/DataStreamUtils.java         | 115 +++++++++++++++++++
 .../streaming/experimental/CollectSink.java     |   2 +-
 .../streaming/experimental/DataStreamUtils.java | 115 -------------------
 .../experimental/SocketStreamIterator.java      |   6 +-
 .../streaming/api/scala/DataStreamUtils.scala   |  47 ++++++++
 .../experimental/scala/DataStreamUtils.scala    |  48 --------
 .../streaming/experimental/CollectITCase.java   |   2 +-
 7 files changed, 168 insertions(+), 167 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/bfe6f84c/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamUtils.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamUtils.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamUtils.java
new file mode 100644
index 0000000..d145d6f
--- /dev/null
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamUtils.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.datastream;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.runtime.net.ConnectionUtils;
+import org.apache.flink.streaming.api.environment.LocalStreamEnvironment;
+import org.apache.flink.streaming.api.environment.RemoteStreamEnvironment;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.experimental.CollectSink;
+import org.apache.flink.streaming.experimental.SocketStreamIterator;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+import java.util.Iterator;
+
+/**
+ * A collection of experimental utilities for {@link DataStream DataStreams}.
+ *
+ * <p>This experimental class is relocated from flink-streaming-contrib. Please see package-info.java
+ * for more information.
+ */
+@PublicEvolving
+public final class DataStreamUtils {
+
+	/**
+	 * Returns an iterator to iterate over the elements of the DataStream.
+	 * @return The iterator
+	 */
+	public static <OUT> Iterator<OUT> collect(DataStream<OUT> stream) throws IOException {
+
+		TypeSerializer<OUT> serializer = stream.getType().createSerializer(
+				stream.getExecutionEnvironment().getConfig());
+
+		SocketStreamIterator<OUT> iter = new SocketStreamIterator<OUT>(serializer);
+
+		//Find out what IP of us should be given to CollectSink, that it will be able to connect to
+		StreamExecutionEnvironment env = stream.getExecutionEnvironment();
+		InetAddress clientAddress;
+
+		if (env instanceof RemoteStreamEnvironment) {
+			String host = ((RemoteStreamEnvironment) env).getHost();
+			int port = ((RemoteStreamEnvironment) env).getPort();
+			try {
+				clientAddress = ConnectionUtils.findConnectingAddress(new InetSocketAddress(host, port), 2000, 400);
+			}
+			catch (Exception e) {
+				throw new IOException("Could not determine an suitable network address to " +
+						"receive back data from the streaming program.", e);
+			}
+		} else if (env instanceof LocalStreamEnvironment) {
+			clientAddress = InetAddress.getLoopbackAddress();
+		} else {
+			try {
+				clientAddress = InetAddress.getLocalHost();
+			} catch (UnknownHostException e) {
+				throw new IOException("Could not determine this machines own local address to " +
+						"receive back data from the streaming program.", e);
+			}
+		}
+
+		DataStreamSink<OUT> sink = stream.addSink(new CollectSink<OUT>(clientAddress, iter.getPort(), serializer));
+		sink.setParallelism(1); // It would not work if multiple instances would connect to the same port
+
+		(new CallExecute(env, iter)).start();
+
+		return iter;
+	}
+
+	private static class CallExecute extends Thread {
+
+		private final StreamExecutionEnvironment toTrigger;
+		private final SocketStreamIterator<?> toNotify;
+
+		private CallExecute(StreamExecutionEnvironment toTrigger, SocketStreamIterator<?> toNotify) {
+			this.toTrigger = toTrigger;
+			this.toNotify = toNotify;
+		}
+
+		@Override
+		public void run(){
+			try {
+				toTrigger.execute();
+			}
+			catch (Throwable t) {
+				toNotify.notifyOfError(t);
+			}
+		}
+	}
+
+	// ------------------------------------------------------------------------
+
+	/**
+	 * Private constructor to prevent instantiation.
+	 */
+	private DataStreamUtils() {}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/bfe6f84c/flink-streaming-java/src/main/java/org/apache/flink/streaming/experimental/CollectSink.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/experimental/CollectSink.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/experimental/CollectSink.java
index 23b5280..aa0b53b 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/experimental/CollectSink.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/experimental/CollectSink.java
@@ -35,7 +35,7 @@ import java.net.Socket;
  * for more information.
  */
 @Internal
-class CollectSink<IN> extends RichSinkFunction<IN> {
+public class CollectSink<IN> extends RichSinkFunction<IN> {
 
 	private static final long serialVersionUID = 1L;
 

http://git-wip-us.apache.org/repos/asf/flink/blob/bfe6f84c/flink-streaming-java/src/main/java/org/apache/flink/streaming/experimental/DataStreamUtils.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/experimental/DataStreamUtils.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/experimental/DataStreamUtils.java
deleted file mode 100644
index 59ad6a8..0000000
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/experimental/DataStreamUtils.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.experimental;
-
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.runtime.net.ConnectionUtils;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.datastream.DataStreamSink;
-import org.apache.flink.streaming.api.environment.LocalStreamEnvironment;
-import org.apache.flink.streaming.api.environment.RemoteStreamEnvironment;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.UnknownHostException;
-import java.util.Iterator;
-
-/**
- * A collection of utilities for {@link DataStream DataStreams}.
- *
- * <p>This experimental class is relocated from flink-streaming-contrib. Please see package-info.java
- * for more information.
- */
-@PublicEvolving
-public final class DataStreamUtils {
-
-	/**
-	 * Returns an iterator to iterate over the elements of the DataStream.
-	 * @return The iterator
-	 */
-	public static <OUT> Iterator<OUT> collect(DataStream<OUT> stream) throws IOException {
-
-		TypeSerializer<OUT> serializer = stream.getType().createSerializer(
-				stream.getExecutionEnvironment().getConfig());
-
-		SocketStreamIterator<OUT> iter = new SocketStreamIterator<OUT>(serializer);
-
-		//Find out what IP of us should be given to CollectSink, that it will be able to connect to
-		StreamExecutionEnvironment env = stream.getExecutionEnvironment();
-		InetAddress clientAddress;
-
-		if (env instanceof RemoteStreamEnvironment) {
-			String host = ((RemoteStreamEnvironment) env).getHost();
-			int port = ((RemoteStreamEnvironment) env).getPort();
-			try {
-				clientAddress = ConnectionUtils.findConnectingAddress(new InetSocketAddress(host, port), 2000, 400);
-			}
-			catch (Exception e) {
-				throw new IOException("Could not determine an suitable network address to " +
-						"receive back data from the streaming program.", e);
-			}
-		} else if (env instanceof LocalStreamEnvironment) {
-			clientAddress = InetAddress.getLoopbackAddress();
-		} else {
-			try {
-				clientAddress = InetAddress.getLocalHost();
-			} catch (UnknownHostException e) {
-				throw new IOException("Could not determine this machines own local address to " +
-						"receive back data from the streaming program.", e);
-			}
-		}
-
-		DataStreamSink<OUT> sink = stream.addSink(new CollectSink<OUT>(clientAddress, iter.getPort(), serializer));
-		sink.setParallelism(1); // It would not work if multiple instances would connect to the same port
-
-		(new CallExecute(env, iter)).start();
-
-		return iter;
-	}
-
-	private static class CallExecute extends Thread {
-
-		private final StreamExecutionEnvironment toTrigger;
-		private final SocketStreamIterator<?> toNotify;
-
-		private CallExecute(StreamExecutionEnvironment toTrigger, SocketStreamIterator<?> toNotify) {
-			this.toTrigger = toTrigger;
-			this.toNotify = toNotify;
-		}
-
-		@Override
-		public void run(){
-			try {
-				toTrigger.execute();
-			}
-			catch (Throwable t) {
-				toNotify.notifyOfError(t);
-			}
-		}
-	}
-
-	// ------------------------------------------------------------------------
-
-	/**
-	 * Private constructor to prevent instantiation.
-	 */
-	private DataStreamUtils() {}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/bfe6f84c/flink-streaming-java/src/main/java/org/apache/flink/streaming/experimental/SocketStreamIterator.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/experimental/SocketStreamIterator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/experimental/SocketStreamIterator.java
index 871c0f7..ccb54ed 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/experimental/SocketStreamIterator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/experimental/SocketStreamIterator.java
@@ -17,6 +17,7 @@
 
 package org.apache.flink.streaming.experimental;
 
+import org.apache.flink.annotation.Internal;
 import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.core.memory.DataInputViewStreamWrapper;
@@ -42,7 +43,7 @@ import java.util.NoSuchElementException;
  * @param <T> The type of elements returned from the iterator.
  */
 @PublicEvolving
-class SocketStreamIterator<T> implements Iterator<T> {
+public class SocketStreamIterator<T> implements Iterator<T> {
 
 	/** Server socket to listen at. */
 	private final ServerSocket socket;
@@ -62,7 +63,8 @@ class SocketStreamIterator<T> implements Iterator<T> {
 	/** Async error, for example by the executor of the program that produces the stream. */
 	private volatile Throwable error;
 
-	SocketStreamIterator(TypeSerializer<T> serializer) throws IOException {
+	@Internal
+	public SocketStreamIterator(TypeSerializer<T> serializer) throws IOException {
 		this.serializer = serializer;
 		try {
 			socket = new ServerSocket(0, 1);

http://git-wip-us.apache.org/repos/asf/flink/blob/bfe6f84c/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStreamUtils.scala
----------------------------------------------------------------------
diff --git a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStreamUtils.scala b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStreamUtils.scala
new file mode 100644
index 0000000..74dd66a
--- /dev/null
+++ b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStreamUtils.scala
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.scala
+
+import org.apache.flink.annotation.PublicEvolving
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.streaming.api.datastream.{DataStreamUtils => JavaStreamUtils}
+
+import scala.collection.JavaConverters._
+import scala.reflect.ClassTag
+
+/**
+  * This class provides simple utility methods for collecting a [[DataStream]],
+  * effectively enriching it with the functionality encapsulated by [[DataStreamUtils]].
+  *
+  * This experimental class is relocated from flink-streaming-contrib.
+  *
+  * @param self DataStream
+  */
+@PublicEvolving
+class DataStreamUtils[T: TypeInformation : ClassTag](val self: DataStream[T]) {
+
+  /**
+    * Returns a scala iterator to iterate over the elements of the DataStream.
+    * @return The iterator
+    */
+  def collect() : Iterator[T] = {
+    JavaStreamUtils.collect(self.javaStream).asScala
+  }
+}
+

http://git-wip-us.apache.org/repos/asf/flink/blob/bfe6f84c/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/experimental/scala/DataStreamUtils.scala
----------------------------------------------------------------------
diff --git a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/experimental/scala/DataStreamUtils.scala b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/experimental/scala/DataStreamUtils.scala
deleted file mode 100644
index 8c4beff..0000000
--- a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/experimental/scala/DataStreamUtils.scala
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.experimental.scala
-
-import org.apache.flink.annotation.PublicEvolving
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.streaming.api.scala.DataStream
-import org.apache.flink.streaming.experimental.{DataStreamUtils => JavaStreamUtils}
-
-import scala.collection.JavaConverters._
-import scala.reflect.ClassTag
-
-/**
-  * This class provides simple utility methods for collecting a [[DataStream]],
-  * effectively enriching it with the functionality encapsulated by [[DataStreamUtils]].
-  *
-  * This experimental class is relocated from flink-streaming-contrib.
-  *
-  * @param self DataStream
-  */
-@PublicEvolving
-class DataStreamUtils[T: TypeInformation : ClassTag](val self: DataStream[T]) {
-
-  /**
-    * Returns a scala iterator to iterate over the elements of the DataStream.
-    * @return The iterator
-    */
-  def collect() : Iterator[T] = {
-    JavaStreamUtils.collect(self.javaStream).asScala
-  }
-}
-

http://git-wip-us.apache.org/repos/asf/flink/blob/bfe6f84c/flink-tests/src/test/java/org/apache/flink/test/streaming/experimental/CollectITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/experimental/CollectITCase.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/experimental/CollectITCase.java
index ad07390..0535bf7 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/streaming/experimental/CollectITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/experimental/CollectITCase.java
@@ -21,8 +21,8 @@ package org.apache.flink.test.streaming.experimental;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
 import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.DataStreamUtils;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.experimental.DataStreamUtils;
 import org.apache.flink.streaming.util.TestStreamEnvironment;
 import org.apache.flink.util.TestLogger;