You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2017/04/05 20:42:26 UTC

[01/13] flink git commit: [FLINK-6201] [py] [dist] move python example files from resources to the examples

Repository: flink
Updated Branches:
  refs/heads/master ffca0e76e -> dc13500dd


[FLINK-6201] [py] [dist] move python example files from resources to the examples

This closes #3628.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/995422c4
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/995422c4
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/995422c4

Branch: refs/heads/master
Commit: 995422c461bc2e58bc1590c313b940e05c0fabef
Parents: c8f8a41
Author: Jinkui Shi <sh...@huawei.com>
Authored: Tue Mar 28 17:51:12 2017 +0800
Committer: zentol <ch...@apache.org>
Committed: Wed Apr 5 20:41:13 2017 +0200

----------------------------------------------------------------------
 flink-dist/src/main/assemblies/bin.xml | 10 ++++++++++
 1 file changed, 10 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/995422c4/flink-dist/src/main/assemblies/bin.xml
----------------------------------------------------------------------
diff --git a/flink-dist/src/main/assemblies/bin.xml b/flink-dist/src/main/assemblies/bin.xml
index 901cac9..e1ee00e 100644
--- a/flink-dist/src/main/assemblies/bin.xml
+++ b/flink-dist/src/main/assemblies/bin.xml
@@ -178,7 +178,17 @@ under the License.
 			<directory>../flink-libraries/flink-python/src/main/python/org/apache/flink/python/api</directory>
 			<outputDirectory>resources/python</outputDirectory>
 			<fileMode>0755</fileMode>
+			<excludes>
+				<exclude>**/example/**</exclude>
+			</excludes>
+		</fileSet>
+		<!-- copy python example to examples of dist -->
+		<fileSet>
+			<directory>../flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example</directory>
+			<outputDirectory>examples/python</outputDirectory>
+			<fileMode>0755</fileMode>
 		</fileSet>
+
 	</fileSets>
 
 </assembly>


[08/13] flink git commit: [FLINK-6086] [py] Clean up PythonSender/-Streamer generics

Posted by ch...@apache.org.
[FLINK-6086] [py] Clean up PythonSender/-Streamer generics


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/a7251c56
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/a7251c56
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/a7251c56

Branch: refs/heads/master
Commit: a7251c56d0d03293f95f0e3b084326a96fe50ce3
Parents: 9bdbe60
Author: zentol <ch...@apache.org>
Authored: Thu Mar 16 23:26:09 2017 +0100
Committer: zentol <ch...@apache.org>
Committed: Wed Apr 5 20:43:30 2017 +0200

----------------------------------------------------------------------
 .../python/api/functions/PythonCoGroup.java     |   6 +-
 .../api/functions/PythonMapPartition.java       |   6 +-
 .../streaming/data/PythonDualInputSender.java   |  70 ++++++++++
 .../streaming/data/PythonDualInputStreamer.java |  97 ++++++++++++++
 .../api/streaming/data/PythonReceiver.java      |   2 +
 .../python/api/streaming/data/PythonSender.java | 100 +++------------
 .../streaming/data/PythonSingleInputSender.java |  50 ++++++++
 .../data/PythonSingleInputStreamer.java         |  89 +++++++++++++
 .../api/streaming/data/PythonStreamer.java      | 128 ++-----------------
 .../data/SingleElementPushBackIterator.java     |  57 +++++++++
 .../data/SingleElementPushBackIteratorTest.java |  76 +++++++++++
 11 files changed, 479 insertions(+), 202 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/a7251c56/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/functions/PythonCoGroup.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/functions/PythonCoGroup.java b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/functions/PythonCoGroup.java
index 2065b98..72d3361 100644
--- a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/functions/PythonCoGroup.java
+++ b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/functions/PythonCoGroup.java
@@ -14,7 +14,7 @@ package org.apache.flink.python.api.functions;
 
 import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
 import org.apache.flink.configuration.Configuration;
-import org.apache.flink.python.api.streaming.data.PythonStreamer;
+import org.apache.flink.python.api.streaming.data.PythonDualInputStreamer;
 import org.apache.flink.util.Collector;
 import java.io.IOException;
 import org.apache.flink.api.common.functions.RichCoGroupFunction;
@@ -31,12 +31,12 @@ public class PythonCoGroup<IN1, IN2, OUT> extends RichCoGroupFunction<IN1, IN2,
 
 	private static final long serialVersionUID = -3997396583317513873L;
 
-	private final PythonStreamer<IN1, IN2, OUT> streamer;
+	private final PythonDualInputStreamer<IN1, IN2, OUT> streamer;
 	private final transient TypeInformation<OUT> typeInformation;
 
 	public PythonCoGroup(int envID, int setID, TypeInformation<OUT> typeInformation) {
 		this.typeInformation = typeInformation;
-		streamer = new PythonStreamer<>(this, envID, setID, true);
+		streamer = new PythonDualInputStreamer<>(this, envID, setID, true);
 	}
 
 	/**

http://git-wip-us.apache.org/repos/asf/flink/blob/a7251c56/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/functions/PythonMapPartition.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/functions/PythonMapPartition.java b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/functions/PythonMapPartition.java
index dc21c7c..9142581 100644
--- a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/functions/PythonMapPartition.java
+++ b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/functions/PythonMapPartition.java
@@ -18,7 +18,7 @@ import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
 import org.apache.flink.configuration.Configuration;
-import org.apache.flink.python.api.streaming.data.PythonStreamer;
+import org.apache.flink.python.api.streaming.data.PythonSingleInputStreamer;
 import org.apache.flink.util.Collector;
 
 /**
@@ -32,12 +32,12 @@ public class PythonMapPartition<IN, OUT> extends RichMapPartitionFunction<IN, OU
 
 	private static final long serialVersionUID = 3866306483023916413L;
 
-	private final PythonStreamer<IN, IN, OUT> streamer;
+	private final PythonSingleInputStreamer<IN, OUT> streamer;
 	private final transient TypeInformation<OUT> typeInformation;
 
 	public PythonMapPartition(int envId, int setId, TypeInformation<OUT> typeInformation) {
 		this.typeInformation = typeInformation;
-		streamer = new PythonStreamer(this, envId, setId, typeInformation instanceof PrimitiveArrayTypeInfo);
+		streamer = new PythonSingleInputStreamer<>(this, envId, setId, typeInformation instanceof PrimitiveArrayTypeInfo);
 	}
 
 	/**

http://git-wip-us.apache.org/repos/asf/flink/blob/a7251c56/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonDualInputSender.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonDualInputSender.java b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonDualInputSender.java
new file mode 100644
index 0000000..3b8e423
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonDualInputSender.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.python.api.streaming.data;
+
+import java.io.IOException;
+
+/**
+ * This class is a {@link PythonSender} for operations with two input streams.
+ *
+ * @param <IN1> first input type
+ * @param <IN2> second input type
+ */
+public class PythonDualInputSender<IN1, IN2> extends PythonSender {
+
+	private static final long serialVersionUID = 614115041181108878L;
+
+	private transient Serializer<IN1> serializer1;
+	private transient Serializer<IN2> serializer2;
+
+	/**
+	 * Extracts records from an iterator and writes them to the memory-mapped file. This method assumes that all values
+	 * in the iterator are of the same type. This method does NOT take care of synchronization. The caller must
+	 * guarantee that the file may be written to before calling this method.
+	 *
+	 * @param input iterator containing records
+	 * @return size of the written buffer
+	 * @throws IOException
+	 */
+	public int sendBuffer1(SingleElementPushBackIterator<IN1> input) throws IOException {
+		if (serializer1 == null) {
+			IN1 value = input.next();
+			serializer1 = getSerializer(value);
+			input.pushBack(value);
+		}
+		return sendBuffer(input, serializer1);
+	}
+
+	/**
+	 * Extracts records from an iterator and writes them to the memory-mapped file. This method assumes that all values
+	 * in the iterator are of the same type. This method does NOT take care of synchronization. The caller must
+	 * guarantee that the file may be written to before calling this method.
+	 *
+	 * @param input iterator containing records
+	 * @return size of the written buffer
+	 * @throws IOException
+	 */
+	public int sendBuffer2(SingleElementPushBackIterator<IN2> input) throws IOException {
+		if (serializer2 == null) {
+			IN2 value = input.next();
+			serializer2 = getSerializer(value);
+			input.pushBack(value);
+		}
+		return sendBuffer(input, serializer2);
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/a7251c56/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonDualInputStreamer.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonDualInputStreamer.java b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonDualInputStreamer.java
new file mode 100644
index 0000000..2e9ba2c
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonDualInputStreamer.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.python.api.streaming.data;
+
+import org.apache.flink.api.common.functions.AbstractRichFunction;
+import org.apache.flink.util.Collector;
+
+import java.io.IOException;
+import java.net.SocketTimeoutException;
+import java.util.Iterator;
+
+/**
+ * This class is a {@link PythonStreamer} for operations with two input stream.
+ *
+ * @param <IN1> first input type
+ * @param <IN2> second input type
+ * @param <OUT> output type
+ */
+public class PythonDualInputStreamer<IN1, IN2, OUT> extends PythonStreamer<PythonDualInputSender<IN1, IN2>, OUT> {
+
+	private static final long serialVersionUID = -607175070491761873L;
+
+	public PythonDualInputStreamer(AbstractRichFunction function, int envID, int setID, boolean usesByteArray) {
+		super(function, envID, setID, usesByteArray, new PythonDualInputSender<IN1, IN2>());
+	}
+
+	/**
+	 * Sends all values contained in both iterators to the external process and collects all results.
+	 *
+	 * @param iterator1 first input stream
+	 * @param iterator2 second input stream
+	 * @param c         collector
+	 * @throws IOException
+	 */
+	public final void streamBufferWithGroups(Iterator<IN1> iterator1, Iterator<IN2> iterator2, Collector<OUT> c) throws IOException {
+		SingleElementPushBackIterator<IN1> i1 = new SingleElementPushBackIterator<>(iterator1);
+		SingleElementPushBackIterator<IN2> i2 = new SingleElementPushBackIterator<>(iterator2);
+		try {
+			int size;
+			if (i1.hasNext() || i2.hasNext()) {
+				while (true) {
+					int sig = in.readInt();
+					switch (sig) {
+						case SIGNAL_BUFFER_REQUEST_G0:
+							if (i1.hasNext()) {
+								size = sender.sendBuffer1(i1);
+								sendWriteNotification(size, i1.hasNext());
+							}
+							break;
+						case SIGNAL_BUFFER_REQUEST_G1:
+							if (i2.hasNext()) {
+								size = sender.sendBuffer2(i2);
+								sendWriteNotification(size, i2.hasNext());
+							}
+							break;
+						case SIGNAL_FINISHED:
+							return;
+						case SIGNAL_ERROR:
+							try {
+								outPrinter.join();
+							} catch (InterruptedException e) {
+								outPrinter.interrupt();
+							}
+							try {
+								errorPrinter.join();
+							} catch (InterruptedException e) {
+								errorPrinter.interrupt();
+							}
+							throw new RuntimeException(
+								"External process for task " + function.getRuntimeContext().getTaskName() + " terminated prematurely due to an error." + msg);
+						default:
+							receiver.collectBuffer(c, sig);
+							sendReadConfirmation();
+							break;
+					}
+				}
+			}
+		} catch (SocketTimeoutException ignored) {
+			throw new RuntimeException("External process for task " + function.getRuntimeContext().getTaskName() + " stopped responding." + msg);
+		}
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/a7251c56/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonReceiver.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonReceiver.java b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonReceiver.java
index ba5d96a..838a261 100644
--- a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonReceiver.java
+++ b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonReceiver.java
@@ -44,6 +44,8 @@ public class PythonReceiver<OUT> implements Serializable {
 	}
 
 	//=====Setup========================================================================================================
+
+	@SuppressWarnings("unchecked")
 	public void open(String path) throws IOException {
 		setupMappedFile(path);
 		deserializer = (Deserializer<OUT>) (readAsByteArray ? new ByteArrayDeserializer() : new TupleDeserializer());

http://git-wip-us.apache.org/repos/asf/flink/blob/a7251c56/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonSender.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonSender.java b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonSender.java
index 8c40a6f..9ada758 100644
--- a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonSender.java
+++ b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonSender.java
@@ -12,6 +12,9 @@
  */
 package org.apache.flink.python.api.streaming.data;
 
+import org.apache.flink.api.java.tuple.Tuple;
+import org.apache.flink.api.java.tuple.Tuple2;
+
 import java.io.File;
 import java.io.IOException;
 import java.io.RandomAccessFile;
@@ -19,16 +22,14 @@ import java.io.Serializable;
 import java.nio.ByteBuffer;
 import java.nio.MappedByteBuffer;
 import java.nio.channels.FileChannel;
-import java.util.Iterator;
-import org.apache.flink.api.java.tuple.Tuple;
-import org.apache.flink.api.java.tuple.Tuple2;
+
 import static org.apache.flink.python.api.PythonPlanBinder.FLINK_TMP_DATA_DIR;
 import static org.apache.flink.python.api.PythonPlanBinder.MAPPED_FILE_SIZE;
 
 /**
  * General-purpose class to write data to memory-mapped files.
  */
-public class PythonSender implements Serializable {
+public abstract class PythonSender implements Serializable {
 
 	private static final long serialVersionUID = -2004095650353962110L;
 
@@ -41,14 +42,8 @@ public class PythonSender implements Serializable {
 	private transient FileChannel outputChannel;
 	private transient MappedByteBuffer fileBuffer;
 
-	private transient ByteBuffer[] saved;
-
-	private transient Serializer[] serializer;
-
 	//=====Setup========================================================================================================
 	public void open(String path) throws IOException {
-		saved = new ByteBuffer[2];
-		serializer = new Serializer[2];
 		setupMappedFile(path);
 	}
 
@@ -80,88 +75,31 @@ public class PythonSender implements Serializable {
 		outputFile.delete();
 	}
 
-	/**
-	 * Resets this object to the post-configuration state.
-	 */
-	public void reset() {
-		serializer[0] = null;
-		serializer[1] = null;
-		fileBuffer.clear();
-	}
-
 	//=====IO===========================================================================================================
 	/**
-	 * Writes a single record to the memory-mapped file. This method does NOT take care of synchronization. The user
-	 * must guarantee that the file may be written to before calling this method. This method essentially reserves the
-	 * whole buffer for one record. As such it imposes some performance restrictions and should only be used when
-	 * absolutely necessary.
-	 *
-	 * @param value record to send
-	 * @return size of the written buffer
-	 * @throws IOException
-	 */
-	@SuppressWarnings("unchecked")
-	public int sendRecord(Object value) throws IOException {
-		fileBuffer.clear();
-		int group = 0;
-
-		serializer[group] = getSerializer(value);
-		ByteBuffer bb = serializer[group].serialize(value);
-		if (bb.remaining() > MAPPED_FILE_SIZE) {
-			throw new RuntimeException("Serialized object does not fit into a single buffer.");
-		}
-		fileBuffer.put(bb);
-
-		int size = fileBuffer.position();
-
-		reset();
-		return size;
-	}
-
-	public boolean hasRemaining(int group) {
-		return saved[group] != null;
-	}
-
-	/**
 	 * Extracts records from an iterator and writes them to the memory-mapped file. This method assumes that all values
 	 * in the iterator are of the same type. This method does NOT take care of synchronization. The caller must
 	 * guarantee that the file may be written to before calling this method.
 	 *
-	 * @param i iterator containing records
-	 * @param group group to which the iterator belongs, most notably used by CoGroup-functions.
+	 * @param input     iterator containing records
+	 * @param serializer serializer for the input records
 	 * @return size of the written buffer
 	 * @throws IOException
 	 */
-	@SuppressWarnings("unchecked")
-	public int sendBuffer(Iterator<?> i, int group) throws IOException {
+	protected <IN> int sendBuffer(SingleElementPushBackIterator<IN> input, Serializer<IN> serializer) throws IOException {
 		fileBuffer.clear();
 
-		Object value;
-		ByteBuffer bb;
-		if (serializer[group] == null) {
-			value = i.next();
-			serializer[group] = getSerializer(value);
-			bb = serializer[group].serialize(value);
-			if (bb.remaining() > MAPPED_FILE_SIZE) {
-				throw new RuntimeException("Serialized object does not fit into a single buffer.");
-			}
-			fileBuffer.put(bb);
-
-		}
-		if (saved[group] != null) {
-			fileBuffer.put(saved[group]);
-			saved[group] = null;
-		}
-		while (i.hasNext() && saved[group] == null) {
-			value = i.next();
-			bb = serializer[group].serialize(value);
+		while (input.hasNext()) {
+			IN value = input.next();
+			ByteBuffer bb = serializer.serialize(value);
 			if (bb.remaining() > MAPPED_FILE_SIZE) {
 				throw new RuntimeException("Serialized object does not fit into a single buffer.");
 			}
 			if (bb.remaining() <= fileBuffer.remaining()) {
 				fileBuffer.put(bb);
 			} else {
-				saved[group] = bb;
+				input.pushBack(value);
+				break;
 			}
 		}
 
@@ -170,20 +108,22 @@ public class PythonSender implements Serializable {
 	}
 
 	//=====Serializer===================================================================================================
-	private Serializer<?> getSerializer(Object value) {
+
+	@SuppressWarnings("unchecked")
+	protected <IN> Serializer<IN> getSerializer(IN value) {
 		if (value instanceof byte[]) {
-			return new ArraySerializer();
+			return (Serializer<IN>) new ArraySerializer();
 		}
 		if (((Tuple2<?, ?>) value).f0 instanceof byte[]) {
-			return new ValuePairSerializer();
+			return (Serializer<IN>) new ValuePairSerializer();
 		}
 		if (((Tuple2<?, ?>) value).f0 instanceof Tuple) {
-			return new KeyValuePairSerializer();
+			return (Serializer<IN>) new KeyValuePairSerializer();
 		}
 		throw new IllegalArgumentException("This object can't be serialized: " + value);
 	}
 
-	private abstract static class Serializer<T> {
+	protected abstract static class Serializer<T> {
 		protected ByteBuffer buffer;
 
 		public ByteBuffer serialize(T value) {

http://git-wip-us.apache.org/repos/asf/flink/blob/a7251c56/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonSingleInputSender.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonSingleInputSender.java b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonSingleInputSender.java
new file mode 100644
index 0000000..42a1799
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonSingleInputSender.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.python.api.streaming.data;
+
+import java.io.IOException;
+
+/**
+ * This class is a {@link PythonSender} for operations with one input stream.
+ *
+ * @param <IN> input type
+ */
+public class PythonSingleInputSender<IN> extends PythonSender {
+
+	private static final long serialVersionUID = 614115041181108878L;
+
+	private transient Serializer<IN> serializer;
+
+	/**
+	 * Extracts records from an iterator and writes them to the memory-mapped file. This method assumes that all values
+	 * in the iterator are of the same type. This method does NOT take care of synchronization. The caller must
+	 * guarantee that the file may be written to before calling this method.
+	 *
+	 * @param input iterator containing records
+	 * @return size of the written buffer
+	 * @throws IOException
+	 */
+	public int sendBuffer(SingleElementPushBackIterator<IN> input) throws IOException {
+		if (serializer == null) {
+			IN value = input.next();
+			serializer = getSerializer(value);
+			input.pushBack(value);
+		}
+		return sendBuffer(input, serializer);
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/a7251c56/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonSingleInputStreamer.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonSingleInputStreamer.java b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonSingleInputStreamer.java
new file mode 100644
index 0000000..d013111
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonSingleInputStreamer.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.python.api.streaming.data;
+
+import org.apache.flink.api.common.functions.AbstractRichFunction;
+import org.apache.flink.util.Collector;
+
+import java.io.IOException;
+import java.net.SocketTimeoutException;
+import java.util.Iterator;
+
+/**
+ * This class is a {@link PythonStreamer} for operations with one input stream.
+ * @param <IN> input type
+ * @param <OUT> output type
+ */
+public class PythonSingleInputStreamer<IN, OUT> extends PythonStreamer<PythonSingleInputSender<IN>, OUT> {
+
+	private static final long serialVersionUID = -5149905918522069034L;
+
+	public PythonSingleInputStreamer(AbstractRichFunction function, int envID, int setID, boolean usesByteArray) {
+		super(function, envID, setID, usesByteArray, new PythonSingleInputSender<IN>());
+	}
+
+	/**
+	 * Sends all values contained in the iterator to the external process and collects all results.
+	 *
+	 * @param iterator input stream
+	 * @param c        collector
+	 * @throws IOException
+	 */
+	public final void streamBufferWithoutGroups(Iterator<IN> iterator, Collector<OUT> c) throws IOException {
+		SingleElementPushBackIterator<IN> i = new SingleElementPushBackIterator<>(iterator);
+		try {
+			int size;
+			if (i.hasNext()) {
+				while (true) {
+					int sig = in.readInt();
+					switch (sig) {
+						case SIGNAL_BUFFER_REQUEST:
+							if (i.hasNext()) {
+								size = sender.sendBuffer(i);
+								sendWriteNotification(size, i.hasNext());
+							} else {
+								throw new RuntimeException("External process requested data even though none is available.");
+							}
+							break;
+						case SIGNAL_FINISHED:
+							return;
+						case SIGNAL_ERROR:
+							try {
+								outPrinter.join();
+							} catch (InterruptedException e) {
+								outPrinter.interrupt();
+							}
+							try {
+								errorPrinter.join();
+							} catch (InterruptedException e) {
+								errorPrinter.interrupt();
+							}
+							throw new RuntimeException(
+								"External process for task " + function.getRuntimeContext().getTaskName() + " terminated prematurely due to an error." + msg);
+						default:
+							receiver.collectBuffer(c, sig);
+							sendReadConfirmation();
+							break;
+					}
+				}
+			}
+		} catch (SocketTimeoutException ignored) {
+			throw new RuntimeException("External process for task " + function.getRuntimeContext().getTaskName() + " stopped responding." + msg);
+		}
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/a7251c56/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonStreamer.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonStreamer.java b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonStreamer.java
index 136bb69..830c843 100644
--- a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonStreamer.java
+++ b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonStreamer.java
@@ -19,7 +19,6 @@ import org.apache.flink.python.api.PythonPlanBinder;
 import org.apache.flink.python.api.streaming.util.SerializationUtils.IntSerializer;
 import org.apache.flink.python.api.streaming.util.SerializationUtils.StringSerializer;
 import org.apache.flink.python.api.streaming.util.StreamPrinter;
-import org.apache.flink.util.Collector;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -45,16 +44,16 @@ import static org.apache.flink.python.api.PythonPlanBinder.PLANBINDER_CONFIG_BCV
 /**
  * This streamer is used by functions to send/receive data to/from an external python process.
  */
-public class PythonStreamer<IN1, IN2, OUT> implements Serializable {
+public class PythonStreamer<S extends PythonSender, OUT> implements Serializable {
 	protected static final Logger LOG = LoggerFactory.getLogger(PythonStreamer.class);
 	private static final long serialVersionUID = -2342256613658373170L;
 
-	private static final int SIGNAL_BUFFER_REQUEST = 0;
-	private static final int SIGNAL_BUFFER_REQUEST_G0 = -3;
-	private static final int SIGNAL_BUFFER_REQUEST_G1 = -4;
-	private static final int SIGNAL_FINISHED = -1;
-	private static final int SIGNAL_ERROR = -2;
-	private static final byte SIGNAL_LAST = 32;
+	protected static final int SIGNAL_BUFFER_REQUEST = 0;
+	protected static final int SIGNAL_BUFFER_REQUEST_G0 = -3;
+	protected static final int SIGNAL_BUFFER_REQUEST_G1 = -4;
+	protected static final int SIGNAL_FINISHED = -1;
+	protected static final int SIGNAL_ERROR = -2;
+	protected static final byte SIGNAL_LAST = 32;
 
 	private final int envID;
 	private final int setID;
@@ -69,7 +68,7 @@ public class PythonStreamer<IN1, IN2, OUT> implements Serializable {
 	protected transient DataOutputStream out;
 	protected int port;
 
-	protected PythonSender sender;
+	protected S sender;
 	protected PythonReceiver<OUT> receiver;
 
 	protected StringBuilder msg = new StringBuilder();
@@ -79,14 +78,14 @@ public class PythonStreamer<IN1, IN2, OUT> implements Serializable {
 	protected transient Thread outPrinter;
 	protected transient Thread errorPrinter;
 
-	public PythonStreamer(AbstractRichFunction function, int envID, int setID, boolean usesByteArray) {
+	public PythonStreamer(AbstractRichFunction function, int envID, int setID, boolean usesByteArray, S sender) {
 		this.envID = envID;
 		this.setID = setID;
 		this.usePython3 = PythonPlanBinder.usePython3;
 		planArguments = PythonPlanBinder.arguments.toString();
-		sender = new PythonSender();
 		receiver = new PythonReceiver(usesByteArray);
 		this.function = function;
+		this.sender = sender;
 	}
 
 	/**
@@ -212,13 +211,13 @@ public class PythonStreamer<IN1, IN2, OUT> implements Serializable {
 		}
 	}
 
-	private void sendWriteNotification(int size, boolean hasNext) throws IOException {
+	protected void sendWriteNotification(int size, boolean hasNext) throws IOException {
 		out.writeInt(size);
 		out.writeByte(hasNext ? 0 : SIGNAL_LAST);
 		out.flush();
 	}
 
-	private void sendReadConfirmation() throws IOException {
+	protected void sendReadConfirmation() throws IOException {
 		out.writeByte(1);
 		out.flush();
 	}
@@ -257,107 +256,4 @@ public class PythonStreamer<IN1, IN2, OUT> implements Serializable {
 			throw new RuntimeException("External process for task " + function.getRuntimeContext().getTaskName() + " stopped responding." + msg);
 		}
 	}
-
-	/**
-	 * Sends all values contained in the iterator to the external process and collects all results.
-	 *
-	 * @param i iterator
-	 * @param c collector
-	 * @throws IOException
-	 */
-	public final void streamBufferWithoutGroups(Iterator<IN1> i, Collector<OUT> c) throws IOException {
-		try {
-			int size;
-			if (i.hasNext()) {
-				while (true) {
-					int sig = in.readInt();
-					switch (sig) {
-						case SIGNAL_BUFFER_REQUEST:
-							if (i.hasNext() || sender.hasRemaining(0)) {
-								size = sender.sendBuffer(i, 0);
-								sendWriteNotification(size, sender.hasRemaining(0) || i.hasNext());
-							} else {
-								throw new RuntimeException("External process requested data even though none is available.");
-							}
-							break;
-						case SIGNAL_FINISHED:
-							return;
-						case SIGNAL_ERROR:
-							try {
-								outPrinter.join(1000);
-							} catch (InterruptedException e) {
-								outPrinter.interrupt();
-							}
-							try {
-								errorPrinter.join(1000);
-							} catch (InterruptedException e) {
-								errorPrinter.interrupt();
-							}
-							throw new RuntimeException(
-									"External process for task " + function.getRuntimeContext().getTaskName() + " terminated prematurely due to an error." + msg);
-						default:
-							receiver.collectBuffer(c, sig);
-							sendReadConfirmation();
-							break;
-					}
-				}
-			}
-		} catch (SocketTimeoutException ignored) {
-			throw new RuntimeException("External process for task " + function.getRuntimeContext().getTaskName() + " stopped responding." + msg);
-		}
-	}
-
-	/**
-	 * Sends all values contained in both iterators to the external process and collects all results.
-	 *
-	 * @param i1 iterator
-	 * @param i2 iterator
-	 * @param c collector
-	 * @throws IOException
-	 */
-	public final void streamBufferWithGroups(Iterator<IN1> i1, Iterator<IN2> i2, Collector<OUT> c) throws IOException {
-		try {
-			int size;
-			if (i1.hasNext() || i2.hasNext()) {
-				while (true) {
-					int sig = in.readInt();
-					switch (sig) {
-						case SIGNAL_BUFFER_REQUEST_G0:
-							if (i1.hasNext() || sender.hasRemaining(0)) {
-								size = sender.sendBuffer(i1, 0);
-								sendWriteNotification(size, sender.hasRemaining(0) || i1.hasNext());
-							}
-							break;
-						case SIGNAL_BUFFER_REQUEST_G1:
-							if (i2.hasNext() || sender.hasRemaining(1)) {
-								size = sender.sendBuffer(i2, 1);
-								sendWriteNotification(size, sender.hasRemaining(1) || i2.hasNext());
-							}
-							break;
-						case SIGNAL_FINISHED:
-							return;
-						case SIGNAL_ERROR:
-							try {
-								outPrinter.join(1000);
-							} catch (InterruptedException e) {
-								outPrinter.interrupt();
-							}
-							try {
-								errorPrinter.join(1000);
-							} catch (InterruptedException e) {
-								errorPrinter.interrupt();
-							}
-							throw new RuntimeException(
-									"External process for task " + function.getRuntimeContext().getTaskName() + " terminated prematurely due to an error." + msg);
-						default:
-							receiver.collectBuffer(c, sig);
-							sendReadConfirmation();
-							break;
-					}
-				}
-			}
-		} catch (SocketTimeoutException ignored) {
-			throw new RuntimeException("External process for task " + function.getRuntimeContext().getTaskName() + " stopped responding." + msg);
-		}
-	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/a7251c56/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/SingleElementPushBackIterator.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/SingleElementPushBackIterator.java b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/SingleElementPushBackIterator.java
new file mode 100644
index 0000000..ef80c98
--- /dev/null
+++ b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/SingleElementPushBackIterator.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.python.api.streaming.data;
+
+import org.apache.flink.util.Preconditions;
+
+import java.util.Iterator;
+
+/**
+ * This class is a wrapper for an {@link Iterator} that supports pushing back a single record.
+ *
+ * @param <IN> input type
+ */
+class SingleElementPushBackIterator<IN> {
+
+	private IN pushBack;
+	private final Iterator<IN> iterator;
+
+	SingleElementPushBackIterator(Iterator<IN> iterator) {
+		this.pushBack = null;
+		this.iterator = iterator;
+	}
+
+	public boolean hasNext() {
+		return pushBack != null || iterator.hasNext();
+	}
+
+	public IN next() {
+		if (pushBack != null) {
+			IN obj = pushBack;
+			pushBack = null;
+			return obj;
+		} else {
+			return iterator.next();
+		}
+	}
+
+	public void pushBack(IN element) {
+		Preconditions.checkState(pushBack == null, "Already contains an element that was pushed back. This indicates a programming error.");
+		pushBack = element;
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/a7251c56/flink-libraries/flink-python/src/test/java/org/apache/flink/python/api/streaming/data/SingleElementPushBackIteratorTest.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/test/java/org/apache/flink/python/api/streaming/data/SingleElementPushBackIteratorTest.java b/flink-libraries/flink-python/src/test/java/org/apache/flink/python/api/streaming/data/SingleElementPushBackIteratorTest.java
new file mode 100644
index 0000000..5e9eb42
--- /dev/null
+++ b/flink-libraries/flink-python/src/test/java/org/apache/flink/python/api/streaming/data/SingleElementPushBackIteratorTest.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.python.api.streaming.data;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+
+public class SingleElementPushBackIteratorTest {
+
+	@Test
+	public void testPushBackIterator() {
+		Collection<Integer> init = new ArrayList<>();
+		init.add(1);
+		init.add(2);
+		init.add(4);
+		init.add(5);
+		SingleElementPushBackIterator<Integer> iterator = new SingleElementPushBackIterator<>(init.iterator());
+
+		Assert.assertTrue(iterator.hasNext());
+		Assert.assertEquals(1, (int) iterator.next());
+
+		Assert.assertTrue(iterator.hasNext());
+		Assert.assertEquals(2, (int) iterator.next());
+
+		Assert.assertTrue(iterator.hasNext());
+		iterator.pushBack(3);
+		Assert.assertTrue(iterator.hasNext());
+		Assert.assertEquals(3, (int) iterator.next());
+
+		Assert.assertTrue(iterator.hasNext());
+		Assert.assertEquals(4, (int) iterator.next());
+
+		Assert.assertTrue(iterator.hasNext());
+		Assert.assertEquals(5, (int) iterator.next());
+
+		Assert.assertFalse(iterator.hasNext());
+		iterator.pushBack(6);
+		Assert.assertTrue(iterator.hasNext());
+		Assert.assertEquals(6, (int) iterator.next());
+
+		Assert.assertFalse(iterator.hasNext());
+	}
+
+	@Test
+	public void testSingleElementLimitation() {
+		Collection<Integer> init = Collections.emptyList();
+		SingleElementPushBackIterator<Integer> iterator = new SingleElementPushBackIterator<>(init.iterator());
+		Assert.assertFalse(iterator.hasNext());
+		iterator.pushBack(1);
+		try {
+			iterator.pushBack(2);
+			Assert.fail("Multiple elements could be pushed back.");
+		} catch (IllegalStateException ignored) {
+			// expected
+		}
+	}
+}


[11/13] flink git commit: [hotfix] [py] Code cleanup - PythonStreamer

Posted by ch...@apache.org.
[hotfix] [py] Code cleanup - PythonStreamer


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/dd588efe
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/dd588efe
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/dd588efe

Branch: refs/heads/master
Commit: dd588efe6470dfdcad51249ff36f12cfde7b75da
Parents: 258ed17
Author: zentol <ch...@apache.org>
Authored: Thu Mar 30 23:13:07 2017 +0200
Committer: zentol <ch...@apache.org>
Committed: Wed Apr 5 20:44:30 2017 +0200

----------------------------------------------------------------------
 .../python/api/streaming/data/PythonStreamer.java | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/dd588efe/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonStreamer.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonStreamer.java b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonStreamer.java
index 3c79b1f..006a1b2 100644
--- a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonStreamer.java
+++ b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonStreamer.java
@@ -84,7 +84,7 @@ public class PythonStreamer<S extends PythonSender, OUT> implements Serializable
 		this.setID = setID;
 		this.usePython3 = PythonPlanBinder.usePython3;
 		planArguments = PythonPlanBinder.arguments.toString();
-		receiver = new PythonReceiver(usesByteArray);
+		receiver = new PythonReceiver<>(usesByteArray);
 		this.function = function;
 		this.sender = sender;
 	}
@@ -162,10 +162,22 @@ public class PythonStreamer<S extends PythonSender, OUT> implements Serializable
 	private void checkPythonProcessHealth() {
 		try {
 			int value = process.exitValue();
+			try {
+				outPrinter.join();
+			} catch (InterruptedException ignored) {
+				outPrinter.interrupt();
+				Thread.interrupted();
+			}
+			try {
+				errorPrinter.join();
+			} catch (InterruptedException ignored) {
+				errorPrinter.interrupt();
+				Thread.interrupted();
+			}
 			if (value != 0) {
-				throw new RuntimeException("Plan file caused an error. Check log-files for details.");
+				throw new RuntimeException("Plan file caused an error. Check log-files for details." + msg.get());
 			} else {
-				throw new RuntimeException("Plan file exited prematurely without an error.");
+				throw new RuntimeException("Plan file exited prematurely without an error." + msg.get());
 			}
 		} catch (IllegalThreadStateException ignored) {//Process still running
 		}


[06/13] flink git commit: [hotfix] Fix unchecked conversion in JobDetailsHandler

Posted by ch...@apache.org.
[hotfix] Fix unchecked conversion in JobDetailsHandler


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/48ce77c7
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/48ce77c7
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/48ce77c7

Branch: refs/heads/master
Commit: 48ce77c79d3efd0ccab2806057ff01cf435f7c1a
Parents: 68605d0
Author: zentol <ch...@apache.org>
Authored: Wed Mar 29 16:09:12 2017 +0200
Committer: zentol <ch...@apache.org>
Committed: Wed Apr 5 20:41:15 2017 +0200

----------------------------------------------------------------------
 .../flink/runtime/webmonitor/handlers/JobDetailsHandler.java       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/48ce77c7/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JobDetailsHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JobDetailsHandler.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JobDetailsHandler.java
index 029a4b5..37a1c19 100644
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JobDetailsHandler.java
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JobDetailsHandler.java
@@ -80,7 +80,7 @@ public class JobDetailsHandler extends AbstractExecutionGraphRequestHandler {
 				.replace(":jobid", graph.getJobID().toString());
 			String path2 = JOB_DETAILS_VERTICES_REST_PATH
 				.replace(":jobid", graph.getJobID().toString());
-			Collection<ArchivedJson> archives = new ArrayList();
+			Collection<ArchivedJson> archives = new ArrayList<>();
 			archives.add(new ArchivedJson(path1, json));
 			archives.add(new ArchivedJson(path2, json));
 			return archives;


[10/13] flink git commit: [hotfix] [py] Code cleanup - StreamPrinter

Posted by ch...@apache.org.
[hotfix] [py] Code cleanup - StreamPrinter

- implements Runnable instead of extending Thread
- use AtomicRefence<String> instead of StringBuilder
- remove redundant wrapInException argument


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/258ed179
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/258ed179
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/258ed179

Branch: refs/heads/master
Commit: 258ed1791a3e66443b29e4a4e14dcc4e03ffd9e6
Parents: f4324ba
Author: zentol <ch...@apache.org>
Authored: Thu Mar 30 19:41:54 2017 +0200
Committer: zentol <ch...@apache.org>
Committed: Wed Apr 5 20:43:55 2017 +0200

----------------------------------------------------------------------
 .../api/streaming/data/PythonStreamer.java      |  7 +++--
 .../api/streaming/plan/PythonPlanStreamer.java  |  4 +--
 .../api/streaming/util/StreamPrinter.java       | 30 ++++++++++++--------
 3 files changed, 24 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/258ed179/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonStreamer.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonStreamer.java b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonStreamer.java
index 219ae27..3c79b1f 100644
--- a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonStreamer.java
+++ b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonStreamer.java
@@ -32,6 +32,7 @@ import java.net.ServerSocket;
 import java.net.Socket;
 import java.net.SocketTimeoutException;
 import java.util.Iterator;
+import java.util.concurrent.atomic.AtomicReference;
 
 import static org.apache.flink.python.api.PythonPlanBinder.FLINK_PYTHON2_BINARY_PATH;
 import static org.apache.flink.python.api.PythonPlanBinder.FLINK_PYTHON3_BINARY_PATH;
@@ -71,7 +72,7 @@ public class PythonStreamer<S extends PythonSender, OUT> implements Serializable
 	protected S sender;
 	protected PythonReceiver<OUT> receiver;
 
-	protected StringBuilder msg = new StringBuilder();
+	protected AtomicReference<String> msg = new AtomicReference<>();
 
 	protected final AbstractRichFunction function;
 
@@ -118,9 +119,9 @@ public class PythonStreamer<S extends PythonSender, OUT> implements Serializable
 		}
 
 		process = Runtime.getRuntime().exec(pythonBinaryPath + " -O -B " + planPath + planArguments);
-		outPrinter = new StreamPrinter(process.getInputStream());
+		outPrinter = new Thread(new StreamPrinter(process.getInputStream()));
 		outPrinter.start();
-		errorPrinter = new StreamPrinter(process.getErrorStream(), true, msg);
+		errorPrinter = new Thread(new StreamPrinter(process.getErrorStream(), msg));
 		errorPrinter.start();
 
 		shutdownThread = new Thread() {

http://git-wip-us.apache.org/repos/asf/flink/blob/258ed179/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/plan/PythonPlanStreamer.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/plan/PythonPlanStreamer.java b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/plan/PythonPlanStreamer.java
index 4eb0f51..9b62563 100644
--- a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/plan/PythonPlanStreamer.java
+++ b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/plan/PythonPlanStreamer.java
@@ -67,8 +67,8 @@ public class PythonPlanStreamer {
 		}
 		process = Runtime.getRuntime().exec(pythonBinaryPath + " -B " + tmpPath + FLINK_PYTHON_PLAN_NAME + args);
 
-		new StreamPrinter(process.getInputStream()).start();
-		new StreamPrinter(process.getErrorStream()).start();
+		new Thread(new StreamPrinter(process.getInputStream())).start();
+		new Thread(new StreamPrinter(process.getErrorStream())).start();
 
 		server = new ServerSocket(0);
 		server.setSoTimeout(50);

http://git-wip-us.apache.org/repos/asf/flink/blob/258ed179/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/util/StreamPrinter.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/util/StreamPrinter.java b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/util/StreamPrinter.java
index 30a728c..c6a1ede 100644
--- a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/util/StreamPrinter.java
+++ b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/util/StreamPrinter.java
@@ -18,40 +18,46 @@ import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.util.concurrent.atomic.AtomicReference;
 
 /**
  * Simple utility class to print all contents of an inputstream to stdout.
  */
-public class StreamPrinter extends Thread {
+public class StreamPrinter implements Runnable {
 	private final BufferedReader reader;
-	private final boolean wrapInException;
-	private StringBuilder msg;
+	private final AtomicReference<String> output;
 
 	public StreamPrinter(InputStream stream) {
-		this(stream, false, null);
+		this(stream, null);
 	}
 
-	public StreamPrinter(InputStream stream, boolean wrapInException, StringBuilder msg) {
+	public StreamPrinter(InputStream stream, AtomicReference<String> output) {
 		this.reader = new BufferedReader(new InputStreamReader(stream, ConfigConstants.DEFAULT_CHARSET));
-		this.wrapInException = wrapInException;
-		this.msg = msg;
+		this.output = output;
 	}
 
 	@Override
 	public void run() {
 		String line;
-		try {
-			if (wrapInException) {
+		if (output != null) {
+			StringBuilder msg = new StringBuilder();
+			try {
 				while ((line = reader.readLine()) != null) {
-					msg.append("\n" + line);
+					msg.append(line);
+					msg.append("\n");
 				}
-			} else {
+			} catch (IOException ignored) {
+			} finally {
+				output.set(msg.toString());
+			}
+		} else {
+			try {
 				while ((line = reader.readLine()) != null) {
 					System.out.println(line);
 					System.out.flush();
 				}
+			} catch (IOException ignored) {
 			}
-		} catch (IOException ex) {
 		}
 	}
 }


[12/13] flink git commit: [hotfix] Fix shading checks

Posted by ch...@apache.org.
[hotfix] Fix shading checks


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/5c44f539
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/5c44f539
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/5c44f539

Branch: refs/heads/master
Commit: 5c44f53966a0ec5e96bc94b09f6067acf24cc969
Parents: dd588ef
Author: Robert Metzger <rm...@apache.org>
Authored: Thu Mar 23 15:13:25 2017 +0100
Committer: zentol <ch...@apache.org>
Committed: Wed Apr 5 20:48:30 2017 +0200

----------------------------------------------------------------------
 tools/travis_mvn_watchdog.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/5c44f539/tools/travis_mvn_watchdog.sh
----------------------------------------------------------------------
diff --git a/tools/travis_mvn_watchdog.sh b/tools/travis_mvn_watchdog.sh
index d1b396c..b4c8285 100755
--- a/tools/travis_mvn_watchdog.sh
+++ b/tools/travis_mvn_watchdog.sh
@@ -164,7 +164,7 @@ watchdog () {
 
 # Check the final fat jar for illegal artifacts
 check_shaded_artifacts() {
-	jar tf build-target/lib/flink-dist-*.jar > allClasses
+	jar tf build-target/lib/flink-dist*.jar > allClasses
 	ASM=`cat allClasses | grep '^org/objectweb/asm/' | wc -l`
 	if [ $ASM != "0" ]; then
 		echo "=============================================================================="


[03/13] flink git commit: [FLINK-6127] [checkstyle] Add MissingDeprecation check

Posted by ch...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AggregatingProcessingTimeWindowOperator.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AggregatingProcessingTimeWindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AggregatingProcessingTimeWindowOperator.java
index 2175fb5..1be9650 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AggregatingProcessingTimeWindowOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AggregatingProcessingTimeWindowOperator.java
@@ -24,6 +24,10 @@ import org.apache.flink.api.common.functions.ReduceFunction;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.java.functions.KeySelector;
 
+/**
+ * @deprecated Deprecated in favour of the generic {@link WindowOperator}. This was an
+ * optimized implementation used for aligned windows.
+ */
 @Internal
 @Deprecated
 public class AggregatingProcessingTimeWindowOperator<KEY, IN> 

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java
----------------------------------------------------------------------
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java b/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java
index 863b442..f4557c9 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java
@@ -105,6 +105,12 @@ public class FlinkYarnSessionCli implements CustomCommandLine<YarnClusterClient>
 	private final Option SLOTS;
 	private final Option DETACHED;
 	private final Option ZOOKEEPER_NAMESPACE;
+
+	/**
+	 * @deprecated Streaming mode has been deprecated without replacement. Set the
+	 * {@link ConfigConstants#TASK_MANAGER_MEMORY_PRE_ALLOCATE_KEY} configuration
+	 * key to true to get the previous batch mode behaviour.
+	 */
 	@Deprecated
 	private final Option STREAMING;
 	private final Option NAME;

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/tools/maven/checkstyle.xml
----------------------------------------------------------------------
diff --git a/tools/maven/checkstyle.xml b/tools/maven/checkstyle.xml
index b8f1b1a..00e173b 100644
--- a/tools/maven/checkstyle.xml
+++ b/tools/maven/checkstyle.xml
@@ -124,6 +124,8 @@ under the License.
 
 		<module name="FileContentsHolder"/>
 
+		<module name="MissingDeprecated"/>
+
 	<!--	<module name="ConstantName" />
 		<module name="LocalFinalVariableName" />
 		<module name="LocalVariableName" />


[05/13] flink git commit: [FLINK-6186] Remove unused import in StreamExecutionEnvironment.scala

Posted by ch...@apache.org.
[FLINK-6186] Remove unused import in StreamExecutionEnvironment.scala

This closes #3612.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/6c224047
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/6c224047
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/6c224047

Branch: refs/heads/master
Commit: 6c224047fb5d7874db442bce3815ac797af8d942
Parents: 995422c
Author: zcb <zh...@huawei.com>
Authored: Sat Mar 25 15:42:03 2017 +0800
Committer: zentol <ch...@apache.org>
Committed: Wed Apr 5 20:41:14 2017 +0200

----------------------------------------------------------------------
 .../flink/streaming/api/scala/StreamExecutionEnvironment.scala      | 1 -
 1 file changed, 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/6c224047/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
index 60798e0..927c94e 100644
--- a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
+++ b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
@@ -23,7 +23,6 @@ import org.apache.flink.annotation.{Internal, Public, PublicEvolving}
 import org.apache.flink.api.common.io.{FileInputFormat, FilePathFilter, InputFormat}
 import org.apache.flink.api.common.restartstrategy.RestartStrategies.RestartStrategyConfiguration
 import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.java.ExecutionEnvironment
 import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer
 import org.apache.flink.api.scala.ClosureCleaner
 import org.apache.flink.configuration.Configuration


[07/13] flink git commit: [FLINK-6246] Fix generic type of OutputTag in operator Output

Posted by ch...@apache.org.
[FLINK-6246] Fix generic type of OutputTag in operator Output

This closes #3662.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/9bdbe607
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/9bdbe607
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/9bdbe607

Branch: refs/heads/master
Commit: 9bdbe6071f1946391598709bfa637fd76a8c7396
Parents: 48ce77c
Author: Aljoscha Krettek <al...@gmail.com>
Authored: Mon Apr 3 16:10:14 2017 +0200
Committer: zentol <ch...@apache.org>
Committed: Wed Apr 5 20:43:00 2017 +0200

----------------------------------------------------------------------
 .../api/collector/selector/DirectedOutput.java     |  2 +-
 .../api/operators/AbstractStreamOperator.java      |  2 +-
 .../flink/streaming/api/operators/Output.java      |  2 +-
 .../streaming/runtime/io/RecordWriterOutput.java   |  2 +-
 .../streaming/runtime/tasks/OperatorChain.java     | 17 ++++++-----------
 .../runtime/tasks/StreamIterationTail.java         |  2 +-
 .../util/AbstractStreamOperatorTestHarness.java    |  2 +-
 .../flink/streaming/util/CollectorOutput.java      |  2 +-
 .../apache/flink/streaming/util/MockOutput.java    |  2 +-
 9 files changed, 14 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/9bdbe607/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/DirectedOutput.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/DirectedOutput.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/DirectedOutput.java
index dabe804..6339506 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/DirectedOutput.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/DirectedOutput.java
@@ -140,7 +140,7 @@ public class DirectedOutput<OUT> implements Output<StreamRecord<OUT>> {
 	}
 
 	@Override
-	public <X> void collect(OutputTag<?> outputTag, StreamRecord<X> record) {
+	public <X> void collect(OutputTag<X> outputTag, StreamRecord<X> record) {
 		throw new UnsupportedOperationException("Cannot use split/select with side outputs.");
 	}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/9bdbe607/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
index cc81a0e..7569170 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
@@ -872,7 +872,7 @@ public abstract class AbstractStreamOperator<OUT>
 		}
 
 		@Override
-		public <X> void collect(OutputTag<?> outputTag, StreamRecord<X> record) {
+		public <X> void collect(OutputTag<X> outputTag, StreamRecord<X> record) {
 			numRecordsOut.inc();
 			output.collect(outputTag, record);
 		}

http://git-wip-us.apache.org/repos/asf/flink/blob/9bdbe607/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/Output.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/Output.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/Output.java
index eb10d8d..7035d28 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/Output.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/Output.java
@@ -48,7 +48,7 @@ public interface Output<T> extends Collector<T> {
 	 *
 	 * @param record The record to collect.
 	 */
-	<X> void collect(OutputTag<?> outputTag, StreamRecord<X> record);
+	<X> void collect(OutputTag<X> outputTag, StreamRecord<X> record);
 
 	void emitLatencyMarker(LatencyMarker latencyMarker);
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/9bdbe607/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/RecordWriterOutput.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/RecordWriterOutput.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/RecordWriterOutput.java
index d22c60d..365c78e 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/RecordWriterOutput.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/RecordWriterOutput.java
@@ -85,7 +85,7 @@ public class RecordWriterOutput<OUT> implements Output<StreamRecord<OUT>> {
 	}
 
 	@Override
-	public <X> void collect(OutputTag<?> outputTag, StreamRecord<X> record) {
+	public <X> void collect(OutputTag<X> outputTag, StreamRecord<X> record) {
 		if (this.outputTag == null || !this.outputTag.equals(outputTag)) {
 			// we are only responsible for emitting to the side-output specified by our
 			// OutputTag.

http://git-wip-us.apache.org/repos/asf/flink/blob/9bdbe607/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
index 499d4a3..be4b456 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
@@ -423,7 +423,7 @@ public class OperatorChain<OUT, OP extends StreamOperator<OUT>> implements Strea
 		}
 
 		@Override
-		public <X> void collect(OutputTag<?> outputTag, StreamRecord<X> record) {
+		public <X> void collect(OutputTag<X> outputTag, StreamRecord<X> record) {
 			if (this.outputTag == null || !this.outputTag.equals(outputTag)) {
 				// we are only responsible for emitting to the side-output specified by our
 				// OutputTag.
@@ -506,7 +506,7 @@ public class OperatorChain<OUT, OP extends StreamOperator<OUT>> implements Strea
 		}
 
 		@Override
-		public <X> void collect(OutputTag<?> outputTag, StreamRecord<X> record) {
+		public <X> void collect(OutputTag<X> outputTag, StreamRecord<X> record) {
 			if (this.outputTag == null || !this.outputTag.equals(outputTag)) {
 				// we are only responsible for emitting to the side-output specified by our
 				// OutputTag.
@@ -579,7 +579,7 @@ public class OperatorChain<OUT, OP extends StreamOperator<OUT>> implements Strea
 		}
 
 		@Override
-		public <X> void collect(OutputTag<?> outputTag, StreamRecord<X> record) {
+		public <X> void collect(OutputTag<X> outputTag, StreamRecord<X> record) {
 			for (Output<StreamRecord<T>> output : outputs) {
 				output.collect(outputTag, record);
 			}
@@ -619,21 +619,16 @@ public class OperatorChain<OUT, OP extends StreamOperator<OUT>> implements Strea
 		}
 
 		@Override
-		public <X> void collect(OutputTag<?> outputTag, StreamRecord<X> record) {
+		public <X> void collect(OutputTag<X> outputTag, StreamRecord<X> record) {
 			for (int i = 0; i < outputs.length - 1; i++) {
 				Output<StreamRecord<T>> output = outputs[i];
 
-				// due to side outputs, StreamRecords of varying types can pass through the broadcasting
-				// collector so we need to cast
-				@SuppressWarnings({"unchecked", "rawtypes"})
-				StreamRecord<T> shallowCopy = (StreamRecord<T>) record.copy(record.getValue());
+				StreamRecord<X> shallowCopy = record.copy(record.getValue());
 				output.collect(outputTag, shallowCopy);
 			}
 
 			// don't copy for the last output
-			@SuppressWarnings({"unchecked", "rawtypes"})
-			StreamRecord<T> castRecord = (StreamRecord<T>) record;
-			outputs[outputs.length - 1].collect(outputTag, castRecord);
+			outputs[outputs.length - 1].collect(outputTag, record);
 		}
 	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/9bdbe607/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamIterationTail.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamIterationTail.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamIterationTail.java
index 35d14e7..e40f834 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamIterationTail.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamIterationTail.java
@@ -123,7 +123,7 @@ public class StreamIterationTail<IN> extends OneInputStreamTask<IN, IN> {
 		}
 
 		@Override
-		public <X> void collect(OutputTag<?> outputTag, StreamRecord<X> record) {
+		public <X> void collect(OutputTag<X> outputTag, StreamRecord<X> record) {
 			throw new UnsupportedOperationException("Side outputs not used in iteration tail");
 
 		}

http://git-wip-us.apache.org/repos/asf/flink/blob/9bdbe607/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/AbstractStreamOperatorTestHarness.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/AbstractStreamOperatorTestHarness.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/AbstractStreamOperatorTestHarness.java
index 912d579..a2a4f79 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/AbstractStreamOperatorTestHarness.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/AbstractStreamOperatorTestHarness.java
@@ -656,7 +656,7 @@ public class AbstractStreamOperatorTestHarness<OUT> {
 		}
 
 		@Override
-		public <X> void collect(OutputTag<?> outputTag, StreamRecord<X> record) {
+		public <X> void collect(OutputTag<X> outputTag, StreamRecord<X> record) {
 			sideOutputSerializer = TypeExtractor.getForObject(record.getValue()).createSerializer(executionConfig);
 
 			ConcurrentLinkedQueue<Object> sideOutputList = sideOutputLists.get(outputTag);

http://git-wip-us.apache.org/repos/asf/flink/blob/9bdbe607/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/CollectorOutput.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/CollectorOutput.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/CollectorOutput.java
index 07b37c8..bd929da 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/CollectorOutput.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/CollectorOutput.java
@@ -54,7 +54,7 @@ public class CollectorOutput<T> implements Output<StreamRecord<T>> {
 	}
 
 	@Override
-	public <X> void collect(OutputTag<?> outputTag, StreamRecord<X> record) {
+	public <X> void collect(OutputTag<X> outputTag, StreamRecord<X> record) {
 		throw new UnsupportedOperationException("Side output not supported for CollectorOutput");
 	}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/9bdbe607/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/MockOutput.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/MockOutput.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/MockOutput.java
index 867080c..8c3226b 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/MockOutput.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/MockOutput.java
@@ -41,7 +41,7 @@ public class MockOutput<T> implements Output<StreamRecord<T>> {
 	}
 
 	@Override
-	public <X> void collect(OutputTag<?> outputTag, StreamRecord<X> record) {
+	public <X> void collect(OutputTag<X> outputTag, StreamRecord<X> record) {
 		throw new UnsupportedOperationException("Side output not supported for MockOutput");
 	}
 


[13/13] flink git commit: [FLINK-6181][Start scripts] Fix regex in start scripts

Posted by ch...@apache.org.
[FLINK-6181][Start scripts] Fix regex in start scripts

This closes #3605.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/dc13500d
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/dc13500d
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/dc13500d

Branch: refs/heads/master
Commit: dc13500dd5c8061eb75a64a62db6aa13f8fd5408
Parents: 5c44f53
Author: Robert Metzger <rm...@apache.org>
Authored: Fri Mar 24 11:50:11 2017 +0100
Committer: zentol <ch...@apache.org>
Committed: Wed Apr 5 20:48:44 2017 +0200

----------------------------------------------------------------------
 flink-dist/src/main/flink-bin/bin/start-zookeeper-quorum.sh | 2 +-
 flink-dist/src/main/flink-bin/bin/stop-zookeeper-quorum.sh  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/dc13500d/flink-dist/src/main/flink-bin/bin/start-zookeeper-quorum.sh
----------------------------------------------------------------------
diff --git a/flink-dist/src/main/flink-bin/bin/start-zookeeper-quorum.sh b/flink-dist/src/main/flink-bin/bin/start-zookeeper-quorum.sh
index 05a5208..cc8917b 100755
--- a/flink-dist/src/main/flink-bin/bin/start-zookeeper-quorum.sh
+++ b/flink-dist/src/main/flink-bin/bin/start-zookeeper-quorum.sh
@@ -35,7 +35,7 @@ while read server ; do
     server=$(echo -e "${server}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') # trim
 
     # match server.id=address[:port[:port]]
-    if [[ $server =~ ^server\.([0-9]+)[[:space:]]*\=([^: \#]+) ]]; then
+    if [[ $server =~ ^server\.([0-9]+)[[:space:]]*\=[[:space:]]*([^: \#]+) ]]; then
         id=${BASH_REMATCH[1]}
         address=${BASH_REMATCH[2]}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/dc13500d/flink-dist/src/main/flink-bin/bin/stop-zookeeper-quorum.sh
----------------------------------------------------------------------
diff --git a/flink-dist/src/main/flink-bin/bin/stop-zookeeper-quorum.sh b/flink-dist/src/main/flink-bin/bin/stop-zookeeper-quorum.sh
index b306648..29ddae4 100755
--- a/flink-dist/src/main/flink-bin/bin/stop-zookeeper-quorum.sh
+++ b/flink-dist/src/main/flink-bin/bin/stop-zookeeper-quorum.sh
@@ -35,7 +35,7 @@ while read server ; do
     server=$(echo -e "${server}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') # trim
 
     # match server.id=address[:port[:port]]
-    if [[ $server =~ ^server\.([0-9])+[[:space:]]*\=([^: \#]+) ]]; then
+    if [[ $server =~ ^server\.([0-9]+)[[:space:]]*\=[[:space:]]*([^: \#]+) ]]; then
         id=${BASH_REMATCH[1]}
         server=${BASH_REMATCH[2]}
 


[04/13] flink git commit: [FLINK-6127] [checkstyle] Add MissingDeprecation check

Posted by ch...@apache.org.
[FLINK-6127] [checkstyle] Add MissingDeprecation check

This closes #3572.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/68605d05
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/68605d05
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/68605d05

Branch: refs/heads/master
Commit: 68605d05107e2c70c12178b1db6a9e49641dbfe4
Parents: 6c22404
Author: Ufuk Celebi <uc...@apache.org>
Authored: Mon Mar 20 11:45:52 2017 +0100
Committer: zentol <ch...@apache.org>
Committed: Wed Apr 5 20:41:14 2017 +0200

----------------------------------------------------------------------
 .../connectors/kafka/FlinkKafkaConsumer081.java |  4 ++
 .../connectors/kafka/FlinkKafkaConsumer082.java |  4 ++
 .../connectors/kafka/FlinkKafkaProducer.java    | 22 +++++++-
 .../state/RocksDBKeyedStateBackend.java         |  2 +
 .../streaming/state/RocksDBStateBackend.java    |  3 +
 .../flink/api/common/JobExecutionResult.java    |  1 +
 .../api/common/io/DelimitedInputFormat.java     |  2 +-
 .../flink/api/common/state/StateDescriptor.java | 12 +++-
 .../flink/configuration/ConfigConstants.java    | 58 ++++++++++----------
 .../flink/migration/util/SerializedValue.java   |  1 +
 .../org/apache/flink/api/java/io/CsvReader.java |  2 +
 .../flink/api/java/operators/CrossOperator.java |  2 +-
 .../flink/api/java/operators/DataSink.java      |  4 ++
 .../flink/api/java/operators/JoinOperator.java  |  2 +-
 .../api/java/operators/ProjectOperator.java     |  2 +-
 ...nkAggregateExpandDistinctAggregatesRule.java | 10 +++-
 .../rules/FlinkAggregateJoinTransposeRule.java  | 20 +++++--
 .../MigrationNamespaceSerializerProxy.java      |  4 +-
 .../api/common/state/ListStateDescriptor.java   |  2 +
 .../runtime/checkpoint/KeyGroupState.java       |  2 +
 .../runtime/checkpoint/SubtaskState.java        |  3 +
 .../migration/runtime/checkpoint/TaskState.java |  3 +
 .../runtime/state/AbstractCloseableHandle.java  |  2 +
 .../runtime/state/AbstractStateBackend.java     |  2 +
 .../runtime/state/KvStateSnapshot.java          |  3 +
 .../migration/runtime/state/StateHandle.java    |  2 +
 .../migration/runtime/state/StateObject.java    |  2 +
 .../runtime/state/StreamStateHandle.java        |  2 +
 .../filesystem/AbstractFileStateHandle.java     |  2 +
 .../filesystem/AbstractFsStateSnapshot.java     |  2 +
 .../filesystem/FileSerializableStateHandle.java |  2 +
 .../state/filesystem/FileStreamStateHandle.java |  2 +
 .../state/filesystem/FsFoldingState.java        |  3 +
 .../runtime/state/filesystem/FsListState.java   |  3 +
 .../state/filesystem/FsReducingState.java       |  3 +
 .../state/filesystem/FsStateBackend.java        |  3 +
 .../runtime/state/filesystem/FsValueState.java  |  3 +
 .../state/memory/AbstractMemStateSnapshot.java  |  3 +
 .../AbstractMigrationRestoreStrategy.java       |  4 +-
 .../state/memory/ByteStreamStateHandle.java     |  3 +
 .../runtime/state/memory/MemFoldingState.java   |  3 +
 .../runtime/state/memory/MemListState.java      |  3 +
 .../runtime/state/memory/MemReducingState.java  |  2 +
 .../runtime/state/memory/MemValueState.java     |  2 +
 .../state/memory/MigrationRestoreSnapshot.java  |  3 +
 .../state/MigrationKeyGroupStateHandle.java     |  2 +
 .../state/MigrationStreamStateHandle.java       |  2 +
 .../runtime/tasks/StreamTaskState.java          |  3 +
 .../runtime/tasks/StreamTaskStateList.java      |  3 +
 .../flink/runtime/checkpoint/SubtaskState.java  |  7 +++
 .../StandaloneLeaderRetrievalService.java       |  1 +
 .../operators/hash/InMemoryPartition.java       |  2 +
 .../flink/runtime/state/TaskStateHandles.java   | 11 +++-
 .../state/heap/HeapKeyedStateBackend.java       |  5 +-
 .../api/checkpoint/CheckpointedRestoring.java   |  2 +
 .../environment/StreamExecutionEnvironment.java |  5 ++
 .../functions/AscendingTimestampExtractor.java  |  2 +
 .../source/FileMonitoringFunction.java          |  5 ++
 .../api/functions/source/FileReadFunction.java  |  5 ++
 .../api/operators/AbstractStreamOperator.java   |  8 +++
 .../CheckpointedRestoringOperator.java          |  4 ++
 .../operators/StreamCheckpointedOperator.java   |  2 +-
 .../api/windowing/triggers/Trigger.java         |  2 +
 ...ractAlignedProcessingTimeWindowOperator.java |  4 ++
 ...ccumulatingProcessingTimeWindowOperator.java |  4 ++
 ...AggregatingProcessingTimeWindowOperator.java |  4 ++
 .../flink/yarn/cli/FlinkYarnSessionCli.java     |  6 ++
 tools/maven/checkstyle.xml                      |  2 +
 68 files changed, 269 insertions(+), 46 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer081.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer081.java b/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer081.java
index 56ccd0b..4e4050f 100644
--- a/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer081.java
+++ b/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer081.java
@@ -22,6 +22,8 @@ import java.util.Properties;
 
 /**
  * THIS CLASS IS DEPRECATED. Use FlinkKafkaConsumer08 instead.
+ *
+ * @deprecated Use {@link FlinkKafkaConsumer08}
  */
 @Deprecated
 public class FlinkKafkaConsumer081<T> extends FlinkKafkaConsumer08<T> {
@@ -30,6 +32,8 @@ public class FlinkKafkaConsumer081<T> extends FlinkKafkaConsumer08<T> {
 
 	/**
 	 * THIS CONSTRUCTOR IS DEPRECATED. Use FlinkKafkaConsumer08 instead.
+	 *
+	 * @deprecated Use {@link FlinkKafkaConsumer08#FlinkKafkaConsumer08(String, DeserializationSchema, Properties)}
 	 */
 	@Deprecated
 	public FlinkKafkaConsumer081(String topic, DeserializationSchema<T> valueDeserializer, Properties props) {

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer082.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer082.java b/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer082.java
index 0520336..aeefcc8 100644
--- a/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer082.java
+++ b/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer082.java
@@ -22,6 +22,8 @@ import java.util.Properties;
 
 /**
  * THIS CLASS IS DEPRECATED. Use FlinkKafkaConsumer08 instead.
+ *
+ * @deprecated Use {@link FlinkKafkaConsumer08}
  */
 @Deprecated
 public class FlinkKafkaConsumer082<T> extends FlinkKafkaConsumer08<T> {
@@ -30,6 +32,8 @@ public class FlinkKafkaConsumer082<T> extends FlinkKafkaConsumer08<T> {
 
 	/**
 	 * THIS CONSTRUCTOR IS DEPRECATED. Use FlinkKafkaConsumer08 instead.
+	 *
+	 * @deprecated Use {@link FlinkKafkaConsumer08#FlinkKafkaConsumer08(String, DeserializationSchema, Properties)}
 	 */
 	@Deprecated
 	public FlinkKafkaConsumer082(String topic, DeserializationSchema<T> valueDeserializer, Properties props) {

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer.java b/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer.java
index 1c2e0b7..a7b89f8 100644
--- a/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer.java
+++ b/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer.java
@@ -26,39 +26,59 @@ import java.util.Properties;
 
 /**
  * THIS CLASS IS DEPRECATED. Use FlinkKafkaProducer08 instead.
+ *
+ * @deprecated Use {@link FlinkKafkaProducer08}.
  */
 @Deprecated
 public class FlinkKafkaProducer<IN> extends FlinkKafkaProducer08<IN>  {
 
+	/**
+	 * @deprecated Use {@link FlinkKafkaProducer08#FlinkKafkaProducer08(String, String, SerializationSchema)}
+	 */
 	@Deprecated
 	public FlinkKafkaProducer(String brokerList, String topicId, SerializationSchema<IN> serializationSchema) {
 		super(topicId, new KeyedSerializationSchemaWrapper<>(serializationSchema), getPropertiesFromBrokerList(brokerList), null);
 	}
 
+	/**
+	 * @deprecated Use {@link FlinkKafkaProducer08#FlinkKafkaProducer08(String, SerializationSchema, Properties)}
+	 */
 	@Deprecated
 	public FlinkKafkaProducer(String topicId, SerializationSchema<IN> serializationSchema, Properties producerConfig) {
 		super(topicId, new KeyedSerializationSchemaWrapper<>(serializationSchema), producerConfig, null);
 	}
 
+	/**
+	 * @deprecated Use {@link FlinkKafkaProducer08#FlinkKafkaProducer08(String, SerializationSchema, Properties, KafkaPartitioner)}
+	 */
 	@Deprecated
 	public FlinkKafkaProducer(String topicId, SerializationSchema<IN> serializationSchema, Properties producerConfig, KafkaPartitioner customPartitioner) {
 		super(topicId, new KeyedSerializationSchemaWrapper<>(serializationSchema), producerConfig, customPartitioner);
 
 	}
 
+	/**
+	 * @deprecated Use {@link FlinkKafkaProducer08#FlinkKafkaProducer08(String, String, KeyedSerializationSchema)}
+	 */
 	@Deprecated
 	public FlinkKafkaProducer(String brokerList, String topicId, KeyedSerializationSchema<IN> serializationSchema) {
 		super(topicId, serializationSchema, getPropertiesFromBrokerList(brokerList), null);
 	}
 
+	/**
+	 * @deprecated Use {@link FlinkKafkaProducer08#FlinkKafkaProducer08(String, KeyedSerializationSchema, Properties)}
+	 */
 	@Deprecated
 	public FlinkKafkaProducer(String topicId, KeyedSerializationSchema<IN> serializationSchema, Properties producerConfig) {
 		super(topicId, serializationSchema, producerConfig, null);
 	}
 
+	/**
+	 * @deprecated Use {@link FlinkKafkaProducer08#FlinkKafkaProducer08(String, KeyedSerializationSchema, Properties, KafkaPartitioner)}
+	 */
 	@Deprecated
 	public FlinkKafkaProducer(String topicId, KeyedSerializationSchema<IN> serializationSchema, Properties producerConfig, KafkaPartitioner customPartitioner) {
 		super(topicId, serializationSchema, producerConfig, customPartitioner);
 	}
 
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
----------------------------------------------------------------------
diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
index 0407070..91d947e 100644
--- a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
+++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java
@@ -1155,6 +1155,8 @@ public class RocksDBKeyedStateBackend<K> extends AbstractKeyedStateBackend<K> {
 
 	/**
 	 * For backwards compatibility, remove again later!
+	 *
+	 * @deprecated Internal method used for backwards compatibility.
 	 */
 	@Deprecated
 	private void restoreOldSavepointKeyedState(Collection<KeyedStateHandle> restoreState) throws Exception {

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/migration/contrib/streaming/state/RocksDBStateBackend.java
----------------------------------------------------------------------
diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/migration/contrib/streaming/state/RocksDBStateBackend.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/migration/contrib/streaming/state/RocksDBStateBackend.java
index fa1cc45..e5a78b6 100644
--- a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/migration/contrib/streaming/state/RocksDBStateBackend.java
+++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/migration/contrib/streaming/state/RocksDBStateBackend.java
@@ -28,6 +28,9 @@ import java.io.IOException;
 
 import static java.util.Objects.requireNonNull;
 
+/**
+ * @deprecated Internal class used for backwards compatibility.
+ */
 @Deprecated
 public class RocksDBStateBackend extends AbstractStateBackend {
 	private static final long serialVersionUID = 1L;

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-core/src/main/java/org/apache/flink/api/common/JobExecutionResult.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/JobExecutionResult.java b/flink-core/src/main/java/org/apache/flink/api/common/JobExecutionResult.java
index 7286cc5..0c87cd0 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/JobExecutionResult.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/JobExecutionResult.java
@@ -104,6 +104,7 @@ public class JobExecutionResult extends JobSubmissionResult {
 	 * @param accumulatorName Name of the counter
 	 * @return Result of the counter, or null if the counter does not exist
 	 * @throws java.lang.ClassCastException Thrown, if the accumulator was not aggregating a {@link java.lang.Integer}
+	 * @deprecated Will be removed in future versions. Use {@link #getAccumulatorResult} instead.
 	 */
 	@Deprecated
 	@PublicEvolving

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-core/src/main/java/org/apache/flink/api/common/io/DelimitedInputFormat.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/io/DelimitedInputFormat.java b/flink-core/src/main/java/org/apache/flink/api/common/io/DelimitedInputFormat.java
index a83d45f..3dc567e 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/io/DelimitedInputFormat.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/io/DelimitedInputFormat.java
@@ -89,7 +89,7 @@ public abstract class DelimitedInputFormat<OT> extends FileInputFormat<OT> imple
 	private static int MAX_SAMPLE_LEN;
 
 	/**
-	 * @Deprecated Please use {@code loadConfigParameters(Configuration config}
+	 * @deprecated Please use {@code loadConfigParameters(Configuration config}
 	 */
 	@Deprecated
 	protected static void loadGlobalConfigParams() {

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-core/src/main/java/org/apache/flink/api/common/state/StateDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/StateDescriptor.java b/flink-core/src/main/java/org/apache/flink/api/common/state/StateDescriptor.java
index a52ea32..073d748 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/state/StateDescriptor.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/state/StateDescriptor.java
@@ -55,7 +55,17 @@ public abstract class StateDescriptor<S extends State, T> implements Serializabl
 	 */
 	// IMPORTANT: Do not change the order of the elements in this enum, ordinal is used in serialization
 	public enum Type {
-		@Deprecated UNKNOWN, VALUE, LIST, REDUCING, FOLDING, AGGREGATING, MAP
+		/**
+		 * @deprecated Enum for migrating from old checkpoints/savepoint versions.
+		 */
+		@Deprecated
+		UNKNOWN,
+		VALUE,
+		LIST,
+		REDUCING,
+		FOLDING,
+		AGGREGATING,
+		MAP
 	}
 
 	private static final long serialVersionUID = 1L;

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java b/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
index 318c7e0..ce44ab8 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
@@ -284,7 +284,7 @@ public final class ConfigConstants {
 	public static final String TASK_MANAGER_REFUSED_REGISTRATION_PAUSE = "taskmanager.refused-registration-pause";
 
 	/**
-	 * Deprecated. Please use {@link TaskManagerOptions#TASK_CANCELLATION_INTERVAL}.
+	 * @deprecated Deprecated. Please use {@link TaskManagerOptions#TASK_CANCELLATION_INTERVAL}.
 	 */
 	@PublicEvolving
 	@Deprecated
@@ -373,6 +373,8 @@ public final class ConfigConstants {
 
 	/**
 	 * Reallocate failed YARN containers.
+	 *
+	 * @deprecated Not used anymore
 	 */
 	@Deprecated
 	public static final String YARN_REALLOCATE_FAILED_CONTAINERS = "yarn.reallocate-failed";
@@ -419,9 +421,11 @@ public final class ConfigConstants {
 	@Deprecated
 	public static final String YARN_APPLICATION_MASTER_ENV_PREFIX = "yarn.application-master.env.";
 
-	// these default values are not used anymore, but remain here until Flink 2.0
+	/** @deprecated Not used anymore, but remain here until Flink 2.0 */
 	@Deprecated
 	public static final String DEFAULT_YARN_APPLICATION_MASTER_PORT = "deprecated";
+
+	/** @deprecated Not used anymore, but remain here until Flink 2.0 */
 	@Deprecated
 	public static final int DEFAULT_YARN_MIN_HEAP_CUTOFF = -1;
 
@@ -817,9 +821,7 @@ public final class ConfigConstants {
 	@Deprecated
 	public static final String FLINK_BASE_DIR_PATH_KEY = "flink.base.dir.path";
 
-	/**
-	 * @deprecated Use {@link CoreOptions#FLINK_JVM_OPTIONS} instead.
-	 */
+	/** @deprecated Use {@link CoreOptions#FLINK_JVM_OPTIONS} instead. */
 	@Deprecated
 	public static final String FLINK_JVM_OPTIONS = "env.java.opts";
 
@@ -837,15 +839,15 @@ public final class ConfigConstants {
 	@PublicEvolving
 	public static final String HA_JOB_DELAY = "high-availability.job.delay";
 
-	/** Deprecated in favour of {@link #HA_MODE}. */
+	/** @deprecated Deprecated in favour of {@link #HA_MODE}. */
 	@Deprecated
 	public static final String RECOVERY_MODE = "recovery.mode";
 
-	/** Deprecated in favour of {@link #HA_JOB_MANAGER_PORT}. */
+	/** @deprecated Deprecated in favour of {@link #HA_JOB_MANAGER_PORT}. */
 	@Deprecated
 	public static final String RECOVERY_JOB_MANAGER_PORT = "recovery.jobmanager.port";
 
-	/** Deprecated in favour of {@link #HA_JOB_DELAY}. */
+	/** @deprecated Deprecated in favour of {@link #HA_JOB_DELAY}. */
 	@Deprecated
 	public static final String RECOVERY_JOB_DELAY = "recovery.job.delay";
 
@@ -912,59 +914,59 @@ public final class ConfigConstants {
 	@PublicEvolving
 	public static final String ZOOKEEPER_SASL_SERVICE_NAME = "zookeeper.sasl.service-name";
 
-	/** Deprecated in favour of {@link #HA_ZOOKEEPER_QUORUM_KEY}. */
+	/** @deprecated Deprecated in favour of {@link #HA_ZOOKEEPER_QUORUM_KEY}. */
 	@Deprecated
 	public static final String ZOOKEEPER_QUORUM_KEY = "recovery.zookeeper.quorum";
 
-	/** Deprecated in favour of {@link #HA_ZOOKEEPER_STORAGE_PATH}. */
+	/** @deprecated Deprecated in favour of {@link #HA_ZOOKEEPER_STORAGE_PATH}. */
 	@Deprecated
 	public static final String ZOOKEEPER_RECOVERY_PATH = "recovery.zookeeper.storageDir";
 
-	/** Deprecated in favour of {@link #HA_ZOOKEEPER_DIR_KEY}. */
+	/** @deprecated Deprecated in favour of {@link #HA_ZOOKEEPER_DIR_KEY}. */
 	@Deprecated
 	public static final String ZOOKEEPER_DIR_KEY = "recovery.zookeeper.path.root";
 
-	/** Deprecated in favour of {@link #HA_ZOOKEEPER_NAMESPACE_KEY}. */
+	/** @deprecated Deprecated in favour of {@link #HA_ZOOKEEPER_NAMESPACE_KEY}. */
 	@Deprecated
 	public static final String ZOOKEEPER_NAMESPACE_KEY = "recovery.zookeeper.path.namespace";
 
-	/** Deprecated in favour of {@link #HA_ZOOKEEPER_LATCH_PATH}. */
+	/** @deprecated Deprecated in favour of {@link #HA_ZOOKEEPER_LATCH_PATH}. */
 	@Deprecated
 	public static final String ZOOKEEPER_LATCH_PATH = "recovery.zookeeper.path.latch";
 
-	/** Deprecated in favour of {@link #HA_ZOOKEEPER_LEADER_PATH}. */
+	/** @deprecated Deprecated in favour of {@link #HA_ZOOKEEPER_LEADER_PATH}. */
 	@Deprecated
 	public static final String ZOOKEEPER_LEADER_PATH = "recovery.zookeeper.path.leader";
 
-	/** Deprecated in favour of {@link #HA_ZOOKEEPER_JOBGRAPHS_PATH}. */
+	/** @deprecated Deprecated in favour of {@link #HA_ZOOKEEPER_JOBGRAPHS_PATH}. */
 	@Deprecated
 	public static final String ZOOKEEPER_JOBGRAPHS_PATH = "recovery.zookeeper.path.jobgraphs";
 
-	/** Deprecated in favour of {@link #HA_ZOOKEEPER_CHECKPOINTS_PATH}. */
+	/** @deprecated Deprecated in favour of {@link #HA_ZOOKEEPER_CHECKPOINTS_PATH}. */
 	@Deprecated
 	public static final String ZOOKEEPER_CHECKPOINTS_PATH = "recovery.zookeeper.path.checkpoints";
 
-	/** Deprecated in favour of {@link #HA_ZOOKEEPER_CHECKPOINT_COUNTER_PATH}. */
+	/** @deprecated Deprecated in favour of {@link #HA_ZOOKEEPER_CHECKPOINT_COUNTER_PATH}. */
 	@Deprecated
 	public static final String ZOOKEEPER_CHECKPOINT_COUNTER_PATH = "recovery.zookeeper.path.checkpoint-counter";
 
-	/** Deprecated in favour of {@link #HA_ZOOKEEPER_MESOS_WORKERS_PATH}. */
+	/** @deprecated Deprecated in favour of {@link #HA_ZOOKEEPER_MESOS_WORKERS_PATH}. */
 	@Deprecated
 	public static final String ZOOKEEPER_MESOS_WORKERS_PATH = "recovery.zookeeper.path.mesos-workers";
 
-	/** Deprecated in favour of {@link #HA_ZOOKEEPER_SESSION_TIMEOUT}. */
+	/** @deprecated Deprecated in favour of {@link #HA_ZOOKEEPER_SESSION_TIMEOUT}. */
 	@Deprecated
 	public static final String ZOOKEEPER_SESSION_TIMEOUT = "recovery.zookeeper.client.session-timeout";
 
-	/** Deprecated in favour of {@link #HA_ZOOKEEPER_CONNECTION_TIMEOUT}. */
+	/** @deprecated Deprecated in favour of {@link #HA_ZOOKEEPER_CONNECTION_TIMEOUT}. */
 	@Deprecated
 	public static final String ZOOKEEPER_CONNECTION_TIMEOUT = "recovery.zookeeper.client.connection-timeout";
 
-	/** Deprecated in favour of {@link #HA_ZOOKEEPER_RETRY_WAIT}. */
+	/** @deprecated Deprecated in favour of {@link #HA_ZOOKEEPER_RETRY_WAIT}. */
 	@Deprecated
 	public static final String ZOOKEEPER_RETRY_WAIT = "recovery.zookeeper.client.retry-wait";
 
-	/** Deprecated in favour of {@link #HA_ZOOKEEPER_MAX_RETRY_ATTEMPTS}. */
+	/** @deprecated Deprecated in favour of {@link #HA_ZOOKEEPER_MAX_RETRY_ATTEMPTS}. */
 	@Deprecated
 	public static final String ZOOKEEPER_MAX_RETRY_ATTEMPTS = "recovery.zookeeper.client.max-retry-attempts";
 
@@ -1038,7 +1040,7 @@ public final class ConfigConstants {
 	public static final String CHECKPOINTS_DIRECTORY_KEY = "state.checkpoints.dir";
 
 	/**
-	 * This key was used in Flink versions <= 1.1.X with the savepoint backend
+	 * @deprecated This key was used in Flink versions <= 1.1.X with the savepoint backend
 	 * configuration. We now always use the FileSystem for savepoints. For this,
 	 * the only relevant config key is {@link #SAVEPOINT_DIRECTORY_KEY}.
 	 */
@@ -1181,9 +1183,7 @@ public final class ConfigConstants {
 	 */
 	public static final boolean DEFAULT_TASK_MANAGER_MEMORY_PRE_ALLOCATE = false;
 
-	/**
-	 * Deprecated. Please use {@link TaskManagerOptions#TASK_CANCELLATION_INTERVAL}.
-	 */
+	/** @deprecated Please use {@link TaskManagerOptions#TASK_CANCELLATION_INTERVAL}. */
 	@Deprecated
 	public static final long DEFAULT_TASK_CANCELLATION_INTERVAL_MILLIS = 30000;
 
@@ -1321,7 +1321,7 @@ public final class ConfigConstants {
 	/** By default, submitting jobs from the web-frontend is allowed. */
 	public static final boolean DEFAULT_JOB_MANAGER_WEB_SUBMIT_ENABLED = true;
 
-	/** Config key has been deprecated. Therefore, no default value required. */
+	/** @deprecated Config key has been deprecated. Therefore, no default value required. */
 	@Deprecated
 	public static final boolean DEFAULT_JOB_MANAGER_WEB_CHECKPOINTS_DISABLE = false;
 
@@ -1402,7 +1402,7 @@ public final class ConfigConstants {
 	@PublicEvolving
 	public static String DEFAULT_HA_MODE = "none";
 
-	/** Deprecated in favour of {@link #DEFAULT_HA_MODE} */
+	/** @deprecated Deprecated in favour of {@link #DEFAULT_HA_MODE} */
 	@Deprecated
 	public static String DEFAULT_RECOVERY_MODE = "standalone";
 
@@ -1413,7 +1413,7 @@ public final class ConfigConstants {
 	@PublicEvolving
 	public static final String DEFAULT_HA_JOB_MANAGER_PORT = "0";
 
-	/** Deprecated in favour of {@link #DEFAULT_HA_JOB_MANAGER_PORT} */
+	/** @deprecated Deprecated in favour of {@link #DEFAULT_HA_JOB_MANAGER_PORT} */
 	@Deprecated
 	public static final String DEFAULT_RECOVERY_JOB_MANAGER_PORT = "0";
 

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-core/src/main/java/org/apache/flink/migration/util/SerializedValue.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/migration/util/SerializedValue.java b/flink-core/src/main/java/org/apache/flink/migration/util/SerializedValue.java
index bd40b83..6fa29d3 100644
--- a/flink-core/src/main/java/org/apache/flink/migration/util/SerializedValue.java
+++ b/flink-core/src/main/java/org/apache/flink/migration/util/SerializedValue.java
@@ -35,6 +35,7 @@ import java.util.Arrays;
  * provide the corresponding class loader.
  *
  * @param <T> The type of the value held.
+ * @deprecated Only used internally when migrating from previous savepoint versions.
  */
 @Deprecated
 @PublicEvolving

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-java/src/main/java/org/apache/flink/api/java/io/CsvReader.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/io/CsvReader.java b/flink-java/src/main/java/org/apache/flink/api/java/io/CsvReader.java
index cbac386..ce2f4fa 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/io/CsvReader.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/io/CsvReader.java
@@ -108,6 +108,8 @@ public class CsvReader {
 	 * 
 	 * @param delimiter The delimiter that separates the fields in one row.
 	 * @return The CSV reader instance itself, to allow for fluent function chaining.
+	 *
+	 * @deprecated Please use {@link #fieldDelimiter(String)}.
 	 */
 	@Deprecated
 	@PublicEvolving

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-java/src/main/java/org/apache/flink/api/java/operators/CrossOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/CrossOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/CrossOperator.java
index 3fdc51d..98956bb 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/CrossOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/CrossOperator.java
@@ -301,7 +301,7 @@ public class CrossOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OUT,
 		}
 
 		/**
-		 * Deprecated method only kept for compatibility.
+		 * @deprecated Deprecated method only kept for compatibility.
 		 */
 		@SuppressWarnings({ "hiding", "unchecked" })
 		@Deprecated

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-java/src/main/java/org/apache/flink/api/java/operators/DataSink.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/DataSink.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/DataSink.java
index fd8190c..ee5ab2e 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/DataSink.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/DataSink.java
@@ -117,6 +117,8 @@ public class DataSink<T> {
 	 *
 	 * @see org.apache.flink.api.java.tuple.Tuple
 	 * @see Order
+	 *
+	 * @deprecated Use {@link DataSet#sortPartition(int, Order)} instead
 	 */
 	@Deprecated
 	@PublicEvolving
@@ -164,6 +166,8 @@ public class DataSink<T> {
 	 * @return This data sink operator with specified output order.
 	 *
 	 * @see Order
+	 *
+	 * @deprecated Use {@link DataSet#sortPartition(String, Order)} instead
 	 */
 	@Deprecated
 	@PublicEvolving

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-java/src/main/java/org/apache/flink/api/java/operators/JoinOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/JoinOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/JoinOperator.java
index 11f6c82..86ee2a2 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/JoinOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/JoinOperator.java
@@ -735,7 +735,7 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
 		}
 
 		/**
-		 * Deprecated method only kept for compatibility.
+		 * @deprecated Deprecated method only kept for compatibility.
 		 *
 		 * @param types
 		 */

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-java/src/main/java/org/apache/flink/api/java/operators/ProjectOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/ProjectOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/ProjectOperator.java
index 6c27167..0068582 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/ProjectOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/ProjectOperator.java
@@ -71,7 +71,7 @@ public class ProjectOperator<IN, OUT extends Tuple>
 		return ppo;
 	}
 	/**
-	 * Deprecated method only kept for compatibility.
+	 * @deprecated Deprecated method only kept for compatibility.
 	 */
 	@SuppressWarnings("unchecked")
 	@Deprecated

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-libraries/flink-table/src/main/java/org/apache/flink/table/calcite/rules/FlinkAggregateExpandDistinctAggregatesRule.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/java/org/apache/flink/table/calcite/rules/FlinkAggregateExpandDistinctAggregatesRule.java b/flink-libraries/flink-table/src/main/java/org/apache/flink/table/calcite/rules/FlinkAggregateExpandDistinctAggregatesRule.java
index d7b1ffa..9d4e08e 100644
--- a/flink-libraries/flink-table/src/main/java/org/apache/flink/table/calcite/rules/FlinkAggregateExpandDistinctAggregatesRule.java
+++ b/flink-libraries/flink-table/src/main/java/org/apache/flink/table/calcite/rules/FlinkAggregateExpandDistinctAggregatesRule.java
@@ -114,7 +114,10 @@ public final class FlinkAggregateExpandDistinctAggregatesRule extends RelOptRule
 		this.useGroupingSets = useGroupingSets;
 	}
 
-	@Deprecated // to be removed before 2.0
+	/**
+	 * @deprecated to be removed before 2.0
+	 */
+	@Deprecated
 	public FlinkAggregateExpandDistinctAggregatesRule(
 			Class<? extends LogicalAggregate> clazz,
 			boolean useGroupingSets,
@@ -122,7 +125,10 @@ public final class FlinkAggregateExpandDistinctAggregatesRule extends RelOptRule
 		this(clazz, useGroupingSets, RelBuilder.proto(Contexts.of(joinFactory)));
 	}
 
-	@Deprecated // to be removed before 2.0
+	/**
+	 * @deprecated to be removed before 2.0
+	 */
+	@Deprecated
 	public FlinkAggregateExpandDistinctAggregatesRule(
 			Class<? extends LogicalAggregate> clazz,
 			RelFactories.JoinFactory joinFactory) {

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-libraries/flink-table/src/main/java/org/apache/flink/table/calcite/rules/FlinkAggregateJoinTransposeRule.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/java/org/apache/flink/table/calcite/rules/FlinkAggregateJoinTransposeRule.java b/flink-libraries/flink-table/src/main/java/org/apache/flink/table/calcite/rules/FlinkAggregateJoinTransposeRule.java
index ac36b3c..a817c91 100644
--- a/flink-libraries/flink-table/src/main/java/org/apache/flink/table/calcite/rules/FlinkAggregateJoinTransposeRule.java
+++ b/flink-libraries/flink-table/src/main/java/org/apache/flink/table/calcite/rules/FlinkAggregateJoinTransposeRule.java
@@ -78,22 +78,34 @@ public class FlinkAggregateJoinTransposeRule extends RelOptRule {
 		this.allowFunctions = allowFunctions;
 	}
 
-	@Deprecated // to be removed before 2.0
+	/**
+	 * @deprecated to be removed before 2.0
+	 */
+	@Deprecated
 	public FlinkAggregateJoinTransposeRule(Class<? extends Aggregate> aggregateClass, RelFactories.AggregateFactory aggregateFactory, Class<? extends Join> joinClass, RelFactories.JoinFactory joinFactory) {
 		this(aggregateClass, joinClass, RelBuilder.proto(aggregateFactory, joinFactory), false);
 	}
 
-	@Deprecated // to be removed before 2.0
+	/**
+	 * @deprecated to be removed before 2.0
+	 */
+	@Deprecated
 	public FlinkAggregateJoinTransposeRule(Class<? extends Aggregate> aggregateClass, RelFactories.AggregateFactory aggregateFactory, Class<? extends Join> joinClass, RelFactories.JoinFactory joinFactory, boolean allowFunctions) {
 		this(aggregateClass, joinClass, RelBuilder.proto(aggregateFactory, joinFactory), allowFunctions);
 	}
 
-	@Deprecated // to be removed before 2.0
+	/**
+	 * @deprecated to be removed before 2.0
+	 */
+	@Deprecated
 	public FlinkAggregateJoinTransposeRule(Class<? extends Aggregate> aggregateClass, RelFactories.AggregateFactory aggregateFactory, Class<? extends Join> joinClass, RelFactories.JoinFactory joinFactory, RelFactories.ProjectFactory projectFactory) {
 		this(aggregateClass, joinClass, RelBuilder.proto(aggregateFactory, joinFactory, projectFactory), false);
 	}
 
-	@Deprecated // to be removed before 2.0
+	/**
+	 * @deprecated to be removed before 2.0
+	 */
+	@Deprecated
 	public FlinkAggregateJoinTransposeRule(Class<? extends Aggregate> aggregateClass, RelFactories.AggregateFactory aggregateFactory, Class<? extends Join> joinClass, RelFactories.JoinFactory joinFactory, RelFactories.ProjectFactory projectFactory, boolean allowFunctions) {
 		this(aggregateClass, joinClass, RelBuilder.proto(aggregateFactory, joinFactory, projectFactory), allowFunctions);
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/migration/MigrationNamespaceSerializerProxy.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/MigrationNamespaceSerializerProxy.java b/flink-runtime/src/main/java/org/apache/flink/migration/MigrationNamespaceSerializerProxy.java
index 91fa633..c6813b6 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/MigrationNamespaceSerializerProxy.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/MigrationNamespaceSerializerProxy.java
@@ -30,6 +30,8 @@ import java.io.Serializable;
  * Flink 1.1 savepoint (which did not include the namespace serializer) to Flink 1.2 (which always must include a
  * (non-null) namespace serializer. This is then replaced as soon as the user is re-registering her state again for
  * the first run under Flink 1.2 and provides again the real namespace serializer.
+ *
+ * @deprecated Internal class for savepoint backwards compatibility. Don't use for other purposes.
  */
 @Deprecated
 @SuppressWarnings("deprecation")
@@ -114,4 +116,4 @@ public class MigrationNamespaceSerializerProxy extends TypeSerializer<Serializab
 	public int hashCode() {
 		return 42;
 	}
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/migration/api/common/state/ListStateDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/api/common/state/ListStateDescriptor.java b/flink-runtime/src/main/java/org/apache/flink/migration/api/common/state/ListStateDescriptor.java
index 1847c08..5196d2d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/api/common/state/ListStateDescriptor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/api/common/state/ListStateDescriptor.java
@@ -28,6 +28,8 @@ import org.apache.flink.api.common.typeutils.TypeSerializer;
 /**
  * The old version of the {@link org.apache.flink.api.common.state.ListStateDescriptor}, retained for
  * serialization backwards compatibility.
+ *
+ * @deprecated Internal class for savepoint backwards compatibility. Don't use for other purposes.
  */
 @Internal
 @Deprecated

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/migration/runtime/checkpoint/KeyGroupState.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/checkpoint/KeyGroupState.java b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/checkpoint/KeyGroupState.java
index 483a403..0b25e08 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/checkpoint/KeyGroupState.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/checkpoint/KeyGroupState.java
@@ -30,6 +30,8 @@ import java.io.Serializable;
  *
  * The key group state handle is kept in serialized form because it can contain user code classes
  * which might not be available on the JobManager.
+ *
+ * @deprecated Internal class for savepoint backwards compatibility. Don't use for other purposes.
  */
 @Deprecated
 @SuppressWarnings("deprecation")

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/migration/runtime/checkpoint/SubtaskState.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/checkpoint/SubtaskState.java b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/checkpoint/SubtaskState.java
index 7e84fdc..d42d146 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/checkpoint/SubtaskState.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/checkpoint/SubtaskState.java
@@ -25,6 +25,9 @@ import java.io.Serializable;
 
 import static org.apache.flink.util.Preconditions.checkNotNull;
 
+/**
+ * @deprecated Internal class for savepoint backwards compatibility. Don't use for other purposes.
+ */
 @Deprecated
 @SuppressWarnings("deprecation")
 public class SubtaskState implements Serializable {

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/migration/runtime/checkpoint/TaskState.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/checkpoint/TaskState.java b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/checkpoint/TaskState.java
index 91d8ba0..c0a7b2d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/checkpoint/TaskState.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/checkpoint/TaskState.java
@@ -29,6 +29,9 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 
+/**
+ * @deprecated Internal class for savepoint backwards compatibility. Don't use for other purposes.
+ */
 @Deprecated
 @SuppressWarnings("deprecation")
 public class TaskState implements Serializable {

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/AbstractCloseableHandle.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/AbstractCloseableHandle.java b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/AbstractCloseableHandle.java
index ad75dae..775b304 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/AbstractCloseableHandle.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/AbstractCloseableHandle.java
@@ -28,6 +28,8 @@ import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
  * 
  * Offers to register a stream (or other closable object) that close calls are delegated to if
  * the handle is closed or was already closed.
+ *
+ * @deprecated Internal class for savepoint backwards compatibility. Don't use for other purposes.
  */
 @Deprecated
 @SuppressWarnings("deprecation")

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/AbstractStateBackend.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/AbstractStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/AbstractStateBackend.java
index ff97e19..7c53c40 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/AbstractStateBackend.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/AbstractStateBackend.java
@@ -26,6 +26,8 @@ import java.io.Serializable;
 
 /**
  * A state backend defines how state is stored and snapshotted during checkpoints.
+ *
+ * @deprecated Internal class for savepoint backwards compatibility. Don't use for other purposes.
  */
 @Deprecated
 @SuppressWarnings("deprecation")

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/KvStateSnapshot.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/KvStateSnapshot.java b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/KvStateSnapshot.java
index 7aecd31..687d415 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/KvStateSnapshot.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/KvStateSnapshot.java
@@ -21,6 +21,9 @@ package org.apache.flink.migration.runtime.state;
 import org.apache.flink.api.common.state.State;
 import org.apache.flink.api.common.state.StateDescriptor;
 
+/**
+ * @deprecated Internal class for savepoint backwards compatibility. Don't use for other purposes.
+ */
 @Deprecated
 @SuppressWarnings("deprecation")
 public interface KvStateSnapshot<K, N, S extends State, SD extends StateDescriptor<S, ?>>

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/StateHandle.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/StateHandle.java b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/StateHandle.java
index 8cdeefa..fd3917f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/StateHandle.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/StateHandle.java
@@ -22,6 +22,8 @@ package org.apache.flink.migration.runtime.state;
  * StateHandle is a general handle interface meant to abstract operator state fetching. 
  * A StateHandle implementation can for example include the state itself in cases where the state 
  * is lightweight or fetching it lazily from some external storage when the state is too large.
+ *
+ * @deprecated Internal class for savepoint backwards compatibility. Don't use for other purposes.
  */
 @Deprecated
 @SuppressWarnings("deprecation")

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/StateObject.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/StateObject.java b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/StateObject.java
index 128a074..59bc0ca 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/StateObject.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/StateObject.java
@@ -32,6 +32,8 @@ package org.apache.flink.migration.runtime.state;
  *         stop the current access or recovery to the state. Called for example when an operation is
  *         canceled during recovery.</li>
  * </ul>
+ *
+ * @deprecated Internal class for savepoint backwards compatibility. Don't use for other purposes.
  */
 @Deprecated
 @SuppressWarnings("deprecation")

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/StreamStateHandle.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/StreamStateHandle.java b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/StreamStateHandle.java
index ee6be97..bfc57bc 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/StreamStateHandle.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/StreamStateHandle.java
@@ -23,6 +23,8 @@ import java.io.Serializable;
 
 /**
  * A state handle that produces an input stream when resolved.
+ *
+ * @deprecated Internal class for savepoint backwards compatibility. Don't use for other purposes.
  */
 @Deprecated
 @SuppressWarnings("deprecation")

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/AbstractFileStateHandle.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/AbstractFileStateHandle.java b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/AbstractFileStateHandle.java
index 3d77064..a522a95 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/AbstractFileStateHandle.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/AbstractFileStateHandle.java
@@ -30,6 +30,8 @@ import static org.apache.flink.util.Preconditions.checkNotNull;
 
 /**
  * Base class for state that is stored in a file.
+ *
+ * @deprecated Internal class for savepoint backwards compatibility. Don't use for other purposes.
  */
 @Deprecated
 @SuppressWarnings("deprecation")

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/AbstractFsStateSnapshot.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/AbstractFsStateSnapshot.java b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/AbstractFsStateSnapshot.java
index a15e49d..7099c61 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/AbstractFsStateSnapshot.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/AbstractFsStateSnapshot.java
@@ -40,6 +40,8 @@ import java.io.IOException;
  * @param <K> The type of the key in the snapshot state.
  * @param <N> The type of the namespace in the snapshot state.
  * @param <SV> The type of the state value.
+ *
+ * @deprecated Internal class for savepoint backwards compatibility. Don't use for other purposes.
  */
 @Deprecated
 @SuppressWarnings("deprecation")

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FileSerializableStateHandle.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FileSerializableStateHandle.java b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FileSerializableStateHandle.java
index bee5ded..b4a3a73 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FileSerializableStateHandle.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FileSerializableStateHandle.java
@@ -31,6 +31,8 @@ import java.io.Serializable;
  * A state handle that points to state stored in a file via Java Serialization.
  * 
  * @param <T> The type of state pointed to by the state handle.
+ *
+ * @deprecated Internal class for savepoint backwards compatibility. Don't use for other purposes.
  */
 @Deprecated
 @SuppressWarnings("deprecation")

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FileStreamStateHandle.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FileStreamStateHandle.java b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FileStreamStateHandle.java
index e44b823..7444be1 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FileStreamStateHandle.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FileStreamStateHandle.java
@@ -28,6 +28,8 @@ import java.io.Serializable;
 
 /**
  * A state handle that points to state in a file system, accessible as an input stream.
+ *
+ * @deprecated Internal class for savepoint backwards compatibility. Don't use for other purposes.
  */
 @Deprecated
 @SuppressWarnings("deprecation")

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FsFoldingState.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FsFoldingState.java b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FsFoldingState.java
index 6e98a30..ec89ab8 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FsFoldingState.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FsFoldingState.java
@@ -23,6 +23,9 @@ import org.apache.flink.api.common.state.FoldingStateDescriptor;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.core.fs.Path;
 
+/**
+ * @deprecated Internal class for savepoint backwards compatibility. Don't use for other purposes.
+ */
 @Deprecated
 @SuppressWarnings("deprecation")
 public class FsFoldingState<K, N, T, ACC> {

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FsListState.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FsListState.java b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FsListState.java
index e8f7a9d..71404ab 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FsListState.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FsListState.java
@@ -25,6 +25,9 @@ import org.apache.flink.core.fs.Path;
 
 import java.util.ArrayList;
 
+/**
+ * @deprecated Internal class for savepoint backwards compatibility. Don't use for other purposes.
+ */
 @Deprecated
 @SuppressWarnings("deprecation")
 public class FsListState<K, N, V> {

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FsReducingState.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FsReducingState.java b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FsReducingState.java
index 8174871..153f88c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FsReducingState.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FsReducingState.java
@@ -23,6 +23,9 @@ import org.apache.flink.api.common.state.ReducingStateDescriptor;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.core.fs.Path;
 
+/**
+ * @deprecated Internal class for savepoint backwards compatibility. Don't use for other purposes.
+ */
 @Deprecated
 @SuppressWarnings("deprecation")
 public class FsReducingState<K, N, V> {

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FsStateBackend.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FsStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FsStateBackend.java
index fbdd393..d177510 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FsStateBackend.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FsStateBackend.java
@@ -23,6 +23,9 @@ import org.apache.flink.migration.runtime.state.AbstractStateBackend;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+/**
+ * @deprecated Internal class for savepoint backwards compatibility. Don't use for other purposes.
+ */
 @Deprecated
 @SuppressWarnings("deprecation")
 public class FsStateBackend extends AbstractStateBackend {

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FsValueState.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FsValueState.java b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FsValueState.java
index b185f88..d2ae48d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FsValueState.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/filesystem/FsValueState.java
@@ -23,6 +23,9 @@ import org.apache.flink.api.common.state.ValueStateDescriptor;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.core.fs.Path;
 
+/**
+ * @deprecated Internal class for savepoint backwards compatibility. Don't use for other purposes.
+ */
 @Deprecated
 @SuppressWarnings("deprecation")
 public class FsValueState<K, N, V> {

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/AbstractMemStateSnapshot.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/AbstractMemStateSnapshot.java b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/AbstractMemStateSnapshot.java
index ff86f7c..aadfe4e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/AbstractMemStateSnapshot.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/AbstractMemStateSnapshot.java
@@ -29,6 +29,9 @@ import org.apache.flink.runtime.util.DataInputDeserializer;
 
 import java.io.IOException;
 
+/**
+ * @deprecated Internal class for savepoint backwards compatibility. Don't use for other purposes.
+ */
 @Deprecated
 @SuppressWarnings("deprecation")
 public abstract class AbstractMemStateSnapshot<K, N, SV, S extends State, SD extends StateDescriptor<S, ?>> 

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/AbstractMigrationRestoreStrategy.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/AbstractMigrationRestoreStrategy.java b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/AbstractMigrationRestoreStrategy.java
index e572619..f47989a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/AbstractMigrationRestoreStrategy.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/AbstractMigrationRestoreStrategy.java
@@ -38,6 +38,8 @@ import java.io.IOException;
  * @param <K> type of key.
  * @param <N> type of namespace.
  * @param <S> type of state.
+ *
+ * @deprecated Internal class for savepoint backwards compatibility. Don't use for other purposes.
  */
 @Deprecated
 public abstract class AbstractMigrationRestoreStrategy<K, N, S> implements MigrationRestoreSnapshot<K, N, S> {
@@ -114,4 +116,4 @@ public abstract class AbstractMigrationRestoreStrategy<K, N, S> implements Migra
 	 * Different state handles require different code to end up with a {@link DataInputView}.
 	 */
 	protected abstract DataInputView openDataInputView() throws IOException;
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/ByteStreamStateHandle.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/ByteStreamStateHandle.java b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/ByteStreamStateHandle.java
index 45785bc..c7fbab6 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/ByteStreamStateHandle.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/ByteStreamStateHandle.java
@@ -27,6 +27,9 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.Serializable;
 
+/**
+ * @deprecated Internal class for savepoint backwards compatibility. Don't use for other purposes.
+ */
 @Deprecated
 @SuppressWarnings("deprecation")
 public final class ByteStreamStateHandle extends AbstractCloseableHandle implements StreamStateHandle {

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/MemFoldingState.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/MemFoldingState.java b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/MemFoldingState.java
index 0956ddd..ad820e4 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/MemFoldingState.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/MemFoldingState.java
@@ -22,6 +22,9 @@ import org.apache.flink.api.common.state.FoldingState;
 import org.apache.flink.api.common.state.FoldingStateDescriptor;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 
+/**
+ * @deprecated Internal class for savepoint backwards compatibility. Don't use for other purposes.
+ */
 @Deprecated
 @SuppressWarnings("deprecation")
 public class MemFoldingState<K, N, T, ACC> {

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/MemListState.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/MemListState.java b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/MemListState.java
index 3ff7c50..d76cda0 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/MemListState.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/MemListState.java
@@ -24,6 +24,9 @@ import org.apache.flink.api.common.typeutils.TypeSerializer;
 
 import java.util.ArrayList;
 
+/**
+ * @deprecated Internal class for savepoint backwards compatibility. Don't use for other purposes.
+ */
 @Deprecated
 @SuppressWarnings("deprecation")
 public class MemListState<K, N, V> {

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/MemReducingState.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/MemReducingState.java b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/MemReducingState.java
index e6154fb..c39111c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/MemReducingState.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/MemReducingState.java
@@ -29,6 +29,8 @@ import org.apache.flink.api.common.typeutils.TypeSerializer;
  * @param <K> The type of the key.
  * @param <N> The type of the namespace.
  * @param <V> The type of the values in the list state.
+ *
+ * @deprecated Internal class for savepoint backwards compatibility. Don't use for other purposes.
  */
 @Deprecated
 @SuppressWarnings("deprecation")

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/MemValueState.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/MemValueState.java b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/MemValueState.java
index 777b0f6..940d489 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/MemValueState.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/MemValueState.java
@@ -28,6 +28,8 @@ import org.apache.flink.api.common.typeutils.TypeSerializer;
  * @param <K> The type of the key.
  * @param <N> The type of the namespace.
  * @param <V> The type of the value.
+ *
+ * @deprecated Internal class for savepoint backwards compatibility. Don't use for other purposes.
  */
 @Deprecated
 @SuppressWarnings("deprecation")

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/MigrationRestoreSnapshot.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/MigrationRestoreSnapshot.java b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/MigrationRestoreSnapshot.java
index ea529db..4277b56 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/MigrationRestoreSnapshot.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/runtime/state/memory/MigrationRestoreSnapshot.java
@@ -25,6 +25,9 @@ import org.apache.flink.util.Migration;
 
 import java.io.IOException;
 
+/**
+ * @deprecated Internal class for savepoint backwards compatibility. Don't use for other purposes.
+ */
 @Deprecated
 @Internal
 public interface MigrationRestoreSnapshot<K, N, S> extends Migration {

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/migration/state/MigrationKeyGroupStateHandle.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/state/MigrationKeyGroupStateHandle.java b/flink-runtime/src/main/java/org/apache/flink/migration/state/MigrationKeyGroupStateHandle.java
index ee15b83..3f1ff55 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/state/MigrationKeyGroupStateHandle.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/state/MigrationKeyGroupStateHandle.java
@@ -27,6 +27,8 @@ import org.apache.flink.util.Migration;
 /**
  * This class is just a KeyGroupsStateHandle that is tagged as migration, to figure out which restore logic to apply,
  * e.g. when restoring backend data from a state handle.
+ *
+ * @deprecated Internal class for savepoint backwards compatibility. Don't use for other purposes.
  */
 @Internal
 @Deprecated

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/migration/state/MigrationStreamStateHandle.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/state/MigrationStreamStateHandle.java b/flink-runtime/src/main/java/org/apache/flink/migration/state/MigrationStreamStateHandle.java
index 797d5d2..2201916 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/state/MigrationStreamStateHandle.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/state/MigrationStreamStateHandle.java
@@ -29,6 +29,8 @@ import java.io.IOException;
 /**
  * This class is just a StreamStateHandle that is tagged as migration, to figure out which restore logic to apply, e.g.
  * when restoring backend data from a state handle.
+ *
+ * @deprecated Internal class for savepoint backwards compatibility. Don't use for other purposes.
  */
 @Internal
 @Deprecated

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/migration/streaming/runtime/tasks/StreamTaskState.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/streaming/runtime/tasks/StreamTaskState.java b/flink-runtime/src/main/java/org/apache/flink/migration/streaming/runtime/tasks/StreamTaskState.java
index a6b6e3f..7a2aab9 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/streaming/runtime/tasks/StreamTaskState.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/streaming/runtime/tasks/StreamTaskState.java
@@ -27,6 +27,9 @@ import java.io.IOException;
 import java.io.Serializable;
 import java.util.HashMap;
 
+/**
+ * @deprecated Internal class for savepoint backwards compatibility. Don't use for other purposes.
+ */
 @Deprecated
 @Internal
 @SuppressWarnings("deprecation")

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/migration/streaming/runtime/tasks/StreamTaskStateList.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/migration/streaming/runtime/tasks/StreamTaskStateList.java b/flink-runtime/src/main/java/org/apache/flink/migration/streaming/runtime/tasks/StreamTaskStateList.java
index 07ce44b..c2357f7 100644
--- a/flink-runtime/src/main/java/org/apache/flink/migration/streaming/runtime/tasks/StreamTaskStateList.java
+++ b/flink-runtime/src/main/java/org/apache/flink/migration/streaming/runtime/tasks/StreamTaskStateList.java
@@ -25,6 +25,9 @@ import org.apache.flink.migration.runtime.state.StateHandle;
 import java.io.IOException;
 import java.util.HashMap;
 
+/**
+ * @deprecated Internal class for savepoint backwards compatibility. Don't use for other purposes.
+ */
 @Deprecated
 @Internal
 @SuppressWarnings("deprecation")

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/SubtaskState.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/SubtaskState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/SubtaskState.java
index 9e195b1..97b08fc 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/SubtaskState.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/SubtaskState.java
@@ -39,6 +39,9 @@ public class SubtaskState implements StateObject {
 
 	/**
 	 * Legacy (non-repartitionable) operator state.
+	 *
+	 * @deprecated Non-repartitionable operator state that has been deprecated.
+	 * Can be removed when we remove the APIs for non-repartitionable operator state.
 	 */
 	@Deprecated
 	private final ChainedStateHandle<StreamStateHandle> legacyOperatorState;
@@ -101,6 +104,10 @@ public class SubtaskState implements StateObject {
 
 	// --------------------------------------------------------------------------------------------
 
+	/**
+	 * @deprecated Non-repartitionable operator state that has been deprecated.
+	 * Can be removed when we remove the APIs for non-repartitionable operator state.
+	 */
 	@Deprecated
 	public ChainedStateHandle<StreamStateHandle> getLegacyOperatorState() {
 		return legacyOperatorState;

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/runtime/leaderretrieval/StandaloneLeaderRetrievalService.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/leaderretrieval/StandaloneLeaderRetrievalService.java b/flink-runtime/src/main/java/org/apache/flink/runtime/leaderretrieval/StandaloneLeaderRetrievalService.java
index 174e106..fefe910 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/leaderretrieval/StandaloneLeaderRetrievalService.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/leaderretrieval/StandaloneLeaderRetrievalService.java
@@ -52,6 +52,7 @@ public class StandaloneLeaderRetrievalService implements LeaderRetrievalService
 	 * The leaderId will be null.
 	 *
 	 * @param leaderAddress The leader's pre-configured address
+	 * @deprecated Use {@link #StandaloneLeaderRetrievalService(String, UUID)} instead
 	 */
 	@Deprecated
 	public StandaloneLeaderRetrievalService(String leaderAddress) {

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/InMemoryPartition.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/InMemoryPartition.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/InMemoryPartition.java
index 7554bc3..7d70246 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/InMemoryPartition.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/InMemoryPartition.java
@@ -248,6 +248,8 @@ public class InMemoryPartition<T> {
 	 * @param pointer pointer to start of record
 	 * @param record record to overwrite old one with
 	 * @throws IOException
+	 * @deprecated Don't use this, overwrites record and causes inconsistency or data loss for
+	 * overwriting everything but records of the exact same size
 	 */
 	@Deprecated
 	public void overwriteRecordAt(long pointer, T record) throws IOException {

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskStateHandles.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskStateHandles.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskStateHandles.java
index 450413a..2fde548 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskStateHandles.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskStateHandles.java
@@ -35,7 +35,12 @@ public class TaskStateHandles implements Serializable {
 
 	private static final long serialVersionUID = 267686583583579359L;
 
-	/** State handle with the (non-partitionable) legacy operator state*/
+	/**
+	 * State handle with the (non-partitionable) legacy operator state
+	 *
+	 * @deprecated Non-repartitionable operator state that has been deprecated.
+	 * Can be removed when we remove the APIs for non-repartitionable operator state.
+	 */
 	@Deprecated
 	private final ChainedStateHandle<StreamStateHandle> legacyOperatorState;
 
@@ -77,6 +82,10 @@ public class TaskStateHandles implements Serializable {
 		this.rawOperatorState = rawOperatorState;
 	}
 
+	/**
+	 * @deprecated Non-repartitionable operator state that has been deprecated.
+	 * Can be removed when we remove the APIs for non-repartitionable operator state.
+	 */
 	@Deprecated
 	public ChainedStateHandle<StreamStateHandle> getLegacyOperatorState() {
 		return legacyOperatorState;

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java
index a332d7d..9247ffa 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java
@@ -459,6 +459,9 @@ public class HeapKeyedStateBackend<K> extends AbstractKeyedStateBackend<K> {
 		return "HeapKeyedStateBackend";
 	}
 
+	/**
+	 * @deprecated Used for backwards compatibility with previous savepoint versions.
+	 */
 	@SuppressWarnings({"unchecked", "rawtypes", "DeprecatedIsStillUsed"})
 	@Deprecated
 	private void restoreOldSavepointKeyedState(
@@ -536,4 +539,4 @@ public class HeapKeyedStateBackend<K> extends AbstractKeyedStateBackend<K> {
 	public boolean supportsAsynchronousSnapshots() {
 		return asynchronousSnapshots;
 	}
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedRestoring.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedRestoring.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedRestoring.java
index c0dd361..1fd652f 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedRestoring.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedRestoring.java
@@ -25,6 +25,8 @@ import java.io.Serializable;
 /**
  * This deprecated interface contains the methods for restoring from the legacy checkpointing mechanism of state.
  * @param <T> type of the restored state.
+ *
+ * @deprecated Please use {@link CheckpointedFunction} or {@link ListCheckpointed} after restoring your legacy state.
  */
 @Deprecated
 @PublicEvolving

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
index 70807fb..88db04e 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
@@ -349,6 +349,9 @@ public abstract class StreamExecutionEnvironment {
 	 *            The checkpointing mode, selecting between "exactly once" and "at least once" guaranteed.
 	 * @param force
 	 *            If true checkpointing will be enabled for iterative jobs as well.
+	 *
+	 * @deprecated Use {@link #enableCheckpointing(long, CheckpointingMode)} instead.
+	 * Forcing checkpoints will be removed in the future.
 	 */
 	@Deprecated
 	@SuppressWarnings("deprecation")
@@ -396,6 +399,8 @@ public abstract class StreamExecutionEnvironment {
 
 	/**
 	 * Returns whether checkpointing is force-enabled.
+	 *
+	 * @deprecated Forcing checkpoints will be removed in future version.
 	 */
 	@Deprecated
 	@SuppressWarnings("deprecation")

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/AscendingTimestampExtractor.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/AscendingTimestampExtractor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/AscendingTimestampExtractor.java
index 60aedbd..d13e461 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/AscendingTimestampExtractor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/AscendingTimestampExtractor.java
@@ -29,6 +29,8 @@ import org.apache.flink.annotation.PublicEvolving;
  * {@link org.apache.flink.streaming.api.functions.timestamps.AscendingTimestampExtractor}.
  *
  * @param <T> The type of the elements that this function can extract timestamps from
+ *
+ * @deprecated Extend {@link org.apache.flink.streaming.api.functions.timestamps.AscendingTimestampExtractor} instead.
  */
 @PublicEvolving
 @Deprecated

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FileMonitoringFunction.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FileMonitoringFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FileMonitoringFunction.java
index 06da8c1..ff941f1 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FileMonitoringFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FileMonitoringFunction.java
@@ -17,6 +17,7 @@
 
 package org.apache.flink.streaming.api.functions.source;
 
+import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.java.tuple.Tuple3;
 import org.apache.flink.core.fs.FileStatus;
 import org.apache.flink.core.fs.FileSystem;
@@ -31,6 +32,10 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+/**
+ * @deprecated Internal class deprecated in favour of {@link ContinuousFileMonitoringFunction}.
+ */
+@Internal
 @Deprecated
 public class FileMonitoringFunction implements SourceFunction<Tuple3<String, Long, Long>> {
 	private static final long serialVersionUID = 1L;

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FileReadFunction.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FileReadFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FileReadFunction.java
index ac1e834..ff8a295 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FileReadFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/FileReadFunction.java
@@ -21,6 +21,7 @@ import java.io.BufferedReader;
 import java.io.InputStreamReader;
 import java.net.URI;
 
+import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.functions.FlatMapFunction;
 import org.apache.flink.api.java.tuple.Tuple3;
 import org.apache.flink.core.fs.FSDataInputStream;
@@ -28,6 +29,10 @@ import org.apache.flink.core.fs.FileSystem;
 import org.apache.flink.core.fs.Path;
 import org.apache.flink.util.Collector;
 
+/**
+ * @deprecated Internal class deprecated in favour of {@link ContinuousFileMonitoringFunction}.
+ */
+@Internal
 @Deprecated
 public class FileReadFunction implements FlatMapFunction<Tuple3<String, Long, Long>, String> {
 

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
index a6a89b5..cc81a0e 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
@@ -249,6 +249,10 @@ public abstract class AbstractStreamOperator<OUT>
 		}
 	}
 
+	/**
+	 * @deprecated Non-repartitionable operator state that has been deprecated.
+	 * Can be removed when we remove the APIs for non-repartitionable operator state.
+	 */
 	@Deprecated
 	private void restoreStreamCheckpointed(OperatorStateHandles stateHandles) throws Exception {
 		StreamStateHandle state = stateHandles.getLegacyOperatorState();
@@ -436,6 +440,10 @@ public abstract class AbstractStreamOperator<OUT>
 		}
 	}
 
+	/**
+	 * @deprecated Non-repartitionable operator state that has been deprecated.
+	 * Can be removed when we remove the APIs for non-repartitionable operator state.
+	 */
 	@SuppressWarnings("deprecation")
 	@Deprecated
 	@Override

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/CheckpointedRestoringOperator.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/CheckpointedRestoringOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/CheckpointedRestoringOperator.java
index 20eb1cf..33304e4 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/CheckpointedRestoringOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/CheckpointedRestoringOperator.java
@@ -19,12 +19,16 @@
 package org.apache.flink.streaming.api.operators;
 
 import org.apache.flink.core.fs.FSDataInputStream;
+import org.apache.flink.streaming.api.checkpoint.Checkpointed;
 import org.apache.flink.streaming.api.graph.StreamConfig;
 import org.apache.flink.streaming.runtime.tasks.StreamTask;
 
 /**
  * Interface for {@link StreamOperator StreamOperators} that can restore from a Flink 1.1
  * legacy snapshot that was done using the {@link StreamCheckpointedOperator} interface.
+ *
+ * @deprecated {@link Checkpointed} has been deprecated as well. This class can be
+ * removed when we remove that interface.
  */
 @Deprecated
 public interface CheckpointedRestoringOperator {

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamCheckpointedOperator.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamCheckpointedOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamCheckpointedOperator.java
index a28cdc4..986e2b7 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamCheckpointedOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamCheckpointedOperator.java
@@ -21,7 +21,7 @@ package org.apache.flink.streaming.api.operators;
 import org.apache.flink.core.fs.FSDataOutputStream;
 
 /**
- * This interface is deprecated without replacement.
+ * @deprecated This interface is deprecated without replacement.
  * All operators are now checkpointed.
  */
 @Deprecated

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/Trigger.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/Trigger.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/Trigger.java
index 11a0d6d..3d11a44 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/Trigger.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/Trigger.java
@@ -201,6 +201,7 @@ public abstract class Trigger<T, W extends Window> implements Serializable {
 		 * @return The partitioned state object.
 		 * @throws UnsupportedOperationException Thrown, if no partitioned state is available for the
 		 *                                       function (function is not part os a KeyedStream).
+		 * @deprecated Use {@link #getPartitionedState(StateDescriptor)}.
 		 */
 		@Deprecated
 		<S extends Serializable> ValueState<S> getKeyValueState(String name, Class<S> stateType, S defaultState);
@@ -221,6 +222,7 @@ public abstract class Trigger<T, W extends Window> implements Serializable {
 		 * @return The partitioned state object.
 		 * @throws UnsupportedOperationException Thrown, if no partitioned state is available for the
 		 *                                       function (function is not part os a KeyedStream).
+		 * @deprecated Use {@link #getPartitionedState(StateDescriptor)}.
 		 */
 		@Deprecated
 		<S extends Serializable> ValueState<S> getKeyValueState(String name, TypeInformation<S> stateType, S defaultState);

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractAlignedProcessingTimeWindowOperator.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractAlignedProcessingTimeWindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractAlignedProcessingTimeWindowOperator.java
index 14500ee..c2aa847 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractAlignedProcessingTimeWindowOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractAlignedProcessingTimeWindowOperator.java
@@ -38,6 +38,10 @@ import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 
 import static java.util.Objects.requireNonNull;
 
+/**
+ * @deprecated Deprecated in favour of the generic {@link WindowOperator}. This was an
+ * optimized implementation used for aligned windows.
+ */
 @Internal
 @Deprecated
 public abstract class AbstractAlignedProcessingTimeWindowOperator<KEY, IN, OUT, STATE, F extends Function> 

http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingProcessingTimeWindowOperator.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingProcessingTimeWindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingProcessingTimeWindowOperator.java
index 094b34d..57beb93 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingProcessingTimeWindowOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingProcessingTimeWindowOperator.java
@@ -29,6 +29,10 @@ import org.apache.flink.streaming.runtime.operators.windowing.functions.Internal
 
 import java.util.ArrayList;
 
+/**
+ * @deprecated Deprecated in favour of the generic {@link WindowOperator}. This was an
+ * optimized implementation used for aligned windows.
+ */
 @Internal
 @Deprecated
 public class AccumulatingProcessingTimeWindowOperator<KEY, IN, OUT>


[02/13] flink git commit: [FLINK-6212] [docs] Add avro dependency reference

Posted by ch...@apache.org.
[FLINK-6212] [docs] Add avro dependency reference

Adds the reference to the maven dependecy for flink-avro
in the "Avro support in Flink" section.

This closes #3638.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/c8f8a41b
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/c8f8a41b
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/c8f8a41b

Branch: refs/heads/master
Commit: c8f8a41b5942c00c05d78a1e1f58ba50a489a509
Parents: ffca0e7
Author: Omar Erminy <om...@gmail.com>
Authored: Wed Mar 29 09:35:23 2017 +0200
Committer: zentol <ch...@apache.org>
Committed: Wed Apr 5 20:41:13 2017 +0200

----------------------------------------------------------------------
 docs/dev/batch/connectors.md | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/c8f8a41b/docs/dev/batch/connectors.md
----------------------------------------------------------------------
diff --git a/docs/dev/batch/connectors.md b/docs/dev/batch/connectors.md
index 912d0af..b7a0718 100644
--- a/docs/dev/batch/connectors.md
+++ b/docs/dev/batch/connectors.md
@@ -87,7 +87,15 @@ This section shows some examples for connecting Flink to other systems.
 ## Avro support in Flink
 
 Flink has extensive build-in support for [Apache Avro](http://avro.apache.org/). This allows to easily read from Avro files with Flink.
-Also, the serialization framework of Flink is able to handle classes generated from Avro schemas.
+Also, the serialization framework of Flink is able to handle classes generated from Avro schemas. Be sure to include the Flink Avro dependency to the pom.xml of your project.
+
+~~~xml
+<dependency>
+  <groupId>org.apache.flink</groupId>
+  <artifactId>flink-avro{{ site.scala_version_suffix }}</artifactId>
+  <version>{{site.version }}</version>
+</dependency>
+~~~
 
 In order to read data from an Avro file, you have to specify an `AvroInputFormat`.
 


[09/13] flink git commit: [hotfix] [py] Code cleanup - PythonStreamer#sendBroadCastVariables()

Posted by ch...@apache.org.
[hotfix] [py] Code cleanup - PythonStreamer#sendBroadCastVariables()


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/f4324ba7
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/f4324ba7
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/f4324ba7

Branch: refs/heads/master
Commit: f4324ba70970ea65f305be0e793616d878319457
Parents: a7251c5
Author: zentol <ch...@apache.org>
Authored: Thu Mar 30 19:38:47 2017 +0200
Committer: zentol <ch...@apache.org>
Committed: Wed Apr 5 20:43:44 2017 +0200

----------------------------------------------------------------------
 .../apache/flink/python/api/streaming/data/PythonStreamer.java | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/f4324ba7/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonStreamer.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonStreamer.java b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonStreamer.java
index 830c843..219ae27 100644
--- a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonStreamer.java
+++ b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonStreamer.java
@@ -242,17 +242,17 @@ public class PythonStreamer<S extends PythonSender, OUT> implements Serializable
 
 			StringSerializer stringSerializer = new StringSerializer();
 			for (String name : names) {
-				Iterator<?> bcv = function.getRuntimeContext().getBroadcastVariable(name).iterator();
+				Iterator<byte[]> bcv = function.getRuntimeContext().<byte[]>getBroadcastVariable(name).iterator();
 
 				out.write(stringSerializer.serializeWithoutTypeInfo(name));
 
 				while (bcv.hasNext()) {
 					out.writeByte(1);
-					out.write((byte[]) bcv.next());
+					out.write(bcv.next());
 				}
 				out.writeByte(0);
 			}
-		} catch (SocketTimeoutException ste) {
+		} catch (SocketTimeoutException ignored) {
 			throw new RuntimeException("External process for task " + function.getRuntimeContext().getTaskName() + " stopped responding." + msg);
 		}
 	}