You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by mb...@apache.org on 2015/01/21 18:53:55 UTC

[1/8] flink git commit: [FLINK-1403] [streaming] Distributed filesystem support for streaming filesinks

Repository: flink
Updated Branches:
  refs/heads/master 7df6a3d72 -> 92947f0d8


[FLINK-1403] [streaming] Distributed filesystem support for streaming filesinks

Now streaming filesinks support the same filesystems as the batch ones


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/dc0d81bb
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/dc0d81bb
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/dc0d81bb

Branch: refs/heads/master
Commit: dc0d81bb8bc056f9c8def5ce701c1aa121d2a13f
Parents: 7df6a3d
Author: szape <ne...@gmail.com>
Authored: Tue Jan 13 14:54:01 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Wed Jan 21 16:05:25 2015 +0100

----------------------------------------------------------------------
 .../streaming/api/datastream/DataStream.java    | 163 +++++++++++++++----
 .../api/function/sink/FileSinkFunction.java     | 118 ++++++++++++++
 .../function/sink/FileSinkFunctionByMillis.java |  59 +++++++
 .../api/function/sink/RichSinkFunction.java     |   2 +-
 .../api/function/sink/SinkFunction.java         |   2 +-
 5 files changed, 306 insertions(+), 38 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/dc0d81bb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index 7d1659f..db644e9 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -20,6 +20,7 @@ package org.apache.flink.streaming.api.datastream;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.commons.lang3.Validate;
 import org.apache.flink.api.common.functions.FilterFunction;
 import org.apache.flink.api.common.functions.FlatMapFunction;
 import org.apache.flink.api.common.functions.MapFunction;
@@ -28,15 +29,20 @@ import org.apache.flink.api.common.functions.RichFilterFunction;
 import org.apache.flink.api.common.functions.RichFlatMapFunction;
 import org.apache.flink.api.common.functions.RichMapFunction;
 import org.apache.flink.api.common.functions.RichReduceFunction;
+import org.apache.flink.api.common.io.OutputFormat;
 import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo;
 import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.java.ClosureCleaner;
 import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.io.CsvOutputFormat;
+import org.apache.flink.api.java.io.TextOutputFormat;
 import org.apache.flink.api.java.operators.Keys;
 import org.apache.flink.api.java.tuple.Tuple;
 import org.apache.flink.api.java.typeutils.TupleTypeInfo;
 import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.core.fs.FileSystem.WriteMode;
+import org.apache.flink.core.fs.Path;
 import org.apache.flink.streaming.api.JobGraphBuilder;
 import org.apache.flink.streaming.api.datastream.temporaloperator.StreamCrossOperator;
 import org.apache.flink.streaming.api.datastream.temporaloperator.StreamJoinOperator;
@@ -45,12 +51,9 @@ import org.apache.flink.streaming.api.function.aggregation.AggregationFunction;
 import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.AggregationType;
 import org.apache.flink.streaming.api.function.aggregation.ComparableAggregator;
 import org.apache.flink.streaming.api.function.aggregation.SumAggregator;
+import org.apache.flink.streaming.api.function.sink.FileSinkFunctionByMillis;
 import org.apache.flink.streaming.api.function.sink.PrintSinkFunction;
 import org.apache.flink.streaming.api.function.sink.SinkFunction;
-import org.apache.flink.streaming.api.function.sink.WriteFormat;
-import org.apache.flink.streaming.api.function.sink.WriteFormatAsCsv;
-import org.apache.flink.streaming.api.function.sink.WriteFormatAsText;
-import org.apache.flink.streaming.api.function.sink.WriteSinkFunctionByMillis;
 import org.apache.flink.streaming.api.invokable.SinkInvokable;
 import org.apache.flink.streaming.api.invokable.StreamInvokable;
 import org.apache.flink.streaming.api.invokable.operator.CounterInvokable;
@@ -897,20 +900,35 @@ public class DataStream<OUT> {
 	}
 
 	/**
+	 * Writes a DataStream to the file specified by path in text format. For
+	 * every element of the DataStream the result of {@link Object#toString()}
+	 * is written.
+	 * 
+	 * @param path
+	 *            the path pointing to the location the text file is written to
+	 * 
+	 * @return the closed DataStream.
+	 */
+	public DataStreamSink<OUT> writeAsText(String path) {
+		return writeToFile(new TextOutputFormat<OUT>(new Path(path)), 0L);
+	}
+
+	/**
 	 * Writes a DataStream to the file specified by path in text format. The
 	 * writing is performed periodically, in every millis milliseconds. For
 	 * every element of the DataStream the result of {@link Object#toString()}
 	 * is written.
 	 * 
 	 * @param path
-	 *            is the path to the location where the tuples are written
+	 *            the path pointing to the location the text file is written to
 	 * @param millis
-	 *            is the file update frequency
+	 *            the file update frequency
 	 * 
-	 * @return The closed DataStream
+	 * @return the closed DataStream
 	 */
 	public DataStreamSink<OUT> writeAsText(String path, long millis) {
-		return writeAsText(path, new WriteFormatAsText<OUT>(), millis);
+		TextOutputFormat<OUT> tof = new TextOutputFormat<OUT>(new Path(path));
+		return writeToFile(tof, millis);
 	}
 
 	/**
@@ -919,64 +937,137 @@ public class DataStream<OUT> {
 	 * is written.
 	 * 
 	 * @param path
-	 *            is the path to the location where the tuples are written
+	 *            the path pointing to the location the text file is written to
+	 * @param writeMode
+	 *            Control the behavior for existing files. Options are
+	 *            NO_OVERWRITE and OVERWRITE.
 	 * 
-	 * @return The closed DataStream
+	 * @return the closed DataStream.
 	 */
-	public DataStreamSink<OUT> writeAsText(String path) {
-		return writeAsText(path, 0);
+	public DataStreamSink<OUT> writeAsText(String path, WriteMode writeMode) {
+		TextOutputFormat<OUT> tof = new TextOutputFormat<OUT>(new Path(path));
+		tof.setWriteMode(writeMode);
+		return writeToFile(tof, 0L);
 	}
 
 	/**
-	 * Writes a DataStream to the file specified by path in text format. The
-	 * writing is performed periodically, in every millis milliseconds. For
+	 * Writes a DataStream to the file specified by path in text format. For
 	 * every element of the DataStream the result of {@link Object#toString()}
 	 * is written.
 	 * 
 	 * @param path
-	 *            is the path to the location where the tuples are written
+	 *            the path pointing to the location the text file is written to
+	 * @param writeMode
+	 *            Controls the behavior for existing files. Options are
+	 *            NO_OVERWRITE and OVERWRITE.
 	 * @param millis
-	 *            is the file update frequency
+	 *            the file update frequency
 	 * 
-	 * @return The closed DataStream
+	 * @return the closed DataStream.
 	 */
-	public DataStreamSink<OUT> writeAsCsv(String path, long millis) {
-		if (!getType().isTupleType()) {
-			throw new RuntimeException("Only tuple data streams can be written in csv format");
-		}
-		return writeAsText(path, new WriteFormatAsCsv<OUT>(), millis);
+	public DataStreamSink<OUT> writeAsText(String path, WriteMode writeMode, long millis) {
+		TextOutputFormat<OUT> tof = new TextOutputFormat<OUT>(new Path(path));
+		tof.setWriteMode(writeMode);
+		return writeToFile(tof, millis);
 	}
 
 	/**
-	 * Writes a DataStream to the file specified by path in text format. For
+	 * Writes a DataStream to the file specified by path in csv format. For
 	 * every element of the DataStream the result of {@link Object#toString()}
-	 * is written.
+	 * is written. This method can only be used on data streams of tuples.
 	 * 
 	 * @param path
-	 *            is the path to the location where the tuples are written
+	 *            the path pointing to the location the text file is written to
 	 * 
-	 * @return The closed DataStream
+	 * @return the closed DataStream
 	 */
-	public DataStreamSink<OUT> writeAsCsv(String path) {
-		return writeAsCsv(path, 0);
+	@SuppressWarnings("unchecked")
+	public <X extends Tuple> DataStreamSink<OUT> writeAsCsv(String path) {
+		Validate.isTrue(getType().isTupleType(),
+				"The writeAsCsv() method can only be used on data sets of tuples.");
+		CsvOutputFormat<X> of = new CsvOutputFormat<X>(new Path(path),
+				CsvOutputFormat.DEFAULT_LINE_DELIMITER, CsvOutputFormat.DEFAULT_FIELD_DELIMITER);
+		return writeToFile((OutputFormat<OUT>) of, 0L);
 	}
 
 	/**
-	 * Writes a DataStream to the file specified by path in text format. The
+	 * Writes a DataStream to the file specified by path in csv format. The
 	 * writing is performed periodically, in every millis milliseconds. For
 	 * every element of the DataStream the result of {@link Object#toString()}
-	 * is written.
+	 * is written. This method can only be used on data streams of tuples.
 	 * 
 	 * @param path
-	 *            is the path to the location where the tuples are written
+	 *            the path pointing to the location the text file is written to
 	 * @param millis
-	 *            is the file update frequency
+	 *            the file update frequency
 	 * 
-	 * @return the data stream constructed
+	 * @return the closed DataStream
 	 */
-	private DataStreamSink<OUT> writeAsText(String path, WriteFormat<OUT> format, long millis) {
-		DataStreamSink<OUT> returnStream = addSink(new WriteSinkFunctionByMillis<OUT>(path, format,
-				millis));
+	@SuppressWarnings("unchecked")
+	public <X extends Tuple> DataStreamSink<OUT> writeAsCsv(String path, long millis) {
+		Validate.isTrue(getType().isTupleType(),
+				"The writeAsCsv() method can only be used on data sets of tuples.");
+		CsvOutputFormat<X> of = new CsvOutputFormat<X>(new Path(path),
+				CsvOutputFormat.DEFAULT_LINE_DELIMITER, CsvOutputFormat.DEFAULT_FIELD_DELIMITER);
+		return writeToFile((OutputFormat<OUT>) of, millis);
+	}
+
+	/**
+	 * Writes a DataStream to the file specified by path in csv format. For
+	 * every element of the DataStream the result of {@link Object#toString()}
+	 * is written. This method can only be used on data streams of tuples.
+	 * 
+	 * @param path
+	 *            the path pointing to the location the text file is written to
+	 * @param writeMode
+	 *            Controls the behavior for existing files. Options are
+	 *            NO_OVERWRITE and OVERWRITE.
+	 * 
+	 * @return the closed DataStream
+	 */
+	@SuppressWarnings("unchecked")
+	public <X extends Tuple> DataStreamSink<OUT> writeAsCsv(String path, WriteMode writeMode) {
+		Validate.isTrue(getType().isTupleType(),
+				"The writeAsCsv() method can only be used on data sets of tuples.");
+		CsvOutputFormat<X> of = new CsvOutputFormat<X>(new Path(path),
+				CsvOutputFormat.DEFAULT_LINE_DELIMITER, CsvOutputFormat.DEFAULT_FIELD_DELIMITER);
+		if (writeMode != null) {
+			of.setWriteMode(writeMode);
+		}
+		return writeToFile((OutputFormat<OUT>) of, 0L);
+	}
+
+	/**
+	 * Writes a DataStream to the file specified by path in csv format. The
+	 * writing is performed periodically, in every millis milliseconds. For
+	 * every element of the DataStream the result of {@link Object#toString()}
+	 * is written. This method can only be used on data streams of tuples.
+	 * 
+	 * @param path
+	 *            the path pointing to the location the text file is written to
+	 * @param writeMode
+	 *            Controls the behavior for existing files. Options are
+	 *            NO_OVERWRITE and OVERWRITE.
+	 * @param millis
+	 *            the file update frequency
+	 * 
+	 * @return the closed DataStream
+	 */
+	@SuppressWarnings("unchecked")
+	public <X extends Tuple> DataStreamSink<OUT> writeAsCsv(String path, WriteMode writeMode,
+			long millis) {
+		Validate.isTrue(getType().isTupleType(),
+				"The writeAsCsv() method can only be used on data sets of tuples.");
+		CsvOutputFormat<X> of = new CsvOutputFormat<X>(new Path(path),
+				CsvOutputFormat.DEFAULT_LINE_DELIMITER, CsvOutputFormat.DEFAULT_FIELD_DELIMITER);
+		if (writeMode != null) {
+			of.setWriteMode(writeMode);
+		}
+		return writeToFile((OutputFormat<OUT>) of, millis);
+	}
+
+	private DataStreamSink<OUT> writeToFile(OutputFormat<OUT> format, long millis) {
+		DataStreamSink<OUT> returnStream = addSink(new FileSinkFunctionByMillis<OUT>(format, millis));
 		return returnStream;
 	}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/dc0d81bb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/FileSinkFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/FileSinkFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/FileSinkFunction.java
new file mode 100644
index 0000000..24beba1
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/FileSinkFunction.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.sink;
+
+import java.io.IOException;
+import java.util.ArrayList;
+
+import org.apache.flink.api.common.io.CleanupWhenUnsuccessful;
+import org.apache.flink.api.common.io.OutputFormat;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.streamvertex.StreamingRuntimeContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Simple implementation of the SinkFunction writing tuples in the specified
+ * OutputFormat format. Tuples are collected to a list and written to the file
+ * periodically. The target path and the overwrite mode are pre-packaged in
+ * format.
+ * 
+ * @param <IN>
+ *            Input type
+ */
+public abstract class FileSinkFunction<IN> extends RichSinkFunction<IN> {
+	private static final long serialVersionUID = 1L;
+	private static final Logger LOG = LoggerFactory.getLogger(FileSinkFunction.class);
+	protected ArrayList<IN> tupleList = new ArrayList<IN>();
+	protected volatile OutputFormat<IN> format;
+	protected volatile boolean cleanupCalled = false;
+	protected int indexInSubtaskGroup;
+	protected int currentNumberOfSubtasks;
+
+	public FileSinkFunction(OutputFormat<IN> format) {
+		this.format = format;
+	}
+
+	@Override
+	public void open(Configuration parameters) throws Exception {
+		StreamingRuntimeContext context = (StreamingRuntimeContext) getRuntimeContext();
+		format.configure(context.getTaskStubParameters());
+		indexInSubtaskGroup = context.getIndexOfThisSubtask();
+		currentNumberOfSubtasks = context.getNumberOfParallelSubtasks();
+		format.open(indexInSubtaskGroup, currentNumberOfSubtasks);
+	}
+
+	@Override
+	public void invoke(IN record) throws Exception {
+		tupleList.add(record);
+		if (updateCondition()) {
+			flush();
+		}
+	}
+
+	@Override
+	public void close() throws IOException {
+		if (!tupleList.isEmpty()) {
+			flush();
+		}
+		try {
+			format.close();
+		} catch (Exception ex) {
+			try {
+				if (!cleanupCalled && format instanceof CleanupWhenUnsuccessful) {
+					cleanupCalled = true;
+					((CleanupWhenUnsuccessful) format).tryCleanupOnError();
+				}
+			} catch (Throwable t) {
+				LOG.error("Cleanup on error failed.", t);
+			}
+		}
+	}
+
+	protected void flush() {
+		try {
+			for (IN rec : tupleList) {
+				format.writeRecord(rec);
+			}
+		} catch (Exception ex) {
+			try {
+				if (!cleanupCalled && format instanceof CleanupWhenUnsuccessful) {
+					cleanupCalled = true;
+					((CleanupWhenUnsuccessful) format).tryCleanupOnError();
+				}
+			} catch (Throwable t) {
+				LOG.error("Cleanup on error failed.", t);
+			}
+		}
+		resetParameters();
+	}
+
+	/**
+	 * Condition for writing the contents of tupleList and clearing it.
+	 * 
+	 * @return value of the updating condition
+	 */
+	protected abstract boolean updateCondition();
+
+	/**
+	 * Statements to be executed after writing a batch goes here.
+	 */
+	protected abstract void resetParameters();
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/dc0d81bb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/FileSinkFunctionByMillis.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/FileSinkFunctionByMillis.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/FileSinkFunctionByMillis.java
new file mode 100644
index 0000000..f049a32
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/FileSinkFunctionByMillis.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.sink;
+
+import org.apache.flink.api.common.io.OutputFormat;
+
+/**
+ * Implementation of FileSinkFunction. Writes tuples to file in every millis
+ * milliseconds.
+ * 
+ * @param <IN>
+ *            Input type
+ */
+public class FileSinkFunctionByMillis<IN> extends FileSinkFunction<IN> {
+	private static final long serialVersionUID = 1L;
+
+	private final long millis;
+	private long lastTime;
+
+	public FileSinkFunctionByMillis(OutputFormat<IN> format, long millis) {
+		super(format);
+		this.millis = millis;
+		lastTime = System.currentTimeMillis();
+	}
+
+	/**
+	 * Condition for writing the contents of tupleList and clearing it.
+	 * 
+	 * @return value of the updating condition
+	 */
+	@Override
+	protected boolean updateCondition() {
+		return System.currentTimeMillis() - lastTime >= millis;
+	}
+
+	/**
+	 * Statements to be executed after writing a batch goes here.
+	 */
+	@Override
+	protected void resetParameters() {
+		tupleList.clear();
+		lastTime = System.currentTimeMillis();
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/dc0d81bb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/RichSinkFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/RichSinkFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/RichSinkFunction.java
index b89c169..3b8a4db 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/RichSinkFunction.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/RichSinkFunction.java
@@ -23,6 +23,6 @@ public abstract class RichSinkFunction<IN> extends AbstractRichFunction implemen
 
 	private static final long serialVersionUID = 1L;
 
-	public abstract void invoke(IN value);
+	public abstract void invoke(IN value) throws Exception;
 
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/dc0d81bb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/SinkFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/SinkFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/SinkFunction.java
index 9db268e..6097603 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/SinkFunction.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/SinkFunction.java
@@ -23,6 +23,6 @@ import org.apache.flink.api.common.functions.Function;
 
 public interface SinkFunction<IN> extends Function, Serializable {
 
-	public abstract void invoke(IN value);
+	public abstract void invoke(IN value) throws Exception;
 
 }


[3/8] flink git commit: [streaming] Streaming runtime collector rework for increased flexibility with directed outputs and operator chains

Posted by mb...@apache.org.
[streaming] Streaming runtime collector rework for increased flexibility with directed outputs and operator chains


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/d9b942b3
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/d9b942b3
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/d9b942b3

Branch: refs/heads/master
Commit: d9b942b38cea01bf78ffab22a164b6622d629bc5
Parents: dc0d81b
Author: Gyula Fora <gy...@apache.org>
Authored: Thu Jan 15 11:14:33 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Wed Jan 21 16:06:33 2015 +0100

----------------------------------------------------------------------
 .../api/collector/CollectorWrapper.java         |  48 ++++++
 .../api/collector/DirectedOutputWrapper.java    | 144 ++++++++++++++++++
 .../api/collector/DirectedStreamCollector.java  | 130 ----------------
 .../api/collector/StreamCollector.java          | 148 -------------------
 .../streaming/api/collector/StreamOutput.java   |  65 ++++++++
 .../api/collector/StreamOutputWrapper.java      | 122 +++++++++++++++
 .../api/streamvertex/InputHandler.java          |   1 -
 .../api/streamvertex/OutputHandler.java         |  26 ++--
 .../flink/streaming/io/StreamRecordWriter.java  |   2 +-
 .../api/collector/StreamCollectorTest.java      |   4 +-
 .../runtime/operators/RegularPactTask.java      |   2 -
 11 files changed, 396 insertions(+), 296 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/d9b942b3/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
new file mode 100644
index 0000000..d782d08
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.collector;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.flink.util.Collector;
+
+public class CollectorWrapper<OUT> implements Collector<OUT> {
+
+	List<Collector<OUT>> outputs;
+
+	public CollectorWrapper() {
+		this.outputs = new LinkedList<Collector<OUT>>();
+	}
+
+	public void addChainedOutput(Collector<OUT> output) {
+		outputs.add(output);
+	}
+
+	@Override
+	public void collect(OUT record) {
+		for(Collector<OUT> output: outputs){
+			output.collect(record);;
+		}
+	}
+
+	@Override
+	public void close() {
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/d9b942b3/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedOutputWrapper.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedOutputWrapper.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedOutputWrapper.java
new file mode 100755
index 0000000..6ff03c4
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedOutputWrapper.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.collector;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.flink.runtime.plugable.SerializationDelegate;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.apache.flink.util.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A StreamCollector that uses user defined output names and a user defined
+ * output selector to make directed emits.
+ * 
+ * @param <OUT>
+ *            Type of the Tuple collected.
+ */
+public class DirectedOutputWrapper<OUT> extends StreamOutputWrapper<OUT> {
+
+	private static final Logger LOG = LoggerFactory.getLogger(DirectedOutputWrapper.class);
+
+	OutputSelector<OUT> outputSelector;
+
+	protected Map<String, List<StreamOutput<OUT>>> outputMap;
+
+	private List<StreamOutput<OUT>> selectAllOutputs;
+	private Set<StreamOutput<OUT>> emitted;
+
+	/**
+	 * Creates a new DirectedStreamCollector
+	 * 
+	 * @param channelID
+	 *            Channel ID of the Task
+	 * @param serializationDelegate
+	 *            Serialization delegate used for serialization
+	 * @param outputSelector
+	 *            User defined {@link OutputSelector}
+	 */
+	public DirectedOutputWrapper(int channelID,
+			SerializationDelegate<StreamRecord<OUT>> serializationDelegate,
+			OutputSelector<OUT> outputSelector) {
+		super(channelID, serializationDelegate);
+		this.outputSelector = outputSelector;
+		this.emitted = new HashSet<StreamOutput<OUT>>();
+		this.selectAllOutputs = new LinkedList<StreamOutput<OUT>>();
+		this.outputMap = new HashMap<String, List<StreamOutput<OUT>>>();
+
+	}
+
+	@Override
+	public void addOutput(StreamOutput<OUT> output) {
+
+		super.addOutput(output);
+
+		if (output.isSelectAll()) {
+			selectAllOutputs.add(output);
+		} else {
+			for (String selectedName : output.getSelectedNames()) {
+				if (selectedName != null) {
+					if (!outputMap.containsKey(selectedName)) {
+						outputMap.put(selectedName, new LinkedList<StreamOutput<OUT>>());
+						outputMap.get(selectedName).add(output);
+					} else {
+						if (!outputMap.get(selectedName).contains(output)) {
+							outputMap.get(selectedName).add(output);
+						}
+					}
+
+				}
+			}
+		}
+	}
+
+	@Override
+	protected void emit() {
+		Iterable<String> outputNames = outputSelector.select(serializationDelegate.getInstance()
+				.getObject());
+		emitted.clear();
+
+		for (StreamOutput<OUT> output : selectAllOutputs) {
+			try {
+				output.collect(serializationDelegate);
+			} catch (Exception e) {
+				if (LOG.isErrorEnabled()) {
+					LOG.error("Emit to {} failed due to: {}", output,
+							StringUtils.stringifyException(e));
+				}
+			}
+		}
+
+		for (String outputName : outputNames) {
+			List<StreamOutput<OUT>> outputList = outputMap.get(outputName);
+			try {
+				if (outputList == null) {
+					if (LOG.isErrorEnabled()) {
+						String format = String.format(
+								"Cannot emit because no output is selected with the name: %s",
+								outputName);
+						LOG.error(format);
+
+					}
+				} else {
+
+					for (StreamOutput<OUT> output : outputList) {
+						if (!emitted.contains(output)) {
+							output.collect(serializationDelegate);
+							emitted.add(output);
+						}
+					}
+
+				}
+
+			} catch (Exception e) {
+				if (LOG.isErrorEnabled()) {
+					LOG.error("Emit to {} failed due to: {}", outputName,
+							StringUtils.stringifyException(e));
+				}
+			}
+
+		}
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/d9b942b3/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedStreamCollector.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedStreamCollector.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedStreamCollector.java
deleted file mode 100755
index f37761f..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedStreamCollector.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.collector;
-
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
-import org.apache.flink.runtime.plugable.SerializationDelegate;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.apache.flink.util.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * A StreamCollector that uses user defined output names and a user defined
- * output selector to make directed emits.
- * 
- * @param <OUT>
- *            Type of the Tuple collected.
- */
-public class DirectedStreamCollector<OUT> extends StreamCollector<OUT> {
-
-	private static final Logger LOG = LoggerFactory.getLogger(DirectedStreamCollector.class);
-
-	OutputSelector<OUT> outputSelector;
-	private List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> selectAllOutputs;
-	private Set<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> emitted;
-
-	/**
-	 * Creates a new DirectedStreamCollector
-	 * 
-	 * @param channelID
-	 *            Channel ID of the Task
-	 * @param serializationDelegate
-	 *            Serialization delegate used for serialization
-	 * @param outputSelector
-	 *            User defined {@link OutputSelector}
-	 */
-	public DirectedStreamCollector(int channelID,
-			SerializationDelegate<StreamRecord<OUT>> serializationDelegate,
-			OutputSelector<OUT> outputSelector) {
-		super(channelID, serializationDelegate);
-		this.outputSelector = outputSelector;
-		this.emitted = new HashSet<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>>();
-		this.selectAllOutputs = new LinkedList<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>>();
-	}
-
-	@Override
-	public void addOutput(RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output,
-			List<String> outputNames, boolean isSelectAllOutput) {
-
-		if (isSelectAllOutput) {
-			selectAllOutputs.add(output);
-		} else {
-			addOneOutput(output, outputNames, isSelectAllOutput);
-		}
-	}
-
-	/**
-	 * Emits a StreamRecord to the outputs selected by the user defined
-	 * OutputSelector
-	 *
-	 */
-	protected void emitToOutputs() {
-		Iterable<String> outputNames = outputSelector.select(streamRecord.getObject());
-		emitted.clear();
-
-		for (RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output : selectAllOutputs) {
-			try {
-				output.emit(serializationDelegate);
-			} catch (Exception e) {
-				if (LOG.isErrorEnabled()) {
-					LOG.error("Emit to {} failed due to: {}", output,
-							StringUtils.stringifyException(e));
-				}
-			}
-		}
-		emitted.addAll(selectAllOutputs);
-
-		for (String outputName : outputNames) {
-			List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> outputList = outputMap
-					.get(outputName);
-			try {
-				if (outputList == null) {
-					if (LOG.isErrorEnabled()) {
-						String format = String.format(
-								"Cannot emit because no output is selected with the name: %s",
-								outputName);
-						LOG.error(format);
-
-					}
-				} else {
-
-					for (RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output : outputList) {
-						if (!emitted.contains(output)) {
-							output.emit(serializationDelegate);
-							emitted.add(output);
-						}
-					}
-
-				}
-
-			} catch (Exception e) {
-				if (LOG.isErrorEnabled()) {
-					LOG.error("Emit to {} failed due to: {}", outputName,
-							StringUtils.stringifyException(e));
-				}
-			}
-
-		}
-	}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/d9b942b3/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamCollector.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamCollector.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamCollector.java
deleted file mode 100755
index a1b9e66..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamCollector.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.collector;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
-import org.apache.flink.runtime.plugable.SerializationDelegate;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.apache.flink.util.Collector;
-import org.apache.flink.util.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Collector for tuples in Apache Flink stream processing. The collected values
- * will be wrapped with ID in a {@link StreamRecord} and then emitted to the
- * outputs.
- * 
- * @param <OUT>
- *            Type of the Tuples/Objects collected.
- */
-public class StreamCollector<OUT> implements Collector<OUT> {
-
-	private static final Logger LOG = LoggerFactory.getLogger(StreamCollector.class);
-
-	protected StreamRecord<OUT> streamRecord;
-	protected int channelID;
-	protected List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> outputs;
-	protected Map<String, List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>>> outputMap;
-	protected SerializationDelegate<StreamRecord<OUT>> serializationDelegate;
-
-	/**
-	 * Creates a new StreamCollector
-	 * 
-	 * @param channelID
-	 *            Channel ID of the Task
-	 * @param serializationDelegate
-	 *            Serialization delegate used for serialization
-	 */
-	public StreamCollector(int channelID,
-			SerializationDelegate<StreamRecord<OUT>> serializationDelegate) {
-		this.serializationDelegate = serializationDelegate;
-
-		if (serializationDelegate != null) {
-			this.streamRecord = serializationDelegate.getInstance();
-		} else {
-			this.streamRecord = new StreamRecord<OUT>();
-		}
-		this.channelID = channelID;
-		this.outputs = new ArrayList<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>>();
-		this.outputMap = new HashMap<String, List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>>>();
-	}
-
-	/**
-	 * Adds an output with the given user defined name
-	 * 
-	 * @param output
-	 *            The RecordWriter object representing the output.
-	 * @param outputNames
-	 *            User defined names of the output.
-	 * @param isSelectAllOutput
-	 *            Marks whether all the outputs are selected.
-	 */
-	public void addOutput(RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output,
-			List<String> outputNames, boolean isSelectAllOutput) {
-		addOneOutput(output, outputNames, isSelectAllOutput);
-	}
-
-	protected void addOneOutput(RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output,
-			List<String> outputNames, boolean isSelectAllOutput) {
-		outputs.add(output);
-		for (String outputName : outputNames) {
-			if (outputName != null) {
-				if (!outputMap.containsKey(outputName)) {
-					outputMap
-							.put(outputName,
-									new ArrayList<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>>());
-					outputMap.get(outputName).add(output);
-				} else {
-					if (!outputMap.get(outputName).contains(output)) {
-						outputMap.get(outputName).add(output);
-					}
-				}
-
-			}
-		}
-	}
-
-	/**
-	 * Collects and emits a tuple/object to the outputs by reusing a
-	 * StreamRecord object.
-	 * 
-	 * @param outputObject
-	 *            Object to be collected and emitted.
-	 */
-	@Override
-	public void collect(OUT outputObject) {
-		streamRecord.setObject(outputObject);
-		emit(streamRecord);
-	}
-
-	/**
-	 * Emits a StreamRecord to the outputs.
-	 * 
-	 * @param streamRecord
-	 *            StreamRecord to emit.
-	 */
-	private void emit(StreamRecord<OUT> streamRecord) {
-		streamRecord.newId(channelID);
-		serializationDelegate.setInstance(streamRecord);
-		emitToOutputs();
-	}
-
-	protected void emitToOutputs() {
-		for (RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output : outputs) {
-			try {
-				output.emit(serializationDelegate);
-			} catch (Exception e) {
-				if (LOG.isErrorEnabled()) {
-					LOG.error("Emit failed due to: {}", StringUtils.stringifyException(e));
-				}
-			}
-		}
-	}
-
-	@Override
-	public void close() {
-	}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/d9b942b3/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java
new file mode 100644
index 0000000..4c21564
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.collector;
+
+import java.util.List;
+
+import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
+import org.apache.flink.runtime.plugable.SerializationDelegate;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.apache.flink.util.Collector;
+
+public class StreamOutput<OUT> implements Collector<SerializationDelegate<StreamRecord<OUT>>> {
+
+	private RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output;
+
+	private List<String> selectedNames;
+	private boolean selectAll = true;
+
+	public StreamOutput(RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output,
+			List<String> selectedNames) {
+
+		this.output = output;
+
+		if (selectedNames != null) {
+			this.selectedNames = selectedNames;
+			selectAll = false;
+		}
+	}
+
+	public void collect(SerializationDelegate<StreamRecord<OUT>> record) {
+		try {
+			output.emit(record);
+		} catch (Exception e) {
+			throw new RuntimeException("Could not emit record: " + record.getInstance());
+		}
+	}
+
+	@Override
+	public void close() {
+	}
+
+	public boolean isSelectAll() {
+		return selectAll;
+	}
+
+	public List<String> getSelectedNames() {
+		return selectedNames;
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/d9b942b3/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutputWrapper.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutputWrapper.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutputWrapper.java
new file mode 100755
index 0000000..fa374b1
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutputWrapper.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.collector;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
+import org.apache.flink.runtime.plugable.SerializationDelegate;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.apache.flink.util.Collector;
+import org.apache.flink.util.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Collector for tuples in Apache Flink stream processing. The collected values
+ * will be wrapped with ID in a {@link StreamRecord} and then emitted to the
+ * outputs.
+ * 
+ * @param <OUT>
+ *            Type of the Tuples/Objects collected.
+ */
+public class StreamOutputWrapper<OUT> implements Collector<OUT> {
+
+	private static final Logger LOG = LoggerFactory.getLogger(StreamOutputWrapper.class);
+
+	protected StreamRecord<OUT> streamRecord;
+	protected int channelID;
+	protected List<StreamOutput<OUT>> outputs;
+	protected SerializationDelegate<StreamRecord<OUT>> serializationDelegate;
+
+	/**
+	 * Creates a new StreamCollector
+	 * 
+	 * @param channelID
+	 *            Channel ID of the Task
+	 * @param serializationDelegate
+	 *            Serialization delegate used for serialization
+	 */
+	public StreamOutputWrapper(int channelID,
+			SerializationDelegate<StreamRecord<OUT>> serializationDelegate) {
+		this.serializationDelegate = serializationDelegate;
+
+		if (serializationDelegate != null) {
+			this.streamRecord = serializationDelegate.getInstance();
+		} else {
+			this.streamRecord = new StreamRecord<OUT>();
+		}
+		this.channelID = channelID;
+		this.outputs = new LinkedList<StreamOutput<OUT>>();
+	}
+
+	/**
+	 * Adds an output with the given user defined name
+	 * 
+	 * @param output
+	 *            The RecordWriter object representing the output.
+	 * @param outputNames
+	 *            User defined names of the output.
+	 * @param isSelectAllOutput
+	 *            Marks whether all the outputs are selected.
+	 */
+	public void addOutput(StreamOutput<OUT> output) {
+		outputs.add(output);
+	}
+
+	protected void addOneOutput(RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output,
+			List<String> outputNames, boolean isSelectAllOutput) {
+
+	}
+
+	/**
+	 * Collects and emits a tuple/object to the outputs by reusing a
+	 * StreamRecord object.
+	 * 
+	 * @param outputObject
+	 *            Object to be collected and emitted.
+	 */
+	@Override
+	public void collect(OUT outputObject) {
+		streamRecord.setObject(outputObject);
+		streamRecord.newId(channelID);
+		serializationDelegate.setInstance(streamRecord);
+
+		emit();
+	}
+
+	/**
+	 * Emits the current streamrecord to the outputs.
+	 */
+	protected void emit() {
+		for (StreamOutput<OUT> output : outputs) {
+			try {
+				output.collect(serializationDelegate);
+			} catch (Exception e) {
+				if (LOG.isErrorEnabled()) {
+					LOG.error("Emit failed due to: {}", StringUtils.stringifyException(e));
+				}
+			}
+		}
+	}
+
+	@Override
+	public void close() {
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/d9b942b3/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/InputHandler.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/InputHandler.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/InputHandler.java
index fdf23f7..ad6d948 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/InputHandler.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/InputHandler.java
@@ -49,7 +49,6 @@ public class InputHandler<IN> {
 
 	}
 
-	@SuppressWarnings("unchecked")
 	protected void setConfigInputs() throws StreamVertexException {
 		inputSerializer = configuration.getTypeSerializerIn1(streamVertex.userClassLoader);
 

http://git-wip-us.apache.org/repos/asf/flink/blob/d9b942b3/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
index f9fe593..61a6eb4 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
@@ -21,9 +21,10 @@ import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
 import org.apache.flink.runtime.plugable.SerializationDelegate;
 import org.apache.flink.streaming.api.StreamConfig;
-import org.apache.flink.streaming.api.collector.DirectedStreamCollector;
+import org.apache.flink.streaming.api.collector.DirectedOutputWrapper;
 import org.apache.flink.streaming.api.collector.OutputSelector;
-import org.apache.flink.streaming.api.collector.StreamCollector;
+import org.apache.flink.streaming.api.collector.StreamOutputWrapper;
+import org.apache.flink.streaming.api.collector.StreamOutput;
 import org.apache.flink.streaming.api.invokable.StreamInvokable;
 import org.apache.flink.streaming.api.streamrecord.StreamRecord;
 import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
@@ -43,7 +44,7 @@ public class OutputHandler<OUT> {
 	private StreamConfig configuration;
 
 	private List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> outputs;
-	private StreamCollector<OUT> collector;
+	private StreamOutputWrapper<OUT> collector;
 	private long bufferTimeout;
 
 	TypeInformation<OUT> outTypeInfo = null;
@@ -79,21 +80,21 @@ public class OutputHandler<OUT> {
 		}
 	}
 
-	private StreamCollector<OUT> setCollector() {
+	private StreamOutputWrapper<OUT> setCollector() {
 		if (streamVertex.configuration.getDirectedEmit()) {
 			OutputSelector<OUT> outputSelector = streamVertex.configuration
 					.getOutputSelector(streamVertex.userClassLoader);
 
-			collector = new DirectedStreamCollector<OUT>(streamVertex.getInstanceID(),
+			collector = new DirectedOutputWrapper<OUT>(streamVertex.getInstanceID(),
 					outSerializationDelegate, outputSelector);
 		} else {
-			collector = new StreamCollector<OUT>(streamVertex.getInstanceID(),
+			collector = new StreamOutputWrapper<OUT>(streamVertex.getInstanceID(),
 					outSerializationDelegate);
 		}
 		return collector;
 	}
 
-	public StreamCollector<OUT> getCollector() {
+	public StreamOutputWrapper<OUT> getCollector() {
 		return collector;
 	}
 
@@ -121,16 +122,16 @@ public class OutputHandler<OUT> {
 		RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output;
 
 		if (bufferTimeout >= 0) {
-			output = new StreamRecordWriter<SerializationDelegate<StreamRecord<OUT>>>(streamVertex.getEnvironment().getWriter(outputNumber),
-					outputPartitioner, bufferTimeout);
+			output = new StreamRecordWriter<SerializationDelegate<StreamRecord<OUT>>>(streamVertex
+					.getEnvironment().getWriter(outputNumber), outputPartitioner, bufferTimeout);
 
 			if (LOG.isTraceEnabled()) {
 				LOG.trace("StreamRecordWriter initiated with {} bufferTimeout for {}",
 						bufferTimeout, streamVertex.getClass().getSimpleName());
 			}
 		} else {
-			output = new RecordWriter<SerializationDelegate<StreamRecord<OUT>>>(streamVertex.getEnvironment().getWriter(outputNumber),
-					outputPartitioner);
+			output = new RecordWriter<SerializationDelegate<StreamRecord<OUT>>>(streamVertex
+					.getEnvironment().getWriter(outputNumber), outputPartitioner);
 
 			if (LOG.isTraceEnabled()) {
 				LOG.trace("RecordWriter initiated for {}", streamVertex.getClass().getSimpleName());
@@ -142,7 +143,8 @@ public class OutputHandler<OUT> {
 		boolean isSelectAllOutput = configuration.getSelectAll(outputNumber);
 
 		if (collector != null) {
-			collector.addOutput(output, outputName, isSelectAllOutput);
+			collector
+					.addOutput(new StreamOutput<OUT>(output, isSelectAllOutput ? null : outputName));
 		}
 
 		if (LOG.isTraceEnabled()) {

http://git-wip-us.apache.org/repos/asf/flink/blob/d9b942b3/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/StreamRecordWriter.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/StreamRecordWriter.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/StreamRecordWriter.java
index fe9cce3..b7af589 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/StreamRecordWriter.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/StreamRecordWriter.java
@@ -25,7 +25,7 @@ import org.apache.flink.runtime.io.network.api.writer.RoundRobinChannelSelector;
 
 import java.io.IOException;
 
-public class StreamRecordWriter<T extends IOReadableWritable> extends RecordWriter {
+public class StreamRecordWriter<T extends IOReadableWritable> extends RecordWriter<T> {
 
 	private long timeout;
 

http://git-wip-us.apache.org/repos/asf/flink/blob/d9b942b3/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/StreamCollectorTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/StreamCollectorTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/StreamCollectorTest.java
index 535e109..0bc3d7d 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/StreamCollectorTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/StreamCollectorTest.java
@@ -37,8 +37,8 @@ public class StreamCollectorTest {
 				null);
 		sd.setInstance(new StreamRecord<Tuple1<Integer>>().setObject(new Tuple1<Integer>()));
 
-		StreamCollector<Tuple1<Integer>> collector = new StreamCollector<Tuple1<Integer>>(2, sd);
-		collector.addOutput(recWriter, new ArrayList<String>(), false);
+		StreamOutputWrapper<Tuple1<Integer>> collector = new StreamOutputWrapper<Tuple1<Integer>>(2, sd);
+		collector.addOutput(new StreamOutput<Tuple1<Integer>>(recWriter, new ArrayList<String>()));
 		collector.collect(new Tuple1<Integer>(3));
 		collector.collect(new Tuple1<Integer>(4));
 		collector.collect(new Tuple1<Integer>(5));

http://git-wip-us.apache.org/repos/asf/flink/blob/d9b942b3/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java
index 2d36c1e..6b6918d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java
@@ -706,7 +706,6 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
 	 *
 	 * This method requires that the task configuration, the driver, and the user-code class loader are set.
 	 */
-	@SuppressWarnings("unchecked")
 	protected void initInputReaders() throws Exception {
 		final int numInputs = getNumTaskInputs();
 		final MutableReader<?>[] inputReaders = new MutableReader[numInputs];
@@ -748,7 +747,6 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
 	 *
 	 * This method requires that the task configuration, the driver, and the user-code class loader are set.
 	 */
-	@SuppressWarnings("unchecked")
 	protected void initBroadcastInputReaders() throws Exception {
 		final int numBroadcastInputs = this.config.getNumBroadcastInputs();
 		final MutableReader<?>[] broadcastInputReaders = new MutableReader[numBroadcastInputs];


[4/8] flink git commit: [FLINK-1345] [streaming] Advanced task chaining added

Posted by mb...@apache.org.
[FLINK-1345] [streaming] Advanced task chaining added


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/7dbb55ec
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/7dbb55ec
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/7dbb55ec

Branch: refs/heads/master
Commit: 7dbb55ece0a9d9777c0e3254bc8f9f5cf566d535
Parents: 3e30c6f
Author: Gyula Fora <gy...@apache.org>
Authored: Sun Jan 18 18:23:57 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Wed Jan 21 16:06:34 2015 +0100

----------------------------------------------------------------------
 .../flink/streaming/api/JobGraphBuilder.java    | 298 ++++++++-----------
 .../flink/streaming/api/StreamConfig.java       | 180 +++++++----
 .../api/collector/CollectorWrapper.java         |   2 +-
 .../streaming/api/collector/StreamOutput.java   |  15 +
 .../api/collector/StreamOutputWrapper.java      |   6 -
 .../api/streamvertex/OutputHandler.java         | 204 ++++++++-----
 .../api/streamvertex/StreamIterationHead.java   |  23 +-
 .../flink/streaming/api/scala/DataStream.scala  |  10 +
 8 files changed, 433 insertions(+), 305 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/7dbb55ec/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
index 0020d48..6ae97c9 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
@@ -24,9 +24,10 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import org.apache.commons.lang3.StringUtils;
 import org.apache.flink.api.common.io.InputFormat;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.configuration.Configuration;
 import org.apache.flink.runtime.jobgraph.AbstractJobVertex;
 import org.apache.flink.runtime.jobgraph.DistributionPattern;
 import org.apache.flink.runtime.jobgraph.JobGraph;
@@ -34,7 +35,6 @@ import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
 import org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup;
 import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
 import org.apache.flink.streaming.api.collector.OutputSelector;
-import org.apache.flink.streaming.api.invokable.ChainableInvokable;
 import org.apache.flink.streaming.api.invokable.StreamInvokable;
 import org.apache.flink.streaming.api.invokable.StreamInvokable.ChainingStrategy;
 import org.apache.flink.streaming.api.invokable.operator.co.CoInvokable;
@@ -85,11 +85,11 @@ public class JobGraphBuilder {
 	private Map<String, Long> iterationWaitTime;
 	private Map<String, Map<String, OperatorState<?>>> operatorStates;
 	private Map<String, InputFormat<String, ?>> inputFormatList;
-	private Map<String, List<String>> chainedVertices;
-	private Map<String, String> lastInChains;
+	private Map<String, Map<String, StreamConfig>> chainedConfigs;
+	private Map<String, StreamConfig> vertexConfigs;
 
 	private Set<String> sources;
-	private Set<String> builtVertices;
+	private Set<String> builtNodes;
 
 	/**
 	 * Creates an new {@link JobGraph} with the given name. A JobGraph is a DAG
@@ -129,11 +129,11 @@ public class JobGraphBuilder {
 		iterationWaitTime = new HashMap<String, Long>();
 		operatorStates = new HashMap<String, Map<String, OperatorState<?>>>();
 		inputFormatList = new HashMap<String, InputFormat<String, ?>>();
-		chainedVertices = new HashMap<String, List<String>>();
-		lastInChains = new HashMap<String, String>();
+		chainedConfigs = new HashMap<String, Map<String, StreamConfig>>();
+		vertexConfigs = new HashMap<String, StreamConfig>();
 
 		sources = new HashSet<String>();
-		builtVertices = new HashSet<String>();
+		builtNodes = new HashSet<String>();
 	}
 
 	/**
@@ -198,6 +198,8 @@ public class JobGraphBuilder {
 
 		addVertex(vertexName, StreamIterationHead.class, null, null, parallelism);
 
+		chaining = false;
+
 		iterationIds.put(vertexName, iterationID);
 		iterationIDtoHeadName.put(iterationID, vertexName);
 
@@ -297,7 +299,6 @@ public class JobGraphBuilder {
 		inEdgeList.put(vertexName, new ArrayList<String>());
 		outPartitioning.put(vertexName, new ArrayList<StreamPartitioner<?>>());
 		iterationTailCount.put(vertexName, 0);
-		lastInChains.put(vertexName, vertexName);
 	}
 
 	private void addTypeSerializers(String vertexName, StreamRecordSerializer<?> in1,
@@ -309,170 +310,175 @@ public class JobGraphBuilder {
 		typeSerializersOut2.put(vertexName, out2);
 	}
 
-	/**
-	 * Creates an {@link AbstractJobVertex} in the {@link JobGraph} and sets its
-	 * config parameters using the ones set previously.
-	 * 
-	 * @param vertexName
-	 *            Name for which the vertex will be created.
-	 */
-	private void createVertex(String vertexName) {
+	private List<Tuple2<String, String>> createChain(String startNode, String current) {
 
-		if (!builtVertices.contains(vertexName)) {
-			if (!outEdgeList.get(vertexName).isEmpty()) {
+		if (!builtNodes.contains(startNode)) {
 
-				for (String outName : outEdgeList.get(vertexName)) {
-					if (isChainable(vertexName, outName)) {
-						chainRecursively(vertexName, vertexName, outName);
-					} else {
-						createVertex(outName);
-					}
+			List<Tuple2<String, String>> transitiveOutEdges = new ArrayList<Tuple2<String, String>>();
+			List<String> chainableOutputs = new ArrayList<String>();
+			List<String> nonChainableOutputs = new ArrayList<String>();
 
+			for (String outName : outEdgeList.get(current)) {
+				if (isChainable(current, outName)) {
+					chainableOutputs.add(outName);
+				} else {
+					nonChainableOutputs.add(outName);
 				}
+
 			}
 
-			List<String> chainedNames = chainedVertices.get(vertexName);
-			boolean isChained = chainedNames != null;
-			int numChained = isChained ? chainedNames.size() : 0;
-			String lastInChain = lastInChains.get(vertexName);
-
-			// Get vertex attributes
-			Class<? extends AbstractInvokable> vertexClass = vertexClasses.get(vertexName);
-			StreamInvokable<?, ?> invokableObject = invokableObjects.get(vertexName);
-			int parallelism = vertexParallelism.get(vertexName);
-			byte[] outputSelector = outputSelectors.get(lastInChain);
-			Map<String, OperatorState<?>> state = operatorStates.get(vertexName);
-
-			// Create vertex object
-			String cname = chainedVertices.get(vertexName) == null ? "" : " => "
-					+ StringUtils.join(chainedVertices.get(vertexName), " => ");
-			AbstractJobVertex vertex = new AbstractJobVertex(vertexName + cname);
-
-			this.jobGraph.addVertex(vertex);
-
-			vertex.setInvokableClass(vertexClass);
-			vertex.setParallelism(parallelism);
-			if (LOG.isDebugEnabled()) {
-				LOG.debug("Parallelism set: {} for {}", parallelism, vertexName);
+			for (String chainable : chainableOutputs) {
+				transitiveOutEdges.addAll(createChain(startNode, chainable));
 			}
 
-			// Set vertex config
+			for (String nonChainable : nonChainableOutputs) {
+				transitiveOutEdges.add(new Tuple2<String, String>(current, nonChainable));
+				transitiveOutEdges.addAll(createChain(nonChainable, nonChainable));
+			}
 
-			StreamConfig config = new StreamConfig(vertex.getConfiguration());
+			StreamConfig config = current.equals(startNode) ? createProcessingVertex(startNode)
+					: new StreamConfig(new Configuration());
 
-			config.setBufferTimeout(bufferTimeout.get(lastInChain));
+			setVertexConfig(current, config, chainableOutputs, nonChainableOutputs);
 
-			config.setTypeSerializerIn1(typeSerializersIn1.get(vertexName));
-			config.setTypeSerializerIn2(typeSerializersIn2.get(vertexName));
-			config.setTypeSerializerOut1(typeSerializersOut1.get(vertexName));
-			config.setTypeSerializerOut2(typeSerializersOut2.get(vertexName));
+			if (current.equals(startNode)) {
 
-			config.setUserInvokable(invokableObject);
-			config.setOutputSelector(outputSelector);
-			config.setOperatorStates(state);
+				config.setChainStart();
+				config.setRecordWriterOrder(transitiveOutEdges);
 
-			if (vertexClass.equals(StreamIterationHead.class)
-					|| vertexClass.equals(StreamIterationTail.class)) {
-				config.setIterationId(iterationIds.get(vertexName));
-				config.setIterationWaitTime(iterationWaitTime.get(vertexName));
-			}
+				for (Tuple2<String, String> edge : transitiveOutEdges) {
+					connect(startNode, edge);
+				}
 
-			if (inputFormatList.containsKey(vertexName)) {
-				vertex.setInputSplitSource(inputFormatList.get(vertexName));
-			}
+				vertexConfigs.get(startNode).setTransitiveChainedTaskConfigs(
+						chainedConfigs.get(startNode));
+
+			} else {
 
-			config.setNumberofChainedTasks(numChained);
+				Map<String, StreamConfig> chainedConfs = chainedConfigs.get(startNode);
 
-			for (int i = 0; i < numChained; i++) {
-				config.setChainedInvokable(
-						(ChainableInvokable<?, ?>) invokableObjects.get(chainedNames.get(i)), i);
-				config.setChainedSerializer(typeSerializersIn1.get(chainedNames.get(i)), i);
+				if (chainedConfs == null) {
+					chainedConfigs.put(startNode, new HashMap<String, StreamConfig>());
+				}
+				chainedConfigs.get(startNode).put(current, config);
 			}
 
-			streamVertices.put(vertexName, vertex);
-			builtVertices.add(vertexName);
+			return transitiveOutEdges;
 
+		} else {
+			return new ArrayList<Tuple2<String, String>>();
 		}
+	}
+
+	private StreamConfig createProcessingVertex(String vertexName) {
+
+		AbstractJobVertex vertex = new AbstractJobVertex(vertexName);
+
+		this.jobGraph.addVertex(vertex);
 
+		int parallelism = vertexParallelism.get(vertexName);
+
+		vertex.setInvokableClass(vertexClasses.get(vertexName));
+		vertex.setParallelism(parallelism);
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("Parallelism set: {} for {}", parallelism, vertexName);
+		}
+
+		if (inputFormatList.containsKey(vertexName)) {
+			vertex.setInputSplitSource(inputFormatList.get(vertexName));
+		}
+
+		streamVertices.put(vertexName, vertex);
+		builtNodes.add(vertexName);
+
+		return new StreamConfig(vertex.getConfiguration());
 	}
 
-	private void chainRecursively(String chainStart, String current, String next) {
-		// We chain the next operator to the start of this chain
-		chainTasks(chainStart, next);
-		// Now recursively chain the outputs of next (depth first)
-		for (String output : outEdgeList.get(next)) {
-			if (isChainable(next, output)) {
-				// Recursive call
-				chainRecursively(chainStart, next, output);
-			} else {
-				// If not chainable we continue building the jobgraph from there
-				createVertex(output);
-			}
+	private void setVertexConfig(String vertexName, StreamConfig config,
+			List<String> chainableOutputs, List<String> nonChainableOutputs) {
+
+		StreamInvokable<?, ?> invokableObject = invokableObjects.get(vertexName);
+		byte[] outputSelector = outputSelectors.get(vertexName);
+		Class<? extends AbstractInvokable> vertexClass = vertexClasses.get(vertexName);
+		Map<String, OperatorState<?>> state = operatorStates.get(vertexName);
+
+		config.setVertexName(vertexName);
+
+		config.setBufferTimeout(bufferTimeout.get(vertexName));
+
+		config.setTypeSerializerIn1(typeSerializersIn1.get(vertexName));
+		config.setTypeSerializerIn2(typeSerializersIn2.get(vertexName));
+		config.setTypeSerializerOut1(typeSerializersOut1.get(vertexName));
+		config.setTypeSerializerOut2(typeSerializersOut2.get(vertexName));
+
+		config.setUserInvokable(invokableObject);
+		config.setOutputSelector(outputSelector);
+		config.setOperatorStates(state);
+
+		config.setNumberOfOutputs(nonChainableOutputs.size());
+		config.setOutputs(nonChainableOutputs);
+		config.setChainedOutputs(chainableOutputs);
+
+		if (vertexClass.equals(StreamIterationHead.class)
+				|| vertexClass.equals(StreamIterationTail.class)) {
+			config.setIterationId(iterationIds.get(vertexName));
+			config.setIterationWaitTime(iterationWaitTime.get(vertexName));
 		}
+
+		vertexConfigs.put(vertexName, config);
 	}
 
 	private boolean isChainable(String vertexName, String outName) {
-		return outEdgeList.get(vertexName).size() == 1
-				&& inEdgeList.get(outName).size() == 1
+		return inEdgeList.get(outName).size() == 1
+				&& invokableObjects.get(outName) != null
 				&& outputSelectors.get(vertexName) == null
 				&& invokableObjects.get(outName).getChainingStrategy() == ChainingStrategy.ALWAYS
 				&& (invokableObjects.get(vertexName).getChainingStrategy() == ChainingStrategy.HEAD || invokableObjects
 						.get(vertexName).getChainingStrategy() == ChainingStrategy.ALWAYS)
-				&& outPartitioning.get(vertexName).get(0).getStrategy() == PartitioningStrategy.FORWARD
+				&& outPartitioning.get(vertexName)
+						.get(outEdgeList.get(vertexName).indexOf(outName)).getStrategy() == PartitioningStrategy.FORWARD
 				&& vertexParallelism.get(vertexName) == vertexParallelism.get(outName) && chaining;
 	}
 
-	private void chainTasks(String first, String second) {
+	private <T> void connect(String headOfChain, Tuple2<String, String> edge) {
 
-		List<String> chained = chainedVertices.get(first);
-		if (chained == null) {
-			chained = new ArrayList<String>();
-		}
-		chained.add(second);
-		chainedVertices.put(first, chained);
-		lastInChains.put(first, second);
-
-	}
+		String upStreamVertexName = edge.f0;
+		String downStreamVertexName = edge.f1;
 
-	/**
-	 * Connects two vertices with the given names, partitioning and channel type
-	 * 
-	 * @param upStreamVertexName
-	 *            Name of the upstream vertex, that will emit the values
-	 * @param downStreamVertexName
-	 *            Name of the downstream vertex, that will receive the values
-	 * @param partitionerObject
-	 *            The partitioner
-	 */
-	private <T> void connect(String upStreamVertexName, String downStreamVertexName,
-			StreamPartitioner<T> partitionerObject) {
+		int outputIndex = outEdgeList.get(upStreamVertexName).indexOf(downStreamVertexName);
 
-		AbstractJobVertex upStreamVertex = streamVertices.get(upStreamVertexName);
+		AbstractJobVertex headVertex = streamVertices.get(headOfChain);
 		AbstractJobVertex downStreamVertex = streamVertices.get(downStreamVertexName);
 
-		StreamConfig config = new StreamConfig(upStreamVertex.getConfiguration());
+		StreamConfig downStreamConfig = new StreamConfig(downStreamVertex.getConfiguration());
+		StreamConfig upStreamConfig = new StreamConfig(headVertex.getConfiguration());
+
+		List<Integer> outEdgeIndexList = outEdgeIndex.get(upStreamVertexName);
+		int numOfInputs = downStreamConfig.getNumberOfInputs();
+
+		downStreamConfig.setInputIndex(numOfInputs++, outEdgeIndexList.get(outputIndex));
+		downStreamConfig.setNumberOfInputs(numOfInputs);
+
+		StreamPartitioner<?> partitionerObject = outPartitioning.get(upStreamVertexName).get(
+				outputIndex);
+
+		upStreamConfig.setPartitioner(downStreamVertexName, partitionerObject);
 
 		if (partitionerObject.getStrategy() == PartitioningStrategy.FORWARD) {
-			downStreamVertex
-					.connectNewDataSetAsInput(upStreamVertex, DistributionPattern.POINTWISE);
+			downStreamVertex.connectNewDataSetAsInput(headVertex, DistributionPattern.POINTWISE);
 		} else {
-			downStreamVertex.connectNewDataSetAsInput(upStreamVertex,
-					DistributionPattern.ALL_TO_ALL);
+			downStreamVertex.connectNewDataSetAsInput(headVertex, DistributionPattern.ALL_TO_ALL);
 		}
 
 		if (LOG.isDebugEnabled()) {
 			LOG.debug("CONNECTED: {} - {} -> {}", partitionerObject.getClass().getSimpleName(),
-					upStreamVertexName, downStreamVertexName);
+					headOfChain, downStreamVertexName);
 		}
 
-		int outputIndex = upStreamVertex.getNumberOfProducedIntermediateDataSets() - 1;
-
-		config.setOutputName(outputIndex, outEdgeNames.get(lastInChains.get(upStreamVertexName))
+		upStreamConfig.setOutputNames(downStreamVertexName, outEdgeNames.get(upStreamVertexName)
 				.get(outputIndex));
-		config.setSelectAll(outputIndex, outEdgeSelectAll.get(lastInChains.get(upStreamVertexName))
+		upStreamConfig.setSelectAll(downStreamVertexName, outEdgeSelectAll.get(upStreamVertexName)
 				.get(outputIndex));
-		config.setPartitioner(outputIndex, partitionerObject);
-		config.setNumberOfOutputChannels(outputIndex, vertexParallelism.get(downStreamVertexName));
 	}
 
 	/**
@@ -630,27 +636,6 @@ public class JobGraphBuilder {
 	}
 
 	/**
-	 * Writes number of inputs into each JobVertex's config
-	 */
-	private void setNumberOfJobInputs() {
-		for (AbstractJobVertex vertex : streamVertices.values()) {
-			(new StreamConfig(vertex.getConfiguration())).setNumberOfInputs(vertex
-					.getNumberOfInputs());
-		}
-	}
-
-	/**
-	 * Writes the number of outputs and output channels into each JobVertex's
-	 * config
-	 */
-	private void setNumberOfJobOutputs() {
-		for (AbstractJobVertex vertex : streamVertices.values()) {
-			(new StreamConfig(vertex.getConfiguration())).setNumberOfOutputs(vertex
-					.getNumberOfProducedIntermediateDataSets());
-		}
-	}
-
-	/**
 	 * Gets the assembled {@link JobGraph} and adds a default name for it.
 	 */
 	public JobGraph getJobGraph() {
@@ -677,33 +662,10 @@ public class JobGraphBuilder {
 	private void buildJobGraph() {
 
 		for (String sourceName : sources) {
-			createVertex(sourceName);
-		}
-
-		for (String upStreamVertexName : builtVertices) {
-			int i = 0;
-
-			List<Integer> outEdgeTypeList = outEdgeIndex.get(lastInChains.get(upStreamVertexName));
-
-			for (String downStreamVertexName : outEdgeList
-					.get(lastInChains.get(upStreamVertexName))) {
-				StreamConfig downStreamVertexConfig = new StreamConfig(streamVertices.get(
-						downStreamVertexName).getConfiguration());
-
-				int inputNumber = downStreamVertexConfig.getNumberOfInputs();
-
-				downStreamVertexConfig.setInputIndex(inputNumber++, outEdgeTypeList.get(i));
-				downStreamVertexConfig.setNumberOfInputs(inputNumber);
-
-				connect(upStreamVertexName, downStreamVertexName,
-						outPartitioning.get(lastInChains.get(upStreamVertexName)).get(i));
-				i++;
-			}
+			createChain(sourceName, sourceName);
 		}
 
 		setSlotSharing();
-		setNumberOfJobInputs();
-		setNumberOfJobOutputs();
 	}
 
 	public void setChaining(boolean chaining) {

http://git-wip-us.apache.org/repos/asf/flink/blob/7dbb55ec/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
index ada3aae..6fffaa6 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
@@ -17,16 +17,16 @@
 
 package org.apache.flink.streaming.api;
 
-import java.io.IOException;
 import java.io.Serializable;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
 import org.apache.commons.lang3.SerializationException;
 import org.apache.commons.lang3.SerializationUtils;
+import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.streaming.api.collector.OutputSelector;
-import org.apache.flink.streaming.api.invokable.ChainableInvokable;
 import org.apache.flink.streaming.api.invokable.StreamInvokable;
 import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
 import org.apache.flink.streaming.api.streamvertex.StreamVertexException;
@@ -35,17 +35,20 @@ import org.apache.flink.streaming.partitioner.StreamPartitioner;
 import org.apache.flink.streaming.state.OperatorState;
 import org.apache.flink.util.InstantiationUtil;
 
-public class StreamConfig {
+public class StreamConfig implements Serializable {
+
+	private static final long serialVersionUID = 1L;
+
 	private static final String INPUT_TYPE = "inputType_";
 	private static final String NUMBER_OF_OUTPUTS = "numberOfOutputs";
 	private static final String NUMBER_OF_INPUTS = "numberOfInputs";
-	private static final String NUMBER_OF_CHAINED_TASKS = "numOfChained";
-	private static final String CHAINED_IN_SERIALIZER = "chainedSerializer_";
-	private static final String CHAINED_INVOKABLE = "chainedInvokable_";
+	private static final String CHAINED_OUTPUTS = "chainedOutputs";
+	private static final String CHAINED_TASK_CONFIG = "chainedTaskConfig_";
+	private static final String IS_CHAINED_VERTEX = "isChainedSubtask";
 	private static final String OUTPUT_NAME = "outputName_";
 	private static final String OUTPUT_SELECT_ALL = "outputSelectAll_";
 	private static final String PARTITIONER_OBJECT = "partitionerObject_";
-	private static final String NUMBER_OF_OUTPUT_CHANNELS = "numOfOutputs_";
+	private static final String VERTEX_NAME = "vertexName";
 	private static final String ITERATION_ID = "iteration-id";
 	private static final String OUTPUT_SELECTOR = "outputSelector";
 	private static final String DIRECTED_EMIT = "directedEmit";
@@ -58,6 +61,8 @@ public class StreamConfig {
 	private static final String TYPE_SERIALIZER_OUT_1 = "typeSerializer_out_1";
 	private static final String TYPE_SERIALIZER_OUT_2 = "typeSerializer_out_2";
 	private static final String ITERATON_WAIT = "iterationWait";
+	private static final String OUTPUTS = "outVertexNames";
+	private static final String RW_ORDER = "rwOrder";
 
 	// DEFAULT VALUES
 
@@ -75,6 +80,14 @@ public class StreamConfig {
 		return config;
 	}
 
+	public void setVertexName(String vertexName) {
+		config.setString(VERTEX_NAME, vertexName);
+	}
+
+	public String getTaskName() {
+		return config.getString(VERTEX_NAME, "Missing");
+	}
+
 	public void setTypeSerializerIn1(StreamRecordSerializer<?> serializer) {
 		setTypeSerializer(TYPE_SERIALIZER_IN_1, serializer);
 	}
@@ -206,25 +219,21 @@ public class StreamConfig {
 		return config.getLong(ITERATON_WAIT, 0);
 	}
 
-	public void setNumberOfOutputChannels(int outputIndex, Integer numberOfOutputChannels) {
-		config.setInteger(NUMBER_OF_OUTPUT_CHANNELS + outputIndex, numberOfOutputChannels);
-	}
+	public <T> void setPartitioner(String output, StreamPartitioner<T> partitionerObject) {
 
-	public int getNumberOfOutputChannels(int outputIndex) {
-		return config.getInteger(NUMBER_OF_OUTPUT_CHANNELS + outputIndex, 0);
-	}
-
-	public <T> void setPartitioner(int outputIndex, StreamPartitioner<T> partitionerObject) {
-
-		config.setBytes(PARTITIONER_OBJECT + outputIndex,
+		config.setBytes(PARTITIONER_OBJECT + output,
 				SerializationUtils.serialize(partitionerObject));
 	}
 
-	public <T> StreamPartitioner<T> getPartitioner(ClassLoader cl, int outputIndex)
-			throws ClassNotFoundException, IOException {
-		@SuppressWarnings("unchecked")
-		StreamPartitioner<T> partitioner = (StreamPartitioner<T>) InstantiationUtil
-				.readObjectFromConfig(this.config, PARTITIONER_OBJECT + outputIndex, cl);
+	@SuppressWarnings("unchecked")
+	public <T> StreamPartitioner<T> getPartitioner(ClassLoader cl, String output) {
+		StreamPartitioner<T> partitioner = null;
+		try {
+			partitioner = (StreamPartitioner<T>) InstantiationUtil.readObjectFromConfig(
+					this.config, PARTITIONER_OBJECT + output, cl);
+		} catch (Exception e) {
+			throw new RuntimeException("Partitioner could not be instantiated.");
+		}
 		if (partitioner != null) {
 			return partitioner;
 		} else {
@@ -232,27 +241,27 @@ public class StreamConfig {
 		}
 	}
 
-	public void setSelectAll(int outputIndex, Boolean selectAll) {
+	public void setSelectAll(String output, Boolean selectAll) {
 		if (selectAll != null) {
-			config.setBoolean(OUTPUT_SELECT_ALL + outputIndex, selectAll);
+			config.setBoolean(OUTPUT_SELECT_ALL + output, selectAll);
 		}
 	}
 
-	public boolean isSelectAll(int outputIndex) {
-		return config.getBoolean(OUTPUT_SELECT_ALL + outputIndex, false);
+	public boolean isSelectAll(String output) {
+		return config.getBoolean(OUTPUT_SELECT_ALL + output, true);
 	}
 
-	public void setOutputName(int outputIndex, List<String> outputName) {
+	public void setOutputNames(String output, List<String> outputName) {
 		if (outputName != null) {
-			config.setBytes(OUTPUT_NAME + outputIndex,
+			config.setBytes(OUTPUT_NAME + output,
 					SerializationUtils.serialize((Serializable) outputName));
 		}
 	}
 
 	@SuppressWarnings("unchecked")
-	public List<String> getOutputNames(int outputIndex) {
-		return (List<String>) SerializationUtils.deserialize(config.getBytes(OUTPUT_NAME
-				+ outputIndex, null));
+	public List<String> getOutputNames(String output) {
+		return (List<String>) SerializationUtils.deserialize(config.getBytes(OUTPUT_NAME + output,
+				null));
 	}
 
 	public void setNumberOfInputs(int numberOfInputs) {
@@ -271,6 +280,38 @@ public class StreamConfig {
 		return config.getInteger(NUMBER_OF_OUTPUTS, 0);
 	}
 
+	public void setOutputs(List<String> outputVertexNames) {
+		config.setBytes(OUTPUTS, SerializationUtils.serialize((Serializable) outputVertexNames));
+	}
+
+	@SuppressWarnings("unchecked")
+	public List<String> getOutputs(ClassLoader cl) {
+		try {
+			return (List<String>) InstantiationUtil.readObjectFromConfig(this.config, OUTPUTS, cl);
+		} catch (Exception e) {
+			throw new RuntimeException("Could not instantiate outputs.");
+		}
+	}
+
+	public void setRecordWriterOrder(List<Tuple2<String, String>> outEdgeList) {
+
+		List<String> outVertices = new ArrayList<String>();
+		for (Tuple2<String, String> edge : outEdgeList) {
+			outVertices.add(edge.f1);
+		}
+
+		config.setBytes(RW_ORDER, SerializationUtils.serialize((Serializable) outVertices));
+	}
+
+	@SuppressWarnings("unchecked")
+	public List<String> getRecordWriterOrder(ClassLoader cl) {
+		try {
+			return (List<String>) InstantiationUtil.readObjectFromConfig(this.config, RW_ORDER, cl);
+		} catch (Exception e) {
+			throw new RuntimeException("Could not instantiate outputs.");
+		}
+	}
+
 	public void setInputIndex(int inputNumber, Integer inputTypeNumber) {
 		config.setInteger(INPUT_TYPE + inputNumber++, inputTypeNumber);
 	}
@@ -293,40 +334,77 @@ public class StreamConfig {
 		}
 	}
 
-	public int getNumberofChainedTasks() {
-		return config.getInteger(NUMBER_OF_CHAINED_TASKS, 0);
+	public void setChainedOutputs(List<String> chainedOutputs) {
+		config.setBytes(CHAINED_OUTPUTS,
+				SerializationUtils.serialize((Serializable) chainedOutputs));
 	}
 
-	public void setNumberofChainedTasks(int n) {
-		config.setInteger(NUMBER_OF_CHAINED_TASKS, n);
-	}
-
-	public ChainableInvokable<?, ?> getChainedInvokable(int chainedTaskIndex, ClassLoader cl) {
+	@SuppressWarnings("unchecked")
+	public List<String> getChainedOutputs(ClassLoader cl) {
 		try {
-			return (ChainableInvokable<?, ?>) InstantiationUtil.readObjectFromConfig(this.config,
-					CHAINED_INVOKABLE + chainedTaskIndex, cl);
+			return (List<String>) InstantiationUtil.readObjectFromConfig(this.config,
+					CHAINED_OUTPUTS, cl);
 		} catch (Exception e) {
-			throw new RuntimeException("Could not instantiate invokable.");
+			throw new RuntimeException("Could not instantiate chained outputs.");
 		}
 	}
 
-	public StreamRecordSerializer<?> getChainedInSerializer(int chainedTaskIndex, ClassLoader cl) {
+	public void setTransitiveChainedTaskConfigs(Map<String, StreamConfig> chainedTaskConfigs) {
+		config.setBytes(CHAINED_TASK_CONFIG,
+				SerializationUtils.serialize((Serializable) chainedTaskConfigs));
+	}
+
+	@SuppressWarnings("unchecked")
+	public Map<String, StreamConfig> getTransitiveChainedTaskConfigs(ClassLoader cl) {
 		try {
-			return (StreamRecordSerializer<?>) InstantiationUtil.readObjectFromConfig(this.config,
-					CHAINED_IN_SERIALIZER + chainedTaskIndex, cl);
+
+			return (Map<String, StreamConfig>) InstantiationUtil.readObjectFromConfig(this.config,
+					CHAINED_TASK_CONFIG, cl);
 		} catch (Exception e) {
-			throw new RuntimeException("Could not instantiate serializer.");
+			throw new RuntimeException("Could not instantiate configuration.");
 		}
 	}
 
-	public void setChainedSerializer(StreamRecordSerializer<?> typeWrapper, int chainedTaskIndex) {
-		config.setBytes(CHAINED_IN_SERIALIZER + chainedTaskIndex,
-				SerializationUtils.serialize(typeWrapper));
+	public void setChainStart() {
+		config.setBoolean(IS_CHAINED_VERTEX, true);
 	}
 
-	public void setChainedInvokable(ChainableInvokable<?, ?> invokable, int chainedTaskIndex) {
-		config.setBytes(CHAINED_INVOKABLE + chainedTaskIndex,
-				SerializationUtils.serialize(invokable));
+	public boolean isChainStart() {
+		return config.getBoolean(IS_CHAINED_VERTEX, false);
 	}
 
+	@Override
+	public String toString() {
+
+		ClassLoader cl = getClass().getClassLoader();
+
+		StringBuilder builder = new StringBuilder();
+		builder.append("\n=======================");
+		builder.append("Stream Config");
+		builder.append("=======================");
+		builder.append("\nTask name: " + getTaskName());
+		builder.append("\nNumber of non-chained inputs: " + getNumberOfInputs());
+		builder.append("\nNumber of non-chained outputs: " + getNumberOfOutputs());
+		builder.append("\nOutput names: " + getOutputs(cl));
+		builder.append("\nPartitioning:");
+		for (String outputname : getOutputs(cl)) {
+			builder.append("\n\t" + outputname + ": "
+					+ getPartitioner(cl, outputname).getClass().getSimpleName());
+		}
+
+		builder.append("\nChained subtasks: " + getChainedOutputs(cl));
+
+		try {
+			builder.append("\nInvokable: " + getUserInvokable(cl).getClass().getSimpleName());
+		} catch (Exception e) {
+			builder.append("\nInvokable: Missing");
+		}
+		builder.append("\nBuffer timeout: " + getBufferTimeout());
+		if (isChainStart() && getChainedOutputs(cl).size() > 0) {
+			builder.append("\n\n\n---------------------\nChained task configs\n---------------------\n");
+			builder.append(getTransitiveChainedTaskConfigs(cl)).toString();
+		}
+
+		return builder.toString();
+	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/7dbb55ec/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
index b7e57e0..a95973b 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
@@ -30,7 +30,7 @@ public class CollectorWrapper<OUT> implements Collector<OUT> {
 		this.outputs = new LinkedList<Collector<OUT>>();
 	}
 
-	public void addChainedOutput(Collector<OUT> output) {
+	public void addCollector(Collector<OUT> output) {
 		outputs.add(output);
 	}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/7dbb55ec/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java
index 4c21564..6fd1b98 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java
@@ -17,11 +17,13 @@
 
 package org.apache.flink.streaming.api.collector;
 
+import java.io.IOException;
 import java.util.List;
 
 import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
 import org.apache.flink.runtime.plugable.SerializationDelegate;
 import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.apache.flink.streaming.io.StreamRecordWriter;
 import org.apache.flink.util.Collector;
 
 public class StreamOutput<OUT> implements Collector<SerializationDelegate<StreamRecord<OUT>>> {
@@ -52,6 +54,15 @@ public class StreamOutput<OUT> implements Collector<SerializationDelegate<Stream
 
 	@Override
 	public void close() {
+		if (output instanceof StreamRecordWriter) {
+			((StreamRecordWriter<SerializationDelegate<StreamRecord<OUT>>>) output).close();
+		} else {
+			try {
+				output.flush();
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+		}
 	}
 
 	public boolean isSelectAll() {
@@ -62,4 +73,8 @@ public class StreamOutput<OUT> implements Collector<SerializationDelegate<Stream
 		return selectedNames;
 	}
 
+	public RecordWriter<SerializationDelegate<StreamRecord<OUT>>> getRecordWriter() {
+		return output;
+	}
+
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/7dbb55ec/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutputWrapper.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutputWrapper.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutputWrapper.java
index fa374b1..c3e4c9d 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutputWrapper.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutputWrapper.java
@@ -20,7 +20,6 @@ package org.apache.flink.streaming.api.collector;
 import java.util.LinkedList;
 import java.util.List;
 
-import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
 import org.apache.flink.runtime.plugable.SerializationDelegate;
 import org.apache.flink.streaming.api.streamrecord.StreamRecord;
 import org.apache.flink.util.Collector;
@@ -80,11 +79,6 @@ public class StreamOutputWrapper<OUT> implements Collector<OUT> {
 		outputs.add(output);
 	}
 
-	protected void addOneOutput(RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output,
-			List<String> outputNames, boolean isSelectAllOutput) {
-
-	}
-
 	/**
 	 * Collects and emits a tuple/object to the outputs by reusing a
 	 * StreamRecord object.

http://git-wip-us.apache.org/repos/asf/flink/blob/7dbb55ec/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
index 60a7b14..99f826d 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
@@ -19,13 +19,15 @@ package org.apache.flink.streaming.api.streamvertex;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.LinkedList;
+import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
-import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
 import org.apache.flink.runtime.plugable.SerializationDelegate;
 import org.apache.flink.streaming.api.StreamConfig;
+import org.apache.flink.streaming.api.collector.CollectorWrapper;
 import org.apache.flink.streaming.api.collector.DirectedOutputWrapper;
 import org.apache.flink.streaming.api.collector.OutputSelector;
 import org.apache.flink.streaming.api.collector.StreamOutput;
@@ -45,83 +47,142 @@ public class OutputHandler<OUT> {
 
 	private StreamVertex<?, OUT> vertex;
 	private StreamConfig configuration;
-
-	private List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> outputs;
+	private ClassLoader cl;
 	private Collector<OUT> outerCollector;
 
-	TypeInformation<OUT> outTypeInfo = null;
-	StreamRecordSerializer<OUT> outSerializer = null;
-	SerializationDelegate<StreamRecord<OUT>> outSerializationDelegate = null;
-
 	public List<ChainableInvokable<?, ?>> chainedInvokables;
 
-	private int numberOfChainedTasks;
+	private Map<String, StreamOutput<?>> outputMap;
+	private Map<String, StreamConfig> chainedConfigs;
+	private List<String> recordWriterOrder;
 
 	public OutputHandler(StreamVertex<?, OUT> vertex) {
+
+		// Initialize some fields
 		this.vertex = vertex;
-		this.outputs = new LinkedList<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>>();
 		this.configuration = new StreamConfig(vertex.getTaskConfiguration());
 		this.chainedInvokables = new ArrayList<ChainableInvokable<?, ?>>();
-		this.numberOfChainedTasks = configuration.getNumberofChainedTasks();
+		this.outputMap = new HashMap<String, StreamOutput<?>>();
+		this.cl = vertex.getUserCodeClassLoader();
+
+		// We read the chained configs, and the order of record writer
+		// registrations by outputname
+		this.chainedConfigs = configuration.getTransitiveChainedTaskConfigs(cl);
+		this.recordWriterOrder = configuration.getRecordWriterOrder(cl);
+
+		// For the network outputs of the chain head we create the stream
+		// outputs
+		for (String outName : configuration.getOutputs(cl)) {
+			StreamOutput<?> streamOutput = createStreamOutput(outName, configuration);
+			outputMap.put(outName, streamOutput);
+		}
 
-		this.outerCollector = createChainedCollector(0);
+		// If we have chained tasks we iterate through them and create the
+		// stream outputs for the network outputs
+		if (chainedConfigs != null) {
+			for (StreamConfig config : chainedConfigs.values()) {
+				for (String outName : config.getOutputs(cl)) {
+					StreamOutput<?> streamOutput = createStreamOutput(outName, config);
+					outputMap.put(outName, streamOutput);
+				}
+			}
+		}
+
+		// We create the outer collector that will be passed to the first task
+		// in the chain
+		this.outerCollector = createChainedCollector(configuration);
 
 	}
 
-	public List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> getOutputs() {
-		return outputs;
+	public Collection<StreamOutput<?>> getOutputs() {
+		return outputMap.values();
 	}
 
-	// We create the outer collector by nesting the chainable invokables into
-	// each other
+	/**
+	 * This method builds up a nested collector which encapsulates all the
+	 * chained operators and their network output. The result of this recursive
+	 * call will be passed as collector to the first invokable in the chain.
+	 * 
+	 * @param chainedTaskConfig
+	 *            The configuration of the starting operator of the chain, we
+	 *            use this paramater to recursively build the whole chain
+	 * @return Returns the collector for the chain starting from the given
+	 *         config
+	 */
 	@SuppressWarnings({ "unchecked", "rawtypes" })
-	private Collector<OUT> createChainedCollector(int chainedTaskIndex) {
+	private Collector<OUT> createChainedCollector(StreamConfig chainedTaskConfig) {
 
-		if (numberOfChainedTasks == chainedTaskIndex) {
-			// At the end of the chain we create the collector that sends data
-			// to the recordwriters
-			return createNetworkCollector();
-		} else {
+		// We create a wrapper that will encapsulate the chained operators and
+		// network outputs
+		CollectorWrapper<OUT> wrapper = new CollectorWrapper<OUT>();
 
-			ChainableInvokable chainableInvokable = configuration.getChainedInvokable(
-					chainedTaskIndex, vertex.getUserCodeClassLoader());
+		// If the task has network outputs we create a collector for those and
+		// pass
+		// it to the wrapper
+		if (chainedTaskConfig.getNumberOfOutputs() > 0) {
+			wrapper.addCollector((Collector<OUT>) createNetworkCollector(chainedTaskConfig));
+		}
 
-			// The nesting is done by calling this method recursively when
-			// passing the collector to the invokable
-			chainableInvokable.setup(
-					createChainedCollector(chainedTaskIndex + 1),
-					configuration.getChainedInSerializer(chainedTaskIndex,
-							vertex.getUserCodeClassLoader()));
+		// If the task has chained outputs we create a chained collector for
+		// each of them and pass it to the wrapper
+		for (String output : chainedTaskConfig.getChainedOutputs(cl)) {
+			wrapper.addCollector(createChainedCollector(chainedConfigs.get(output)));
+		}
 
-			// We hold a list of the chained invokables for initializaton
-			// afterwards
-			chainedInvokables.add(chainableInvokable);
+		if (chainedTaskConfig.isChainStart()) {
+			// The current task is the first chained task at this vertex so we
+			// return the wrapper
+			return wrapper;
+		} else {
+			// The current task is a part of the chain so we get the chainable
+			// invokable which will be returned and set it up using the wrapper
+			ChainableInvokable chainableInvokable = chainedTaskConfig.getUserInvokable(vertex
+					.getUserCodeClassLoader());
+			chainableInvokable.setup(wrapper,
+					chainedTaskConfig.getTypeSerializerIn1(vertex.getUserCodeClassLoader()));
 
+			chainedInvokables.add(chainableInvokable);
 			return chainableInvokable;
 		}
 
 	}
 
-	private Collector<OUT> createNetworkCollector() {
+	/**
+	 * We create the collector for the network outputs of the task represented
+	 * by the config using the StreamOutputs that we have set up in the
+	 * constructor.
+	 * 
+	 * @param config
+	 *            The config of the task
+	 * @return We return a collector that represents all the network outputs of
+	 *         this task
+	 */
+	@SuppressWarnings("unchecked")
+	private <T> Collector<T> createNetworkCollector(StreamConfig config) {
+
+		StreamRecordSerializer<T> outSerializer = config
+				.getTypeSerializerOut1(vertex.userClassLoader);
+		SerializationDelegate<StreamRecord<T>> outSerializationDelegate = null;
 
-		createOutSerializer();
+		if (outSerializer != null) {
+			outSerializationDelegate = new SerializationDelegate<StreamRecord<T>>(outSerializer);
+			outSerializationDelegate.setInstance(outSerializer.createInstance());
+		}
 
-		StreamOutputWrapper<OUT> collector;
+		StreamOutputWrapper<T> collector;
 
 		if (vertex.configuration.isDirectedEmit()) {
-			OutputSelector<OUT> outputSelector = vertex.configuration
+			OutputSelector<T> outputSelector = vertex.configuration
 					.getOutputSelector(vertex.userClassLoader);
 
-			collector = new DirectedOutputWrapper<OUT>(vertex.getInstanceID(),
+			collector = new DirectedOutputWrapper<T>(vertex.getInstanceID(),
 					outSerializationDelegate, outputSelector);
 		} else {
-			collector = new StreamOutputWrapper<OUT>(vertex.getInstanceID(),
-					outSerializationDelegate);
+			collector = new StreamOutputWrapper<T>(vertex.getInstanceID(), outSerializationDelegate);
 		}
 
-		int numberOfOutputs = configuration.getNumberOfOutputs();
-		for (int i = 0; i < numberOfOutputs; i++) {
-			collector = (StreamOutputWrapper<OUT>) addStreamOutput(i, collector);
+		for (String output : config.getOutputs(cl)) {
+			collector.addOutput((StreamOutput<T>) outputMap.get(output));
 		}
 
 		return collector;
@@ -131,32 +192,35 @@ public class OutputHandler<OUT> {
 		return outerCollector;
 	}
 
-	void createOutSerializer() {
-		outSerializer = configuration.getTypeSerializerOut1(vertex.userClassLoader);
-		if (outSerializer != null) {
-			outSerializationDelegate = new SerializationDelegate<StreamRecord<OUT>>(outSerializer);
-			outSerializationDelegate.setInstance(outSerializer.createInstance());
-		}
-	}
+	/**
+	 * We create the StreamOutput for the specific output given by the name, and
+	 * the configuration of its source task
+	 * 
+	 * @param name
+	 *            Name of the output to which the streamoutput will be set up
+	 * @param configuration
+	 *            The config of upStream task
+	 * @return
+	 */
+	private <T> StreamOutput<T> createStreamOutput(String name, StreamConfig configuration) {
 
-	Collector<OUT> addStreamOutput(int outputNumber, StreamOutputWrapper<OUT> networkCollector) {
+		int outputNumber = recordWriterOrder.indexOf(name);
 
-		StreamPartitioner<OUT> outputPartitioner;
+		StreamPartitioner<T> outputPartitioner;
 
 		try {
-			outputPartitioner = configuration.getPartitioner(vertex.userClassLoader,
-					outputNumber);
+			outputPartitioner = configuration.getPartitioner(vertex.userClassLoader, name);
 		} catch (Exception e) {
 			throw new StreamVertexException("Cannot deserialize partitioner for "
-					+ vertex.getName() + " with " + outputNumber + " outputs", e);
+					+ vertex.getName() + " with " + name + " outputs", e);
 		}
 
-		RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output;
+		RecordWriter<SerializationDelegate<StreamRecord<T>>> output;
 
 		long bufferTimeout = configuration.getBufferTimeout();
 
 		if (bufferTimeout >= 0) {
-			output = new StreamRecordWriter<SerializationDelegate<StreamRecord<OUT>>>(vertex
+			output = new StreamRecordWriter<SerializationDelegate<StreamRecord<T>>>(vertex
 					.getEnvironment().getWriter(outputNumber), outputPartitioner, bufferTimeout);
 
 			if (LOG.isTraceEnabled()) {
@@ -164,7 +228,7 @@ public class OutputHandler<OUT> {
 						bufferTimeout, vertex.getClass().getSimpleName());
 			}
 		} else {
-			output = new RecordWriter<SerializationDelegate<StreamRecord<OUT>>>(vertex
+			output = new RecordWriter<SerializationDelegate<StreamRecord<T>>>(vertex
 					.getEnvironment().getWriter(outputNumber), outputPartitioner);
 
 			if (LOG.isTraceEnabled()) {
@@ -172,34 +236,28 @@ public class OutputHandler<OUT> {
 			}
 		}
 
-		outputs.add(output);
-
-		networkCollector.addOutput(new StreamOutput<OUT>(output, configuration
-				.isSelectAll(outputNumber) ? null : configuration.getOutputNames(outputNumber)));
+		StreamOutput<T> streamOutput = new StreamOutput<T>(output,
+				configuration.isSelectAll(name) ? null : configuration.getOutputNames(name));
 
 		if (LOG.isTraceEnabled()) {
 			LOG.trace("Partitioner set: {} with {} outputs for {}", outputPartitioner.getClass()
 					.getSimpleName(), outputNumber, vertex.getClass().getSimpleName());
 		}
 
-		return networkCollector;
+		return streamOutput;
 	}
 
 	public void flushOutputs() throws IOException, InterruptedException {
-		for (RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output : outputs) {
-			if (output instanceof StreamRecordWriter) {
-				((StreamRecordWriter<SerializationDelegate<StreamRecord<OUT>>>) output).close();
-			} else {
-				output.flush();
-			}
+		for (StreamOutput<?> streamOutput : getOutputs()) {
+			streamOutput.close();
 		}
 	}
 
 	public void invokeUserFunction(String componentTypeName, StreamInvokable<?, OUT> userInvokable)
 			throws IOException, InterruptedException {
 		if (LOG.isDebugEnabled()) {
-			LOG.debug("{} {} invoked with instance id {}", componentTypeName,
-					vertex.getName(), vertex.getInstanceID());
+			LOG.debug("{} {} invoked with instance id {}", componentTypeName, vertex.getName(),
+					vertex.getInstanceID());
 		}
 
 		try {
@@ -210,8 +268,8 @@ public class OutputHandler<OUT> {
 		}
 
 		if (LOG.isDebugEnabled()) {
-			LOG.debug("{} {} invoke finished instance id {}", componentTypeName,
-					vertex.getName(), vertex.getInstanceID());
+			LOG.debug("{} {} invoke finished instance id {}", componentTypeName, vertex.getName(),
+					vertex.getInstanceID());
 		}
 
 		flushOutputs();

http://git-wip-us.apache.org/repos/asf/flink/blob/7dbb55ec/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationHead.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationHead.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationHead.java
index 43b455e..cba23b8 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationHead.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationHead.java
@@ -17,19 +17,22 @@
 
 package org.apache.flink.streaming.api.streamvertex;
 
+import java.util.LinkedList;
+import java.util.List;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.flink.api.java.tuple.Tuple;
-import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
 import org.apache.flink.runtime.plugable.SerializationDelegate;
+import org.apache.flink.streaming.api.collector.StreamOutput;
 import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
 import org.apache.flink.streaming.io.BlockingQueueBroker;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-public class StreamIterationHead<OUT extends Tuple> extends StreamVertex<OUT,OUT> {
+public class StreamIterationHead<OUT extends Tuple> extends StreamVertex<OUT, OUT> {
 
 	private static final Logger LOG = LoggerFactory.getLogger(StreamIterationHead.class);
 
@@ -72,6 +75,15 @@ public class StreamIterationHead<OUT extends Tuple> extends StreamVertex<OUT,OUT
 		}
 
 		StreamRecord<OUT> nextRecord;
+		StreamRecordSerializer<OUT> serializer = configuration
+				.getTypeSerializerOut1(userClassLoader);
+		SerializationDelegate<StreamRecord<OUT>> serializationDelegate = new SerializationDelegate<StreamRecord<OUT>>(
+				serializer);
+
+		List<StreamOutput<OUT>> outputs = new LinkedList<StreamOutput<OUT>>();
+		for (StreamOutput<?> output : outputHandler.getOutputs()) {
+			outputs.add((StreamOutput<OUT>) output);
+		}
 
 		while (true) {
 			if (shouldWait) {
@@ -82,10 +94,9 @@ public class StreamIterationHead<OUT extends Tuple> extends StreamVertex<OUT,OUT
 			if (nextRecord == null) {
 				break;
 			}
-			for (RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output : outputHandler
-					.getOutputs()) {
-				outputHandler.outSerializationDelegate.setInstance(nextRecord);
-				output.emit(outputHandler.outSerializationDelegate);
+			for (StreamOutput<OUT> output : outputs) {
+				serializationDelegate.setInstance(nextRecord);
+				output.collect(serializationDelegate);
 			}
 		}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/7dbb55ec/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
index 53b75a0..698b193 100644
--- a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
+++ b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
@@ -46,6 +46,7 @@ import org.apache.flink.streaming.api.function.aggregation.AggregationFunction
 import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.AggregationType
 import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
 import org.apache.flink.api.streaming.scala.ScalaStreamingAggregator
+import org.apache.flink.streaming.api.invokable.StreamInvokable.ChainingStrategy
 
 class DataStream[T](javaStream: JavaStream[T]) {
 
@@ -78,6 +79,15 @@ class DataStream[T](javaStream: JavaStream[T]) {
         " "  +
         "parallelism.")
   }
+  
+  def setChainingStrategy(strategy: ChainingStrategy): DataStream[T] = {
+    javaStream match {
+      case ds: SingleOutputStreamOperator[_, _] => ds.setChainingStrategy(strategy)
+      case _ =>
+        throw new UnsupportedOperationException("Only supported for operators.")
+    }
+    this
+  }
 
   /**
    * Creates a new DataStream by merging DataStream outputs of


[7/8] flink git commit: [FLINK-1345] [streaming] Added support for chaining operators with directed outputs

Posted by mb...@apache.org.
[FLINK-1345] [streaming] Added support for chaining operators with directed outputs


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/92947f0d
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/92947f0d
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/92947f0d

Branch: refs/heads/master
Commit: 92947f0d8308afa74b666e11d5ec3af08f0198c1
Parents: e3b608c
Author: Gyula Fora <gy...@apache.org>
Authored: Wed Jan 21 13:41:44 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Wed Jan 21 16:06:34 2015 +0100

----------------------------------------------------------------------
 .../flink/streaming/api/StreamConfig.java       |  23 ++-
 .../apache/flink/streaming/api/StreamGraph.java |  22 ++-
 .../api/StreamingJobGraphGenerator.java         |  18 +--
 .../api/collector/CollectorWrapper.java         |   7 +-
 .../api/collector/DirectedCollectorWrapper.java | 128 +++++++++++++++++
 .../api/collector/DirectedOutputWrapper.java    | 144 -------------------
 .../streaming/api/collector/StreamOutput.java   |  56 ++++----
 .../api/collector/StreamOutputWrapper.java      | 116 ---------------
 .../streaming/api/datastream/DataStream.java    |   5 +-
 .../api/datastream/SplitDataStream.java         |   7 +-
 .../environment/StreamExecutionEnvironment.java |   2 +-
 .../api/streamvertex/OutputHandler.java         |  95 +++++-------
 .../api/streamvertex/StreamIterationHead.java   |  12 +-
 .../api/streamvertex/StreamIterationTail.java   |   3 +-
 .../api/collector/StreamCollectorTest.java      |  96 ++++++-------
 15 files changed, 283 insertions(+), 451 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/92947f0d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
index 213a892..94349d7 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
@@ -18,6 +18,7 @@
 package org.apache.flink.streaming.api;
 
 import java.io.Serializable;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -46,7 +47,6 @@ public class StreamConfig implements Serializable {
 	private static final String CHAINED_TASK_CONFIG = "chainedTaskConfig_";
 	private static final String IS_CHAINED_VERTEX = "isChainedSubtask";
 	private static final String OUTPUT_NAME = "outputName_";
-	private static final String OUTPUT_SELECT_ALL = "outputSelectAll_";
 	private static final String PARTITIONER_OBJECT = "partitionerObject_";
 	private static final String VERTEX_NAME = "vertexName";
 	private static final String ITERATION_ID = "iteration-id";
@@ -241,25 +241,18 @@ public class StreamConfig implements Serializable {
 		}
 	}
 
-	public void setSelectAll(String output, Boolean selectAll) {
-		if (selectAll != null) {
-			config.setBoolean(OUTPUT_SELECT_ALL + output, selectAll);
-		}
-	}
-
-	public boolean isSelectAll(String output) {
-		return config.getBoolean(OUTPUT_SELECT_ALL + output, true);
-	}
-
-	public void setOutputNames(String output, List<String> outputName) {
-		if (outputName != null) {
+	public void setSelectedNames(String output, List<String> selected) {
+		if (selected != null) {
+			config.setBytes(OUTPUT_NAME + output,
+					SerializationUtils.serialize((Serializable) selected));
+		} else {
 			config.setBytes(OUTPUT_NAME + output,
-					SerializationUtils.serialize((Serializable) outputName));
+					SerializationUtils.serialize((Serializable) new ArrayList<String>()));
 		}
 	}
 
 	@SuppressWarnings("unchecked")
-	public List<String> getOutputNames(String output) {
+	public List<String> getSelectedNames(String output) {
 		return (List<String>) SerializationUtils.deserialize(config.getBytes(OUTPUT_NAME + output,
 				null));
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/92947f0d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamGraph.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamGraph.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamGraph.java
index 3dd1bdb..c9ecd55 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamGraph.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamGraph.java
@@ -58,7 +58,6 @@ public class StreamGraph {
 	private Map<String, List<String>> outEdgeLists;
 	private Map<String, List<Integer>> outEdgeTypes;
 	private Map<String, List<List<String>>> selectedNames;
-	private Map<String, List<Boolean>> outEdgeSelectAlls;
 	private Map<String, List<String>> inEdgeLists;
 	private Map<String, List<StreamPartitioner<?>>> outputPartitioners;
 	private Map<String, String> operatorNames;
@@ -94,7 +93,6 @@ public class StreamGraph {
 		outEdgeLists = new HashMap<String, List<String>>();
 		outEdgeTypes = new HashMap<String, List<Integer>>();
 		selectedNames = new HashMap<String, List<List<String>>>();
-		outEdgeSelectAlls = new HashMap<String, List<Boolean>>();
 		inEdgeLists = new HashMap<String, List<String>>();
 		outputPartitioners = new HashMap<String, List<StreamPartitioner<?>>>();
 		operatorNames = new HashMap<String, String>();
@@ -186,7 +184,7 @@ public class StreamGraph {
 
 		setEdge(vertexName, iterationHead,
 				outputPartitioners.get(inEdgeLists.get(iterationHead).get(0)).get(0), 0,
-				new ArrayList<String>(), false);
+				new ArrayList<String>());
 
 		iterationTimeouts.put(iterationIDtoHeadName.get(iterationID), waitTime);
 
@@ -274,7 +272,6 @@ public class StreamGraph {
 		outEdgeLists.put(vertexName, new ArrayList<String>());
 		outEdgeTypes.put(vertexName, new ArrayList<Integer>());
 		selectedNames.put(vertexName, new ArrayList<List<String>>());
-		outEdgeSelectAlls.put(vertexName, new ArrayList<Boolean>());
 		inEdgeLists.put(vertexName, new ArrayList<String>());
 		outputPartitioners.put(vertexName, new ArrayList<StreamPartitioner<?>>());
 		iterationTailCount.put(vertexName, 0);
@@ -296,14 +293,12 @@ public class StreamGraph {
 	 *            User defined names of the out edge
 	 */
 	public void setEdge(String upStreamVertexName, String downStreamVertexName,
-			StreamPartitioner<?> partitionerObject, int typeNumber, List<String> outputNames,
-			boolean selectAll) {
+			StreamPartitioner<?> partitionerObject, int typeNumber, List<String> outputNames) {
 		outEdgeLists.get(upStreamVertexName).add(downStreamVertexName);
 		outEdgeTypes.get(upStreamVertexName).add(typeNumber);
 		inEdgeLists.get(downStreamVertexName).add(upStreamVertexName);
 		outputPartitioners.get(upStreamVertexName).add(partitionerObject);
 		selectedNames.get(upStreamVertexName).add(outputNames);
-		outEdgeSelectAlls.get(upStreamVertexName).add(selectAll);
 	}
 
 	private void addTypeSerializers(String vertexName, StreamRecordSerializer<?> in1,
@@ -494,16 +489,15 @@ public class StreamGraph {
 		return outEdgeTypes.get(vertexName);
 	}
 
-	public StreamPartitioner<?> getOutPartitioner(String vertexName, int outputIndex) {
-		return outputPartitioners.get(vertexName).get(outputIndex);
+	public StreamPartitioner<?> getOutPartitioner(String upStreamVertex, String downStreamVertex) {
+		return outputPartitioners.get(upStreamVertex).get(
+				outEdgeLists.get(upStreamVertex).indexOf(downStreamVertex));
 	}
 
-	public List<String> getSelectedNames(String vertexName, int outputIndex) {
-		return selectedNames.get(vertexName).get(outputIndex);
-	}
+	public List<String> getSelectedNames(String upStreamVertex, String downStreamVertex) {
 
-	public Boolean isSelectAll(String vertexName, int outputIndex) {
-		return outEdgeSelectAlls.get(vertexName).get(outputIndex);
+		return selectedNames.get(upStreamVertex).get(
+				outEdgeLists.get(upStreamVertex).indexOf(downStreamVertex));
 	}
 
 	public Collection<Integer> getIterationIDs() {

http://git-wip-us.apache.org/repos/asf/flink/blob/92947f0d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamingJobGraphGenerator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamingJobGraphGenerator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamingJobGraphGenerator.java
index 6c0cc20..3b2d135 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamingJobGraphGenerator.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamingJobGraphGenerator.java
@@ -206,6 +206,13 @@ public class StreamingJobGraphGenerator {
 			config.setIterationWaitTime(streamGraph.getIterationTimeout(vertexName));
 		}
 
+		List<String> allOutputs = new ArrayList<String>(chainableOutputs);
+		allOutputs.addAll(nonChainableOutputs);
+
+		for (String output : allOutputs) {
+			config.setSelectedNames(output, streamGraph.getSelectedNames(vertexName, output));
+		}
+
 		vertexConfigs.put(vertexName, config);
 	}
 
@@ -229,7 +236,7 @@ public class StreamingJobGraphGenerator {
 		downStreamConfig.setNumberOfInputs(numOfInputs);
 
 		StreamPartitioner<?> partitioner = streamGraph.getOutPartitioner(upStreamVertexName,
-				outputIndex);
+				downStreamVertexName);
 
 		upStreamConfig.setPartitioner(downStreamVertexName, partitioner);
 
@@ -243,11 +250,6 @@ public class StreamingJobGraphGenerator {
 			LOG.debug("CONNECTED: {} - {} -> {}", partitioner.getClass().getSimpleName(),
 					headOfChain, downStreamVertexName);
 		}
-
-		upStreamConfig.setOutputNames(downStreamVertexName,
-				streamGraph.getSelectedNames(upStreamVertexName, outputIndex));
-		upStreamConfig.setSelectAll(downStreamVertexName,
-				streamGraph.isSelectAll(upStreamVertexName, outputIndex));
 	}
 
 	private boolean isChainable(String vertexName, String outName) {
@@ -257,12 +259,10 @@ public class StreamingJobGraphGenerator {
 
 		return streamGraph.getInEdges(outName).size() == 1
 				&& outInvokable != null
-				&& streamGraph.getOutputSelector(vertexName) == null
 				&& outInvokable.getChainingStrategy() == ChainingStrategy.ALWAYS
 				&& (headInvokable.getChainingStrategy() == ChainingStrategy.HEAD || headInvokable
 						.getChainingStrategy() == ChainingStrategy.ALWAYS)
-				&& streamGraph.getOutPartitioner(vertexName,
-						streamGraph.getOutEdges(vertexName).indexOf(outName)).getStrategy() == PartitioningStrategy.FORWARD
+				&& streamGraph.getOutPartitioner(vertexName, outName).getStrategy() == PartitioningStrategy.FORWARD
 				&& streamGraph.getParallelism(vertexName) == streamGraph.getParallelism(outName)
 				&& streamGraph.chaining;
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/92947f0d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
index a95973b..1281bf0 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
@@ -24,14 +24,15 @@ import org.apache.flink.util.Collector;
 
 public class CollectorWrapper<OUT> implements Collector<OUT> {
 
-	List<Collector<OUT>> outputs;
+	private List<Collector<OUT>> outputs;
 
 	public CollectorWrapper() {
 		this.outputs = new LinkedList<Collector<OUT>>();
 	}
 
-	public void addCollector(Collector<OUT> output) {
-		outputs.add(output);
+	@SuppressWarnings("unchecked")
+	public void addCollector(Collector<?> output) {
+		outputs.add((Collector<OUT>) output);
 	}
 
 	@Override

http://git-wip-us.apache.org/repos/asf/flink/blob/92947f0d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedCollectorWrapper.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedCollectorWrapper.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedCollectorWrapper.java
new file mode 100755
index 0000000..66fb667
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedCollectorWrapper.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.collector;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.flink.util.Collector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A StreamCollector that uses user defined output names and a user defined
+ * output selector to make directed emits.
+ * 
+ * @param <OUT>
+ *            Type of the Tuple collected.
+ */
+public class DirectedCollectorWrapper<OUT> extends CollectorWrapper<OUT> {
+
+	private static final Logger LOG = LoggerFactory.getLogger(DirectedCollectorWrapper.class);
+
+	OutputSelector<OUT> outputSelector;
+
+	protected Map<String, List<Collector<OUT>>> outputMap;
+
+	private List<Collector<OUT>> selectAllOutputs;
+	private Set<Collector<OUT>> emitted;
+
+	/**
+	 * Creates a new DirectedStreamCollector
+	 * 
+	 * @param outputSelector
+	 *            User defined {@link OutputSelector}
+	 */
+	public DirectedCollectorWrapper(OutputSelector<OUT> outputSelector) {
+		this.outputSelector = outputSelector;
+		this.emitted = new HashSet<Collector<OUT>>();
+		this.selectAllOutputs = new LinkedList<Collector<OUT>>();
+		this.outputMap = new HashMap<String, List<Collector<OUT>>>();
+
+	}
+
+	@Override
+	public void addCollector(Collector<?> output) {
+		addCollector(output, new ArrayList<String>());
+	}
+
+	@SuppressWarnings("unchecked")
+	public void addCollector(Collector<?> output, List<String> selectedNames) {
+
+		if (selectedNames.isEmpty()) {
+			selectAllOutputs.add((Collector<OUT>) output);
+		} else {
+			for (String selectedName : selectedNames) {
+
+				if (!outputMap.containsKey(selectedName)) {
+					outputMap.put(selectedName, new LinkedList<Collector<OUT>>());
+					outputMap.get(selectedName).add((Collector<OUT>) output);
+				} else {
+					if (!outputMap.get(selectedName).contains(output)) {
+						outputMap.get(selectedName).add((Collector<OUT>) output);
+					}
+				}
+
+			}
+		}
+	}
+
+	@Override
+	public void collect(OUT record) {
+		emitted.clear();
+
+		Iterable<String> outputNames = outputSelector.select(record);
+
+		for (Collector<OUT> output : selectAllOutputs) {
+			output.collect(record);
+			emitted.add(output);
+		}
+
+		for (String outputName : outputNames) {
+			List<Collector<OUT>> outputList = outputMap.get(outputName);
+			if (outputList == null) {
+				if (LOG.isErrorEnabled()) {
+					String format = String.format(
+							"Cannot emit because no output is selected with the name: %s",
+							outputName);
+					LOG.error(format);
+
+				}
+			} else {
+				for (Collector<OUT> output : outputList) {
+					if (!emitted.contains(output)) {
+						output.collect(record);
+						emitted.add(output);
+					}
+				}
+
+			}
+
+		}
+	}
+
+	@Override
+	public void close() {
+
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/92947f0d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedOutputWrapper.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedOutputWrapper.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedOutputWrapper.java
deleted file mode 100755
index 6ff03c4..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedOutputWrapper.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.collector;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.flink.runtime.plugable.SerializationDelegate;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.apache.flink.util.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * A StreamCollector that uses user defined output names and a user defined
- * output selector to make directed emits.
- * 
- * @param <OUT>
- *            Type of the Tuple collected.
- */
-public class DirectedOutputWrapper<OUT> extends StreamOutputWrapper<OUT> {
-
-	private static final Logger LOG = LoggerFactory.getLogger(DirectedOutputWrapper.class);
-
-	OutputSelector<OUT> outputSelector;
-
-	protected Map<String, List<StreamOutput<OUT>>> outputMap;
-
-	private List<StreamOutput<OUT>> selectAllOutputs;
-	private Set<StreamOutput<OUT>> emitted;
-
-	/**
-	 * Creates a new DirectedStreamCollector
-	 * 
-	 * @param channelID
-	 *            Channel ID of the Task
-	 * @param serializationDelegate
-	 *            Serialization delegate used for serialization
-	 * @param outputSelector
-	 *            User defined {@link OutputSelector}
-	 */
-	public DirectedOutputWrapper(int channelID,
-			SerializationDelegate<StreamRecord<OUT>> serializationDelegate,
-			OutputSelector<OUT> outputSelector) {
-		super(channelID, serializationDelegate);
-		this.outputSelector = outputSelector;
-		this.emitted = new HashSet<StreamOutput<OUT>>();
-		this.selectAllOutputs = new LinkedList<StreamOutput<OUT>>();
-		this.outputMap = new HashMap<String, List<StreamOutput<OUT>>>();
-
-	}
-
-	@Override
-	public void addOutput(StreamOutput<OUT> output) {
-
-		super.addOutput(output);
-
-		if (output.isSelectAll()) {
-			selectAllOutputs.add(output);
-		} else {
-			for (String selectedName : output.getSelectedNames()) {
-				if (selectedName != null) {
-					if (!outputMap.containsKey(selectedName)) {
-						outputMap.put(selectedName, new LinkedList<StreamOutput<OUT>>());
-						outputMap.get(selectedName).add(output);
-					} else {
-						if (!outputMap.get(selectedName).contains(output)) {
-							outputMap.get(selectedName).add(output);
-						}
-					}
-
-				}
-			}
-		}
-	}
-
-	@Override
-	protected void emit() {
-		Iterable<String> outputNames = outputSelector.select(serializationDelegate.getInstance()
-				.getObject());
-		emitted.clear();
-
-		for (StreamOutput<OUT> output : selectAllOutputs) {
-			try {
-				output.collect(serializationDelegate);
-			} catch (Exception e) {
-				if (LOG.isErrorEnabled()) {
-					LOG.error("Emit to {} failed due to: {}", output,
-							StringUtils.stringifyException(e));
-				}
-			}
-		}
-
-		for (String outputName : outputNames) {
-			List<StreamOutput<OUT>> outputList = outputMap.get(outputName);
-			try {
-				if (outputList == null) {
-					if (LOG.isErrorEnabled()) {
-						String format = String.format(
-								"Cannot emit because no output is selected with the name: %s",
-								outputName);
-						LOG.error(format);
-
-					}
-				} else {
-
-					for (StreamOutput<OUT> output : outputList) {
-						if (!emitted.contains(output)) {
-							output.collect(serializationDelegate);
-							emitted.add(output);
-						}
-					}
-
-				}
-
-			} catch (Exception e) {
-				if (LOG.isErrorEnabled()) {
-					LOG.error("Emit to {} failed due to: {}", outputName,
-							StringUtils.stringifyException(e));
-				}
-			}
-
-		}
-	}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/92947f0d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java
index 6fd1b98..4551c5a 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java
@@ -18,37 +18,55 @@
 package org.apache.flink.streaming.api.collector;
 
 import java.io.IOException;
-import java.util.List;
 
 import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
 import org.apache.flink.runtime.plugable.SerializationDelegate;
 import org.apache.flink.streaming.api.streamrecord.StreamRecord;
 import org.apache.flink.streaming.io.StreamRecordWriter;
 import org.apache.flink.util.Collector;
+import org.apache.flink.util.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
-public class StreamOutput<OUT> implements Collector<SerializationDelegate<StreamRecord<OUT>>> {
+public class StreamOutput<OUT> implements Collector<OUT> {
 
-	private RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output;
+	private static final Logger LOG = LoggerFactory.getLogger(StreamOutput.class);
 
-	private List<String> selectedNames;
-	private boolean selectAll = true;
+	private RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output;
+	private SerializationDelegate<StreamRecord<OUT>> serializationDelegate;
+	private StreamRecord<OUT> streamRecord;
+	private int channelID;
 
 	public StreamOutput(RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output,
-			List<String> selectedNames) {
+			int channelID, SerializationDelegate<StreamRecord<OUT>> serializationDelegate) {
 
-		this.output = output;
+		this.serializationDelegate = serializationDelegate;
 
-		if (selectedNames != null) {
-			this.selectedNames = selectedNames;
-			selectAll = false;
+		if (serializationDelegate != null) {
+			this.streamRecord = serializationDelegate.getInstance();
+		} else {
+			throw new RuntimeException("Serializer cannot be null");
 		}
+		this.channelID = channelID;
+		this.output = output;
 	}
 
-	public void collect(SerializationDelegate<StreamRecord<OUT>> record) {
+	public RecordWriter<SerializationDelegate<StreamRecord<OUT>>> getRecordWriter() {
+		return output;
+	}
+
+	@Override
+	public void collect(OUT record) {
+		streamRecord.setObject(record);
+		streamRecord.newId(channelID);
+		serializationDelegate.setInstance(streamRecord);
+
 		try {
-			output.emit(record);
+			output.emit(serializationDelegate);
 		} catch (Exception e) {
-			throw new RuntimeException("Could not emit record: " + record.getInstance());
+			if (LOG.isErrorEnabled()) {
+				LOG.error("Emit failed due to: {}", StringUtils.stringifyException(e));
+			}
 		}
 	}
 
@@ -65,16 +83,4 @@ public class StreamOutput<OUT> implements Collector<SerializationDelegate<Stream
 		}
 	}
 
-	public boolean isSelectAll() {
-		return selectAll;
-	}
-
-	public List<String> getSelectedNames() {
-		return selectedNames;
-	}
-
-	public RecordWriter<SerializationDelegate<StreamRecord<OUT>>> getRecordWriter() {
-		return output;
-	}
-
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/92947f0d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutputWrapper.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutputWrapper.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutputWrapper.java
deleted file mode 100755
index c3e4c9d..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutputWrapper.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.collector;
-
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.flink.runtime.plugable.SerializationDelegate;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.apache.flink.util.Collector;
-import org.apache.flink.util.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Collector for tuples in Apache Flink stream processing. The collected values
- * will be wrapped with ID in a {@link StreamRecord} and then emitted to the
- * outputs.
- * 
- * @param <OUT>
- *            Type of the Tuples/Objects collected.
- */
-public class StreamOutputWrapper<OUT> implements Collector<OUT> {
-
-	private static final Logger LOG = LoggerFactory.getLogger(StreamOutputWrapper.class);
-
-	protected StreamRecord<OUT> streamRecord;
-	protected int channelID;
-	protected List<StreamOutput<OUT>> outputs;
-	protected SerializationDelegate<StreamRecord<OUT>> serializationDelegate;
-
-	/**
-	 * Creates a new StreamCollector
-	 * 
-	 * @param channelID
-	 *            Channel ID of the Task
-	 * @param serializationDelegate
-	 *            Serialization delegate used for serialization
-	 */
-	public StreamOutputWrapper(int channelID,
-			SerializationDelegate<StreamRecord<OUT>> serializationDelegate) {
-		this.serializationDelegate = serializationDelegate;
-
-		if (serializationDelegate != null) {
-			this.streamRecord = serializationDelegate.getInstance();
-		} else {
-			this.streamRecord = new StreamRecord<OUT>();
-		}
-		this.channelID = channelID;
-		this.outputs = new LinkedList<StreamOutput<OUT>>();
-	}
-
-	/**
-	 * Adds an output with the given user defined name
-	 * 
-	 * @param output
-	 *            The RecordWriter object representing the output.
-	 * @param outputNames
-	 *            User defined names of the output.
-	 * @param isSelectAllOutput
-	 *            Marks whether all the outputs are selected.
-	 */
-	public void addOutput(StreamOutput<OUT> output) {
-		outputs.add(output);
-	}
-
-	/**
-	 * Collects and emits a tuple/object to the outputs by reusing a
-	 * StreamRecord object.
-	 * 
-	 * @param outputObject
-	 *            Object to be collected and emitted.
-	 */
-	@Override
-	public void collect(OUT outputObject) {
-		streamRecord.setObject(outputObject);
-		streamRecord.newId(channelID);
-		serializationDelegate.setInstance(streamRecord);
-
-		emit();
-	}
-
-	/**
-	 * Emits the current streamrecord to the outputs.
-	 */
-	protected void emit() {
-		for (StreamOutput<OUT> output : outputs) {
-			try {
-				output.collect(serializationDelegate);
-			} catch (Exception e) {
-				if (LOG.isErrorEnabled()) {
-					LOG.error("Emit failed due to: {}", StringUtils.stringifyException(e));
-				}
-			}
-		}
-	}
-
-	@Override
-	public void close() {
-	}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/92947f0d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index 8e7d823..0f633ec 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -96,7 +96,6 @@ public class DataStream<OUT> {
 	protected final String id;
 	protected int degreeOfParallelism;
 	protected List<String> userDefinedNames;
-	protected boolean selectAll;
 	protected StreamPartitioner<OUT> partitioner;
 	@SuppressWarnings("rawtypes")
 	protected TypeInformation typeInfo;
@@ -127,7 +126,6 @@ public class DataStream<OUT> {
 		this.degreeOfParallelism = environment.getDegreeOfParallelism();
 		this.streamGraph = environment.getStreamGraph();
 		this.userDefinedNames = new ArrayList<String>();
-		this.selectAll = true;
 		this.partitioner = new DistributePartitioner<OUT>(true);
 		this.typeInfo = typeInfo;
 		this.mergedStreams = new ArrayList<DataStream<OUT>>();
@@ -145,7 +143,6 @@ public class DataStream<OUT> {
 		this.id = dataStream.id;
 		this.degreeOfParallelism = dataStream.degreeOfParallelism;
 		this.userDefinedNames = new ArrayList<String>(dataStream.userDefinedNames);
-		this.selectAll = dataStream.selectAll;
 		this.partitioner = dataStream.partitioner;
 		this.streamGraph = dataStream.streamGraph;
 		this.typeInfo = dataStream.typeInfo;
@@ -1158,7 +1155,7 @@ public class DataStream<OUT> {
 	protected <X> void connectGraph(DataStream<X> inputStream, String outputID, int typeNumber) {
 		for (DataStream<X> stream : inputStream.mergedStreams) {
 			streamGraph.setEdge(stream.getId(), outputID, stream.partitioner, typeNumber,
-					inputStream.userDefinedNames, inputStream.selectAll);
+					inputStream.userDefinedNames);
 		}
 
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/92947f0d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SplitDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SplitDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SplitDataStream.java
index 1bf4f9c..2b5b7c5 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SplitDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SplitDataStream.java
@@ -49,8 +49,13 @@ public class SplitDataStream<OUT> extends DataStream<OUT> {
 	}
 
 	private DataStream<OUT> selectOutput(String[] outputNames) {
+		for (String outName : outputNames) {
+			if (outName == null) {
+				throw new RuntimeException("Selected names must not be null");
+			}
+		}
+
 		DataStream<OUT> returnStream = copy();
-		returnStream.selectAll = false;
 		returnStream.userDefinedNames = Arrays.asList(outputNames);
 		return returnStream;
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/92947f0d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
index 51dc0ae..8a9be93 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
@@ -313,7 +313,7 @@ public abstract class StreamExecutionEnvironment {
 	 */
 	public DataStreamSource<String> socketTextStream(String hostname, int port, char delimiter) {
 		return addSource(new SocketTextStreamFunction(hostname, port, delimiter), null,
-				"socketStrean");
+				"socketStream");
 	}
 
 	/**

http://git-wip-us.apache.org/repos/asf/flink/blob/92947f0d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
index 135f742..0d60939 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
@@ -29,10 +29,8 @@ import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
 import org.apache.flink.runtime.plugable.SerializationDelegate;
 import org.apache.flink.streaming.api.StreamConfig;
 import org.apache.flink.streaming.api.collector.CollectorWrapper;
-import org.apache.flink.streaming.api.collector.DirectedOutputWrapper;
-import org.apache.flink.streaming.api.collector.OutputSelector;
+import org.apache.flink.streaming.api.collector.DirectedCollectorWrapper;
 import org.apache.flink.streaming.api.collector.StreamOutput;
-import org.apache.flink.streaming.api.collector.StreamOutputWrapper;
 import org.apache.flink.streaming.api.invokable.ChainableInvokable;
 import org.apache.flink.streaming.api.invokable.StreamInvokable;
 import org.apache.flink.streaming.api.streamrecord.StreamRecord;
@@ -70,7 +68,7 @@ public class OutputHandler<OUT> {
 		// registrations by outputname
 		this.chainedConfigs = configuration.getTransitiveChainedTaskConfigs(cl);
 		this.chainedConfigs.put(configuration.getTaskName(), configuration);
-				
+
 		this.outEdgesInOrder = configuration.getOutEdgesInOrder(cl);
 
 		// We iterate through all the out edges from this job vertex and create
@@ -105,21 +103,35 @@ public class OutputHandler<OUT> {
 	@SuppressWarnings({ "unchecked", "rawtypes" })
 	private Collector<OUT> createChainedCollector(StreamConfig chainedTaskConfig) {
 
+		boolean isDirectEmit = chainedTaskConfig.isDirectedEmit();
+
 		// We create a wrapper that will encapsulate the chained operators and
 		// network outputs
-		CollectorWrapper<OUT> wrapper = new CollectorWrapper<OUT>();
+		CollectorWrapper<OUT> wrapper = isDirectEmit ? new DirectedCollectorWrapper(
+				chainedTaskConfig.getOutputSelector(cl)) : new CollectorWrapper<OUT>();
+
+		// Create collectors for the network outputs
+		for (String output : chainedTaskConfig.getOutputs(cl)) {
 
-		// If the task has network outputs we create a collector for those and
-		// pass
-		// it to the wrapper
-		if (chainedTaskConfig.getNumberOfOutputs() > 0) {
-			wrapper.addCollector((Collector<OUT>) createNetworkCollector(chainedTaskConfig));
+			Collector<?> outCollector = outputMap.get(output);
+
+			if (isDirectEmit) {
+				((DirectedCollectorWrapper<OUT>) wrapper).addCollector(outCollector,
+						chainedTaskConfig.getSelectedNames(output));
+			} else {
+				wrapper.addCollector(outCollector);
+			}
 		}
 
-		// If the task has chained outputs we create a chained collector for
-		// each of them and pass it to the wrapper
+		// Create collectors for the chained outputs
 		for (String output : chainedTaskConfig.getChainedOutputs(cl)) {
-			wrapper.addCollector(createChainedCollector(chainedConfigs.get(output)));
+			Collector<?> outCollector = createChainedCollector(chainedConfigs.get(output));
+			if (isDirectEmit) {
+				((DirectedCollectorWrapper<OUT>) wrapper).addCollector(outCollector,
+						chainedTaskConfig.getSelectedNames(output));
+			} else {
+				wrapper.addCollector(outCollector);
+			}
 		}
 
 		if (chainedTaskConfig.isChainStart()) {
@@ -140,47 +152,6 @@ public class OutputHandler<OUT> {
 
 	}
 
-	/**
-	 * We create the collector for the network outputs of the task represented
-	 * by the config using the StreamOutputs that we have set up in the
-	 * constructor.
-	 * 
-	 * @param config
-	 *            The config of the task
-	 * @return We return a collector that represents all the network outputs of
-	 *         this task
-	 */
-	@SuppressWarnings("unchecked")
-	private <T> Collector<T> createNetworkCollector(StreamConfig config) {
-
-		StreamRecordSerializer<T> outSerializer = config
-				.getTypeSerializerOut1(vertex.userClassLoader);
-		SerializationDelegate<StreamRecord<T>> outSerializationDelegate = null;
-
-		if (outSerializer != null) {
-			outSerializationDelegate = new SerializationDelegate<StreamRecord<T>>(outSerializer);
-			outSerializationDelegate.setInstance(outSerializer.createInstance());
-		}
-
-		StreamOutputWrapper<T> collector;
-
-		if (vertex.configuration.isDirectedEmit()) {
-			OutputSelector<T> outputSelector = vertex.configuration
-					.getOutputSelector(vertex.userClassLoader);
-
-			collector = new DirectedOutputWrapper<T>(vertex.getInstanceID(),
-					outSerializationDelegate, outputSelector);
-		} else {
-			collector = new StreamOutputWrapper<T>(vertex.getInstanceID(), outSerializationDelegate);
-		}
-
-		for (String output : config.getOutputs(cl)) {
-			collector.addOutput((StreamOutput<T>) outputMap.get(output));
-		}
-
-		return collector;
-	}
-
 	public Collector<OUT> getCollector() {
 		return outerCollector;
 	}
@@ -197,7 +168,16 @@ public class OutputHandler<OUT> {
 	 */
 	private <T> StreamOutput<T> createStreamOutput(String outputVertex, StreamConfig configuration,
 			int outputIndex) {
-		
+
+		StreamRecordSerializer<T> outSerializer = configuration
+				.getTypeSerializerOut1(vertex.userClassLoader);
+		SerializationDelegate<StreamRecord<T>> outSerializationDelegate = null;
+
+		if (outSerializer != null) {
+			outSerializationDelegate = new SerializationDelegate<StreamRecord<T>>(outSerializer);
+			outSerializationDelegate.setInstance(outSerializer.createInstance());
+		}
+
 		StreamPartitioner<T> outputPartitioner = configuration.getPartitioner(cl, outputVertex);
 
 		RecordWriter<SerializationDelegate<StreamRecord<T>>> output;
@@ -221,9 +201,8 @@ public class OutputHandler<OUT> {
 			}
 		}
 
-		StreamOutput<T> streamOutput = new StreamOutput<T>(output,
-				configuration.isSelectAll(outputVertex) ? null
-						: configuration.getOutputNames(outputVertex));
+		StreamOutput<T> streamOutput = new StreamOutput<T>(output, vertex.instanceID,
+				outSerializationDelegate);
 
 		if (LOG.isTraceEnabled()) {
 			LOG.trace("Partitioner set: {} with {} outputs for {}", outputPartitioner.getClass()

http://git-wip-us.apache.org/repos/asf/flink/blob/92947f0d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationHead.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationHead.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationHead.java
index cba23b8..bc89241 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationHead.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationHead.java
@@ -23,16 +23,13 @@ import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.flink.api.java.tuple.Tuple;
-import org.apache.flink.runtime.plugable.SerializationDelegate;
 import org.apache.flink.streaming.api.collector.StreamOutput;
 import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
 import org.apache.flink.streaming.io.BlockingQueueBroker;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-public class StreamIterationHead<OUT extends Tuple> extends StreamVertex<OUT, OUT> {
+public class StreamIterationHead<OUT> extends StreamVertex<OUT, OUT> {
 
 	private static final Logger LOG = LoggerFactory.getLogger(StreamIterationHead.class);
 
@@ -75,10 +72,6 @@ public class StreamIterationHead<OUT extends Tuple> extends StreamVertex<OUT, OU
 		}
 
 		StreamRecord<OUT> nextRecord;
-		StreamRecordSerializer<OUT> serializer = configuration
-				.getTypeSerializerOut1(userClassLoader);
-		SerializationDelegate<StreamRecord<OUT>> serializationDelegate = new SerializationDelegate<StreamRecord<OUT>>(
-				serializer);
 
 		List<StreamOutput<OUT>> outputs = new LinkedList<StreamOutput<OUT>>();
 		for (StreamOutput<?> output : outputHandler.getOutputs()) {
@@ -95,8 +88,7 @@ public class StreamIterationHead<OUT extends Tuple> extends StreamVertex<OUT, OU
 				break;
 			}
 			for (StreamOutput<OUT> output : outputs) {
-				serializationDelegate.setInstance(nextRecord);
-				output.collect(serializationDelegate);
+				output.collect(nextRecord.getObject());
 			}
 		}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/92947f0d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationTail.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationTail.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationTail.java
index 1883c06..b3ecdf8 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationTail.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationTail.java
@@ -20,14 +20,13 @@ package org.apache.flink.streaming.api.streamvertex;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.flink.api.java.tuple.Tuple;
 import org.apache.flink.streaming.api.streamrecord.StreamRecord;
 import org.apache.flink.streaming.io.BlockingQueueBroker;
 import org.apache.flink.util.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-public class StreamIterationTail<IN extends Tuple> extends StreamVertex<IN, IN> {
+public class StreamIterationTail<IN> extends StreamVertex<IN, IN> {
 
 	private static final Logger LOG = LoggerFactory.getLogger(StreamIterationTail.class);
 

http://git-wip-us.apache.org/repos/asf/flink/blob/92947f0d/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/StreamCollectorTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/StreamCollectorTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/StreamCollectorTest.java
index 0bc3d7d..49b3bf8 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/StreamCollectorTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/StreamCollectorTest.java
@@ -1,54 +1,52 @@
 /*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
  * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.collector;
-
-import static org.junit.Assert.assertArrayEquals;
+ */
 
-import java.util.ArrayList;
+package org.apache.flink.streaming.api.collector;
 
-import org.apache.flink.api.java.tuple.Tuple1;
-import org.apache.flink.runtime.plugable.SerializationDelegate;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import static org.junit.Assert.assertArrayEquals;
+
+import org.apache.flink.api.java.tuple.Tuple1;
+import org.apache.flink.runtime.plugable.SerializationDelegate;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
 import org.apache.flink.streaming.api.streamvertex.MockRecordWriter;
-import org.apache.flink.streaming.util.MockRecordWriterFactory;
-import org.junit.Test;
-
-public class StreamCollectorTest {
-
-	@Test
-	public void testCollect() {
-		MockRecordWriter recWriter = MockRecordWriterFactory.create();
-		SerializationDelegate<StreamRecord<Tuple1<Integer>>> sd = new SerializationDelegate<StreamRecord<Tuple1<Integer>>>(
-				null);
-		sd.setInstance(new StreamRecord<Tuple1<Integer>>().setObject(new Tuple1<Integer>()));
-
-		StreamOutputWrapper<Tuple1<Integer>> collector = new StreamOutputWrapper<Tuple1<Integer>>(2, sd);
-		collector.addOutput(new StreamOutput<Tuple1<Integer>>(recWriter, new ArrayList<String>()));
-		collector.collect(new Tuple1<Integer>(3));
-		collector.collect(new Tuple1<Integer>(4));
-		collector.collect(new Tuple1<Integer>(5));
-		collector.collect(new Tuple1<Integer>(6));
-
-		assertArrayEquals(new Integer[] { 3, 4, 5, 6 }, recWriter.emittedRecords.toArray());
-	}
-
-	@Test
-	public void testClose() {
-	}
-
-}
+import org.apache.flink.streaming.util.MockRecordWriterFactory;
+import org.apache.flink.util.Collector;
+import org.junit.Test;
+
+public class StreamCollectorTest {
+
+	@Test
+	public void testCollect() {
+		MockRecordWriter recWriter = MockRecordWriterFactory.create();
+		SerializationDelegate<StreamRecord<Tuple1<Integer>>> sd = new SerializationDelegate<StreamRecord<Tuple1<Integer>>>(
+				null);
+		sd.setInstance(new StreamRecord<Tuple1<Integer>>().setObject(new Tuple1<Integer>()));
+
+		Collector<Tuple1<Integer>> collector = new StreamOutput<Tuple1<Integer>>(recWriter, 2, sd);
+		collector.collect(new Tuple1<Integer>(3));
+		collector.collect(new Tuple1<Integer>(4));
+		collector.collect(new Tuple1<Integer>(5));
+		collector.collect(new Tuple1<Integer>(6));
+
+		assertArrayEquals(new Integer[] { 3, 4, 5, 6 }, recWriter.emittedRecords.toArray());
+	}
+
+	@Test
+	public void testClose() {
+	}
+
+}


[8/8] flink git commit: [FLINK-1345] [streaming] Chaining refactor + ChainingStrategy exposed through the API for operators

Posted by mb...@apache.org.
[FLINK-1345] [streaming] Chaining refactor + ChainingStrategy exposed through the API for operators


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/3e30c6f7
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/3e30c6f7
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/3e30c6f7

Branch: refs/heads/master
Commit: 3e30c6f73a12e2f50449a4d0dce452031f9a7317
Parents: 26535c4
Author: Gyula Fora <gy...@apache.org>
Authored: Fri Jan 16 15:11:38 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Wed Jan 21 16:06:34 2015 +0100

----------------------------------------------------------------------
 .../flink/streaming/api/JobGraphBuilder.java    | 103 ++++++++---------
 .../flink/streaming/api/StreamConfig.java       |  21 +---
 .../api/datastream/ConnectedDataStream.java     |   2 +-
 .../streaming/api/datastream/DataStream.java    |  13 ++-
 .../api/datastream/DataStreamSink.java          |   5 +-
 .../api/datastream/DataStreamSource.java        |   5 +-
 .../api/datastream/IterativeDataStream.java     |   3 +-
 .../datastream/SingleOutputStreamOperator.java  |  12 +-
 .../environment/StreamExecutionEnvironment.java |   9 +-
 .../api/invokable/ChainableInvokable.java       |   1 +
 .../api/invokable/StreamInvokable.java          |  20 +++-
 .../operator/GroupedReduceInvokable.java        |   6 +-
 .../operator/GroupedWindowInvokable.java        |   1 +
 .../api/invokable/operator/WindowInvokable.java |   3 +-
 .../api/streamvertex/CoStreamVertex.java        |   2 +-
 .../api/streamvertex/OutputHandler.java         | 110 +++++++++----------
 16 files changed, 159 insertions(+), 157 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/3e30c6f7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
index 35e145e..0020d48 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
@@ -36,6 +36,7 @@ import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
 import org.apache.flink.streaming.api.collector.OutputSelector;
 import org.apache.flink.streaming.api.invokable.ChainableInvokable;
 import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.invokable.StreamInvokable.ChainingStrategy;
 import org.apache.flink.streaming.api.invokable.operator.co.CoInvokable;
 import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
 import org.apache.flink.streaming.api.streamvertex.CoStreamVertex;
@@ -64,7 +65,7 @@ public class JobGraphBuilder {
 	private Map<String, Integer> vertexParallelism;
 	private Map<String, Long> bufferTimeout;
 	private Map<String, List<String>> outEdgeList;
-	private Map<String, List<Integer>> outEdgeType;
+	private Map<String, List<Integer>> outEdgeIndex;
 	private Map<String, List<List<String>>> outEdgeNames;
 	private Map<String, List<Boolean>> outEdgeSelectAll;
 	private Map<String, List<String>> inEdgeList;
@@ -85,8 +86,7 @@ public class JobGraphBuilder {
 	private Map<String, Map<String, OperatorState<?>>> operatorStates;
 	private Map<String, InputFormat<String, ?>> inputFormatList;
 	private Map<String, List<String>> chainedVertices;
-	private Map<String, List<ChainableInvokable<?, ?>>> chainedInvokable;
-	private Map<String, List<StreamRecordSerializer<?>>> chainedSerializer;
+	private Map<String, String> lastInChains;
 
 	private Set<String> sources;
 	private Set<String> builtVertices;
@@ -97,11 +97,19 @@ public class JobGraphBuilder {
 	 */
 	public JobGraphBuilder() {
 
+		initGraph();
+
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("JobGraph created");
+		}
+	}
+
+	public void initGraph() {
 		streamVertices = new HashMap<String, AbstractJobVertex>();
 		vertexParallelism = new HashMap<String, Integer>();
 		bufferTimeout = new HashMap<String, Long>();
 		outEdgeList = new HashMap<String, List<String>>();
-		outEdgeType = new HashMap<String, List<Integer>>();
+		outEdgeIndex = new HashMap<String, List<Integer>>();
 		outEdgeNames = new HashMap<String, List<List<String>>>();
 		outEdgeSelectAll = new HashMap<String, List<Boolean>>();
 		inEdgeList = new HashMap<String, List<String>>();
@@ -121,16 +129,11 @@ public class JobGraphBuilder {
 		iterationWaitTime = new HashMap<String, Long>();
 		operatorStates = new HashMap<String, Map<String, OperatorState<?>>>();
 		inputFormatList = new HashMap<String, InputFormat<String, ?>>();
-		chainedInvokable = new HashMap<String, List<ChainableInvokable<?, ?>>>();
-		chainedSerializer = new HashMap<String, List<StreamRecordSerializer<?>>>();
 		chainedVertices = new HashMap<String, List<String>>();
+		lastInChains = new HashMap<String, String>();
 
 		sources = new HashSet<String>();
 		builtVertices = new HashSet<String>();
-
-		if (LOG.isDebugEnabled()) {
-			LOG.debug("JobGraph created");
-		}
 	}
 
 	/**
@@ -198,7 +201,7 @@ public class JobGraphBuilder {
 		iterationIds.put(vertexName, iterationID);
 		iterationIDtoHeadName.put(iterationID, vertexName);
 
-		setBytesFrom(iterationHead, vertexName);
+		setSerializersFrom(iterationHead, vertexName);
 
 		setEdge(vertexName, iterationHead, outPartitioning
 				.get(inEdgeList.get(iterationHead).get(0)).get(0), 0, new ArrayList<String>(),
@@ -241,7 +244,7 @@ public class JobGraphBuilder {
 		iterationIds.put(vertexName, iterationID);
 		iterationIDtoTailName.put(iterationID, vertexName);
 
-		setBytesFrom(iterationTail, vertexName);
+		setSerializersFrom(iterationTail, vertexName);
 		iterationWaitTime.put(iterationIDtoTailName.get(iterationID), waitTime);
 
 		if (LOG.isDebugEnabled()) {
@@ -288,12 +291,13 @@ public class JobGraphBuilder {
 		invokableObjects.put(vertexName, invokableObject);
 		operatorNames.put(vertexName, operatorName);
 		outEdgeList.put(vertexName, new ArrayList<String>());
-		outEdgeType.put(vertexName, new ArrayList<Integer>());
+		outEdgeIndex.put(vertexName, new ArrayList<Integer>());
 		outEdgeNames.put(vertexName, new ArrayList<List<String>>());
 		outEdgeSelectAll.put(vertexName, new ArrayList<Boolean>());
 		inEdgeList.put(vertexName, new ArrayList<String>());
 		outPartitioning.put(vertexName, new ArrayList<StreamPartitioner<?>>());
 		iterationTailCount.put(vertexName, 0);
+		lastInChains.put(vertexName, vertexName);
 	}
 
 	private void addTypeSerializers(String vertexName, StreamRecordSerializer<?> in1,
@@ -327,11 +331,16 @@ public class JobGraphBuilder {
 				}
 			}
 
+			List<String> chainedNames = chainedVertices.get(vertexName);
+			boolean isChained = chainedNames != null;
+			int numChained = isChained ? chainedNames.size() : 0;
+			String lastInChain = lastInChains.get(vertexName);
+
 			// Get vertex attributes
 			Class<? extends AbstractInvokable> vertexClass = vertexClasses.get(vertexName);
 			StreamInvokable<?, ?> invokableObject = invokableObjects.get(vertexName);
 			int parallelism = vertexParallelism.get(vertexName);
-			byte[] outputSelector = outputSelectors.get(vertexName);
+			byte[] outputSelector = outputSelectors.get(lastInChain);
 			Map<String, OperatorState<?>> state = operatorStates.get(vertexName);
 
 			// Create vertex object
@@ -347,16 +356,17 @@ public class JobGraphBuilder {
 				LOG.debug("Parallelism set: {} for {}", parallelism, vertexName);
 			}
 
+			// Set vertex config
+
 			StreamConfig config = new StreamConfig(vertex.getConfiguration());
 
-			config.setBufferTimeout(bufferTimeout.get(vertexName));
+			config.setBufferTimeout(bufferTimeout.get(lastInChain));
 
 			config.setTypeSerializerIn1(typeSerializersIn1.get(vertexName));
 			config.setTypeSerializerIn2(typeSerializersIn2.get(vertexName));
 			config.setTypeSerializerOut1(typeSerializersOut1.get(vertexName));
 			config.setTypeSerializerOut2(typeSerializersOut2.get(vertexName));
 
-			// Set vertex config
 			config.setUserInvokable(invokableObject);
 			config.setOutputSelector(outputSelector);
 			config.setOperatorStates(state);
@@ -371,15 +381,12 @@ public class JobGraphBuilder {
 				vertex.setInputSplitSource(inputFormatList.get(vertexName));
 			}
 
-			List<ChainableInvokable<?, ?>> chainedInvokables = chainedInvokable.get(vertexName);
-			List<StreamRecordSerializer<?>> chainedSerializers = chainedSerializer.get(vertexName);
-
-			int numChained = chainedInvokables == null ? 0 : chainedInvokables.size();
 			config.setNumberofChainedTasks(numChained);
 
 			for (int i = 0; i < numChained; i++) {
-				config.setChainedInvokable(chainedInvokables.get(i), i);
-				config.setChainedSerializer(chainedSerializers.get(i), i);
+				config.setChainedInvokable(
+						(ChainableInvokable<?, ?>) invokableObjects.get(chainedNames.get(i)), i);
+				config.setChainedSerializer(typeSerializersIn1.get(chainedNames.get(i)), i);
 			}
 
 			streamVertices.put(vertexName, vertex);
@@ -390,12 +397,15 @@ public class JobGraphBuilder {
 	}
 
 	private void chainRecursively(String chainStart, String current, String next) {
+		// We chain the next operator to the start of this chain
 		chainTasks(chainStart, next);
-		// Add multiple chaining here
+		// Now recursively chain the outputs of next (depth first)
 		for (String output : outEdgeList.get(next)) {
 			if (isChainable(next, output)) {
+				// Recursive call
 				chainRecursively(chainStart, next, output);
 			} else {
+				// If not chainable we continue building the jobgraph from there
 				createVertex(output);
 			}
 		}
@@ -405,25 +415,14 @@ public class JobGraphBuilder {
 		return outEdgeList.get(vertexName).size() == 1
 				&& inEdgeList.get(outName).size() == 1
 				&& outputSelectors.get(vertexName) == null
-				&& invokableObjects.get(outName).isChainable()
+				&& invokableObjects.get(outName).getChainingStrategy() == ChainingStrategy.ALWAYS
+				&& (invokableObjects.get(vertexName).getChainingStrategy() == ChainingStrategy.HEAD || invokableObjects
+						.get(vertexName).getChainingStrategy() == ChainingStrategy.ALWAYS)
 				&& outPartitioning.get(vertexName).get(0).getStrategy() == PartitioningStrategy.FORWARD
 				&& vertexParallelism.get(vertexName) == vertexParallelism.get(outName) && chaining;
 	}
 
 	private void chainTasks(String first, String second) {
-		List<ChainableInvokable<?, ?>> chainedInvokables = chainedInvokable.get(first);
-		if (chainedInvokables == null) {
-			chainedInvokables = new ArrayList<ChainableInvokable<?, ?>>();
-		}
-		chainedInvokables.add((ChainableInvokable<?, ?>) invokableObjects.get(second));
-		chainedInvokable.put(first, chainedInvokables);
-
-		List<StreamRecordSerializer<?>> chainedSerializers = chainedSerializer.get(first);
-		if (chainedSerializers == null) {
-			chainedSerializers = new ArrayList<StreamRecordSerializer<?>>();
-		}
-		chainedSerializers.add(typeSerializersIn1.get(second));
-		chainedSerializer.put(first, chainedSerializers);
 
 		List<String> chained = chainedVertices.get(first);
 		if (chained == null) {
@@ -431,16 +430,7 @@ public class JobGraphBuilder {
 		}
 		chained.add(second);
 		chainedVertices.put(first, chained);
-
-		outEdgeList.put(first, outEdgeList.get(second));
-		typeSerializersOut1.put(first, typeSerializersOut1.get(second));
-		outPartitioning.put(first, outPartitioning.get(second));
-		outEdgeType.put(first, outEdgeType.get(second));
-		outEdgeNames.put(first, outEdgeNames.get(second));
-		outEdgeSelectAll.put(first, outEdgeSelectAll.get(second));
-		outPartitioning.put(first, outPartitioning.get(second));
-		bufferTimeout.put(first, bufferTimeout.get(second));
-		outputSelectors.put(first, outputSelectors.get(second));
+		lastInChains.put(first, second);
 
 	}
 
@@ -477,8 +467,10 @@ public class JobGraphBuilder {
 
 		int outputIndex = upStreamVertex.getNumberOfProducedIntermediateDataSets() - 1;
 
-		config.setOutputName(outputIndex, outEdgeNames.get(upStreamVertexName).get(outputIndex));
-		config.setSelectAll(outputIndex, outEdgeSelectAll.get(upStreamVertexName).get(outputIndex));
+		config.setOutputName(outputIndex, outEdgeNames.get(lastInChains.get(upStreamVertexName))
+				.get(outputIndex));
+		config.setSelectAll(outputIndex, outEdgeSelectAll.get(lastInChains.get(upStreamVertexName))
+				.get(outputIndex));
 		config.setPartitioner(outputIndex, partitionerObject);
 		config.setNumberOfOutputChannels(outputIndex, vertexParallelism.get(downStreamVertexName));
 	}
@@ -547,7 +539,7 @@ public class JobGraphBuilder {
 			StreamPartitioner<?> partitionerObject, int typeNumber, List<String> outputNames,
 			boolean selectAll) {
 		outEdgeList.get(upStreamVertexName).add(downStreamVertexName);
-		outEdgeType.get(upStreamVertexName).add(typeNumber);
+		outEdgeIndex.get(upStreamVertexName).add(typeNumber);
 		inEdgeList.get(downStreamVertexName).add(upStreamVertexName);
 		outPartitioning.get(upStreamVertexName).add(partitionerObject);
 		outEdgeNames.get(upStreamVertexName).add(outputNames);
@@ -608,7 +600,7 @@ public class JobGraphBuilder {
 	 * @param to
 	 *            to
 	 */
-	public void setBytesFrom(String from, String to) {
+	public void setSerializersFrom(String from, String to) {
 		operatorNames.put(to, operatorNames.get(from));
 
 		typeSerializersIn1.put(to, typeSerializersOut1.get(from));
@@ -691,19 +683,20 @@ public class JobGraphBuilder {
 		for (String upStreamVertexName : builtVertices) {
 			int i = 0;
 
-			List<Integer> outEdgeTypeList = outEdgeType.get(upStreamVertexName);
+			List<Integer> outEdgeTypeList = outEdgeIndex.get(lastInChains.get(upStreamVertexName));
 
-			for (String downStreamVertexName : outEdgeList.get(upStreamVertexName)) {
+			for (String downStreamVertexName : outEdgeList
+					.get(lastInChains.get(upStreamVertexName))) {
 				StreamConfig downStreamVertexConfig = new StreamConfig(streamVertices.get(
 						downStreamVertexName).getConfiguration());
 
 				int inputNumber = downStreamVertexConfig.getNumberOfInputs();
 
-				downStreamVertexConfig.setInputType(inputNumber++, outEdgeTypeList.get(i));
+				downStreamVertexConfig.setInputIndex(inputNumber++, outEdgeTypeList.get(i));
 				downStreamVertexConfig.setNumberOfInputs(inputNumber);
 
 				connect(upStreamVertexName, downStreamVertexName,
-						outPartitioning.get(upStreamVertexName).get(i));
+						outPartitioning.get(lastInChains.get(upStreamVertexName)).get(i));
 				i++;
 			}
 		}

http://git-wip-us.apache.org/repos/asf/flink/blob/3e30c6f7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
index 22bcf92..ada3aae 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
@@ -24,7 +24,6 @@ import java.util.Map;
 
 import org.apache.commons.lang3.SerializationException;
 import org.apache.commons.lang3.SerializationUtils;
-import org.apache.flink.api.common.functions.AbstractRichFunction;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.streaming.api.collector.OutputSelector;
 import org.apache.flink.streaming.api.invokable.ChainableInvokable;
@@ -170,7 +169,7 @@ public class StreamConfig {
 		config.setBoolean(DIRECTED_EMIT, directedEmit);
 	}
 
-	public boolean getDirectedEmit() {
+	public boolean isDirectedEmit() {
 		return config.getBoolean(DIRECTED_EMIT, false);
 	}
 
@@ -239,7 +238,7 @@ public class StreamConfig {
 		}
 	}
 
-	public boolean getSelectAll(int outputIndex) {
+	public boolean isSelectAll(int outputIndex) {
 		return config.getBoolean(OUTPUT_SELECT_ALL + outputIndex, false);
 	}
 
@@ -272,26 +271,14 @@ public class StreamConfig {
 		return config.getInteger(NUMBER_OF_OUTPUTS, 0);
 	}
 
-	public void setInputType(int inputNumber, Integer inputTypeNumber) {
+	public void setInputIndex(int inputNumber, Integer inputTypeNumber) {
 		config.setInteger(INPUT_TYPE + inputNumber++, inputTypeNumber);
 	}
 
-	public int getInputType(int inputNumber) {
+	public int getInputIndex(int inputNumber) {
 		return config.getInteger(INPUT_TYPE + inputNumber, 0);
 	}
 
-	public void setFunctionClass(Class<? extends AbstractRichFunction> functionClass) {
-		config.setClass("functionClass", functionClass);
-	}
-
-	public Class<? extends AbstractRichFunction> getFunctionClass(ClassLoader cl) {
-		try {
-			return config.getClass("functionClass", null, cl);
-		} catch (ClassNotFoundException e) {
-			throw new RuntimeException("Could not load function class", e);
-		}
-	}
-
 	public void setOperatorStates(Map<String, OperatorState<?>> states) {
 		config.setBytes(OPERATOR_STATES, SerializationUtils.serialize((Serializable) states));
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/3e30c6f7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
index a0c8ff8..80ea970 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
@@ -400,7 +400,7 @@ public class ConnectedDataStream<IN1, IN2> {
 
 		@SuppressWarnings({ "unchecked", "rawtypes" })
 		SingleOutputStreamOperator<OUT, ?> returnStream = new SingleOutputStreamOperator(
-				environment, functionName, outTypeInfo);
+				environment, functionName, outTypeInfo, functionInvokable);
 
 		dataStream1.jobGraphBuilder.addCoTask(returnStream.getId(), functionInvokable,
 				getInputType1(), getInputType2(), outTypeInfo, functionName,

http://git-wip-us.apache.org/repos/asf/flink/blob/3e30c6f7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index db644e9..f68ab68 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -1083,7 +1083,7 @@ public class DataStream<OUT> {
 	protected <R> DataStream<OUT> addIterationSource(Integer iterationID, long waitTime) {
 
 		DataStream<R> returnStream = new DataStreamSource<R>(environment, "iterationSource", null,
-				true);
+				null, true);
 
 		jobGraphBuilder.addIterationHead(returnStream.getId(), this.getId(), iterationID,
 				degreeOfParallelism, waitTime);
@@ -1110,7 +1110,7 @@ public class DataStream<OUT> {
 		DataStream<OUT> inputStream = this.copy();
 		@SuppressWarnings({ "unchecked", "rawtypes" })
 		SingleOutputStreamOperator<R, ?> returnStream = new SingleOutputStreamOperator(environment,
-				operatorName, outTypeInfo);
+				operatorName, outTypeInfo, invokable);
 
 		jobGraphBuilder.addStreamVertex(returnStream.getId(), invokable, getType(), outTypeInfo,
 				operatorName, degreeOfParallelism);
@@ -1174,10 +1174,13 @@ public class DataStream<OUT> {
 	 */
 	public DataStreamSink<OUT> addSink(SinkFunction<OUT> sinkFunction) {
 
-		DataStreamSink<OUT> returnStream = new DataStreamSink<OUT>(environment, "sink", getType());
+		StreamInvokable<OUT, OUT> sinkInvokable = new SinkInvokable<OUT>(clean(sinkFunction));
 
-		jobGraphBuilder.addStreamVertex(returnStream.getId(), new SinkInvokable<OUT>(
-				clean(sinkFunction)), getType(), null, "sink", degreeOfParallelism);
+		DataStreamSink<OUT> returnStream = new DataStreamSink<OUT>(environment, "sink", getType(),
+				sinkInvokable);
+
+		jobGraphBuilder.addStreamVertex(returnStream.getId(), sinkInvokable, getType(), null,
+				"sink", degreeOfParallelism);
 
 		this.connectGraph(this.copy(), returnStream.getId(), 0);
 

http://git-wip-us.apache.org/repos/asf/flink/blob/3e30c6f7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
index 61fc557..f064332 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
@@ -19,6 +19,7 @@ package org.apache.flink.streaming.api.datastream;
 
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
 
 /**
  * Represents the end of a DataStream.
@@ -29,8 +30,8 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 public class DataStreamSink<IN> extends SingleOutputStreamOperator<IN, DataStreamSink<IN>> {
 
 	protected DataStreamSink(StreamExecutionEnvironment environment, String operatorType,
-			TypeInformation<IN> outTypeInfo) {
-		super(environment, operatorType, outTypeInfo);
+			TypeInformation<IN> outTypeInfo, StreamInvokable<?,?> invokable) {
+		super(environment, operatorType, outTypeInfo, invokable);
 	}
 
 	protected DataStreamSink(DataStream<IN> dataStream) {

http://git-wip-us.apache.org/repos/asf/flink/blob/3e30c6f7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java
index a649f59..b596cbd 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java
@@ -19,6 +19,7 @@ package org.apache.flink.streaming.api.datastream;
 
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
 
 /**
  * The DataStreamSource represents the starting point of a DataStream.
@@ -31,8 +32,8 @@ public class DataStreamSource<OUT> extends SingleOutputStreamOperator<OUT, DataS
 	boolean isParallel;
 
 	public DataStreamSource(StreamExecutionEnvironment environment, String operatorType,
-			TypeInformation<OUT> outTypeInfo, boolean isParallel) {
-		super(environment, operatorType, outTypeInfo);
+			TypeInformation<OUT> outTypeInfo, StreamInvokable<?, ?> invokable, boolean isParallel) {
+		super(environment, operatorType, outTypeInfo, invokable);
 		this.isParallel = isParallel;
 	}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/3e30c6f7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
index 88feb5d..c4cd1e1 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
@@ -61,7 +61,8 @@ public class IterativeDataStream<IN> extends
 	 * 
 	 */
 	public DataStream<IN> closeWith(DataStream<IN> iterationTail) {
-		DataStream<IN> iterationSink = new DataStreamSink<IN>(environment, "iterationSink", null);
+		DataStream<IN> iterationSink = new DataStreamSink<IN>(environment, "iterationSink", null,
+				null);
 
 		jobGraphBuilder.addIterationTail(iterationSink.getId(), iterationTail.getId(), iterationID,
 				iterationTail.getParallelism(), waitTime);

http://git-wip-us.apache.org/repos/asf/flink/blob/3e30c6f7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
index 7a0abe4..b8ca42c 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
@@ -26,6 +26,8 @@ import org.apache.flink.api.common.functions.RichFunction;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.streaming.api.collector.OutputSelector;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.invokable.StreamInvokable.ChainingStrategy;
 import org.apache.flink.streaming.api.streamvertex.StreamingRuntimeContext;
 import org.apache.flink.streaming.state.OperatorState;
 
@@ -42,12 +44,14 @@ public class SingleOutputStreamOperator<OUT, O extends SingleOutputStreamOperato
 		DataStream<OUT> {
 
 	protected boolean isSplit;
+	protected StreamInvokable<?, ?> invokable;
 
 	protected SingleOutputStreamOperator(StreamExecutionEnvironment environment,
-			String operatorType, TypeInformation<OUT> outTypeInfo) {
+			String operatorType, TypeInformation<OUT> outTypeInfo, StreamInvokable<?, ?> invokable) {
 		super(environment, operatorType, outTypeInfo);
 		setBufferTimeout(environment.getBufferTimeout());
 		this.isSplit = false;
+		this.invokable = invokable;
 	}
 
 	@SuppressWarnings("unchecked")
@@ -55,6 +59,7 @@ public class SingleOutputStreamOperator<OUT, O extends SingleOutputStreamOperato
 		super(dataStream);
 		if (dataStream instanceof SingleOutputStreamOperator) {
 			this.isSplit = ((SingleOutputStreamOperator<OUT, ?>) dataStream).isSplit;
+			this.invokable = ((SingleOutputStreamOperator<OUT, ?>) dataStream).invokable;
 		}
 	}
 
@@ -192,4 +197,9 @@ public class SingleOutputStreamOperator<OUT, O extends SingleOutputStreamOperato
 		return new SingleOutputStreamOperator<OUT, O>(this);
 	}
 
+	public SingleOutputStreamOperator<OUT, O> setChainingStrategy(ChainingStrategy strategy) {
+		this.invokable.setChainingStrategy(strategy);
+		return this;
+	}
+
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/3e30c6f7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
index 08fd19b..4194864 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
@@ -46,6 +46,7 @@ import org.apache.flink.streaming.api.function.source.RichParallelSourceFunction
 import org.apache.flink.streaming.api.function.source.SocketTextStreamFunction;
 import org.apache.flink.streaming.api.function.source.SourceFunction;
 import org.apache.flink.streaming.api.invokable.SourceInvokable;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
 
 /**
  * {@link ExecutionEnvironment} for streaming jobs. An instance of it is
@@ -431,11 +432,13 @@ public abstract class StreamExecutionEnvironment {
 		boolean isParallel = function instanceof ParallelSourceFunction;
 		int dop = isParallel ? getDegreeOfParallelism() : 1;
 
+		StreamInvokable<OUT, OUT> sourceInvokable = new SourceInvokable<OUT>(function);
+
 		DataStreamSource<OUT> returnStream = new DataStreamSource<OUT>(this, sourceName,
-				outTypeInfo, isParallel);
+				outTypeInfo, sourceInvokable, isParallel);
 
-		jobGraphBuilder.addSourceVertex(returnStream.getId(), new SourceInvokable<OUT>(function),
-				null, outTypeInfo, sourceName, dop);
+		jobGraphBuilder.addSourceVertex(returnStream.getId(), sourceInvokable, null, outTypeInfo,
+				sourceName, dop);
 
 		return returnStream;
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/3e30c6f7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/ChainableInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/ChainableInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/ChainableInvokable.java
index 373b4e8..24c0319 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/ChainableInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/ChainableInvokable.java
@@ -28,6 +28,7 @@ public abstract class ChainableInvokable<IN, OUT> extends StreamInvokable<IN, OU
 
 	public ChainableInvokable(Function userFunction) {
 		super(userFunction);
+		setChainingStrategy(ChainingStrategy.ALWAYS);
 	}
 
 	public void setup(Collector<OUT> collector, StreamRecordSerializer<IN> inSerializer) {

http://git-wip-us.apache.org/repos/asf/flink/blob/3e30c6f7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java
index 25e9221..614b67f 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java
@@ -59,6 +59,8 @@ public abstract class StreamInvokable<IN, OUT> implements Serializable {
 	protected Function userFunction;
 	protected volatile boolean isRunning;
 
+	private ChainingStrategy chainingStrategy = ChainingStrategy.HEAD;
+
 	public StreamInvokable(Function userFunction) {
 		this.userFunction = userFunction;
 	}
@@ -160,7 +162,21 @@ public abstract class StreamInvokable<IN, OUT> implements Serializable {
 		return objectSerializer.copy(record);
 	}
 
-	public boolean isChainable() {
-		return this instanceof ChainableInvokable;
+	public void setChainingStrategy(ChainingStrategy strategy) {
+		if (strategy == ChainingStrategy.ALWAYS) {
+			if (!(this instanceof ChainableInvokable)) {
+				throw new RuntimeException(
+						"Invokable needs to extend ChainableInvokable to be chained");
+			}
+		}
+		this.chainingStrategy = strategy;
+	}
+
+	public ChainingStrategy getChainingStrategy() {
+		return chainingStrategy;
+	}
+
+	public static enum ChainingStrategy {
+		ALWAYS, NEVER, HEAD;
 	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/3e30c6f7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java
index 84258d6..01c0545 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java
@@ -34,6 +34,7 @@ public class GroupedReduceInvokable<IN> extends StreamReduceInvokable<IN> {
 		super(reducer);
 		this.keySelector = keySelector;
 		values = new HashMap<Object, IN>();
+		setChainingStrategy(ChainingStrategy.NEVER);
 	}
 
 	@Override
@@ -56,9 +57,4 @@ public class GroupedReduceInvokable<IN> extends StreamReduceInvokable<IN> {
 		reduced = reducer.reduce(currentValue, nextValue);
 	}
 
-	@Override
-	public boolean isChainable() {
-		return false;
-	}
-
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/3e30c6f7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
index 33348e4..bbd7b0c 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
@@ -154,6 +154,7 @@ public class GroupedWindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
 			LinkedList<EvictionPolicy<IN>> centralEvictionPolicies) {
 
 		super(userFunction);
+		setChainingStrategy(ChainingStrategy.NEVER);
 
 		this.keySelector = keySelector;
 

http://git-wip-us.apache.org/repos/asf/flink/blob/3e30c6f7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
index ea891c9..de9c664 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
@@ -61,7 +61,8 @@ public abstract class WindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT>
 	public WindowInvokable(Function userFunction, LinkedList<TriggerPolicy<IN>> triggerPolicies,
 			LinkedList<EvictionPolicy<IN>> evictionPolicies) {
 		super(userFunction);
-
+		setChainingStrategy(ChainingStrategy.NEVER);
+		
 		this.triggerPolicies = triggerPolicies;
 		this.evictionPolicies = evictionPolicies;
 

http://git-wip-us.apache.org/repos/asf/flink/blob/3e30c6f7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/CoStreamVertex.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/CoStreamVertex.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/CoStreamVertex.java
index f065d9c..2b650be 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/CoStreamVertex.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/CoStreamVertex.java
@@ -80,7 +80,7 @@ public class CoStreamVertex<IN1, IN2, OUT> extends StreamVertex<IN1, OUT> {
 		ArrayList<BufferReader> inputList2 = new ArrayList<BufferReader>();
 
 		for (int i = 0; i < numberOfInputs; i++) {
-			int inputType = configuration.getInputType(i);
+			int inputType = configuration.getInputIndex(i);
 			BufferReader reader = getEnvironment().getReader(i);
 			switch (inputType) {
 				case 1:

http://git-wip-us.apache.org/repos/asf/flink/blob/3e30c6f7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
index 457f3f8..60a7b14 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
@@ -26,7 +26,6 @@ import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
 import org.apache.flink.runtime.plugable.SerializationDelegate;
 import org.apache.flink.streaming.api.StreamConfig;
-import org.apache.flink.streaming.api.collector.CollectorWrapper;
 import org.apache.flink.streaming.api.collector.DirectedOutputWrapper;
 import org.apache.flink.streaming.api.collector.OutputSelector;
 import org.apache.flink.streaming.api.collector.StreamOutput;
@@ -44,11 +43,11 @@ import org.slf4j.LoggerFactory;
 public class OutputHandler<OUT> {
 	private static final Logger LOG = LoggerFactory.getLogger(OutputHandler.class);
 
-	private StreamVertex<?, OUT> streamVertex;
+	private StreamVertex<?, OUT> vertex;
 	private StreamConfig configuration;
 
 	private List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> outputs;
-	private Collector<OUT> endCollector;
+	private Collector<OUT> outerCollector;
 
 	TypeInformation<OUT> outTypeInfo = null;
 	StreamRecordSerializer<OUT> outSerializer = null;
@@ -58,18 +57,14 @@ public class OutputHandler<OUT> {
 
 	private int numberOfChainedTasks;
 
-	public OutputHandler(StreamVertex<?, OUT> streamComponent) {
-		this.streamVertex = streamComponent;
+	public OutputHandler(StreamVertex<?, OUT> vertex) {
+		this.vertex = vertex;
 		this.outputs = new LinkedList<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>>();
-		this.configuration = new StreamConfig(streamComponent.getTaskConfiguration());
+		this.configuration = new StreamConfig(vertex.getTaskConfiguration());
 		this.chainedInvokables = new ArrayList<ChainableInvokable<?, ?>>();
+		this.numberOfChainedTasks = configuration.getNumberofChainedTasks();
 
-		try {
-			setConfigOutputs();
-		} catch (StreamVertexException e) {
-			throw new StreamVertexException("Cannot register outputs for "
-					+ streamComponent.getClass().getSimpleName(), e);
-		}
+		this.outerCollector = createChainedCollector(0);
 
 	}
 
@@ -77,84 +72,83 @@ public class OutputHandler<OUT> {
 		return outputs;
 	}
 
-	private void setConfigOutputs() {
-		numberOfChainedTasks = configuration.getNumberofChainedTasks();
-		endCollector = createChainedOutputs(0);
-	}
-
-	@SuppressWarnings("unchecked")
-	private Collector<OUT> createChainedOutputs(int chainedTaskIndex) {
+	// We create the outer collector by nesting the chainable invokables into
+	// each other
+	@SuppressWarnings({ "unchecked", "rawtypes" })
+	private Collector<OUT> createChainedCollector(int chainedTaskIndex) {
 
 		if (numberOfChainedTasks == chainedTaskIndex) {
-			return createEndCollector();
+			// At the end of the chain we create the collector that sends data
+			// to the recordwriters
+			return createNetworkCollector();
 		} else {
-			CollectorWrapper<OUT> chainedCollector = new CollectorWrapper<OUT>();
 
-			@SuppressWarnings("rawtypes")
 			ChainableInvokable chainableInvokable = configuration.getChainedInvokable(
-					chainedTaskIndex, streamVertex.getUserCodeClassLoader());
+					chainedTaskIndex, vertex.getUserCodeClassLoader());
 
+			// The nesting is done by calling this method recursively when
+			// passing the collector to the invokable
 			chainableInvokable.setup(
-					createChainedOutputs(chainedTaskIndex + 1),
+					createChainedCollector(chainedTaskIndex + 1),
 					configuration.getChainedInSerializer(chainedTaskIndex,
-							streamVertex.getUserCodeClassLoader()));
+							vertex.getUserCodeClassLoader()));
 
+			// We hold a list of the chained invokables for initializaton
+			// afterwards
 			chainedInvokables.add(chainableInvokable);
 
-			chainedCollector.addChainedOutput((Collector<OUT>) chainableInvokable);
-
-			return chainedCollector;
+			return chainableInvokable;
 		}
 
 	}
 
-	private Collector<OUT> createEndCollector() {
+	private Collector<OUT> createNetworkCollector() {
 
-		setSerializers();
+		createOutSerializer();
 
 		StreamOutputWrapper<OUT> collector;
 
-		if (streamVertex.configuration.getDirectedEmit()) {
-			OutputSelector<OUT> outputSelector = streamVertex.configuration
-					.getOutputSelector(streamVertex.userClassLoader);
+		if (vertex.configuration.isDirectedEmit()) {
+			OutputSelector<OUT> outputSelector = vertex.configuration
+					.getOutputSelector(vertex.userClassLoader);
 
-			collector = new DirectedOutputWrapper<OUT>(streamVertex.getInstanceID(),
+			collector = new DirectedOutputWrapper<OUT>(vertex.getInstanceID(),
 					outSerializationDelegate, outputSelector);
 		} else {
-			collector = new StreamOutputWrapper<OUT>(streamVertex.getInstanceID(),
+			collector = new StreamOutputWrapper<OUT>(vertex.getInstanceID(),
 					outSerializationDelegate);
 		}
 
 		int numberOfOutputs = configuration.getNumberOfOutputs();
 		for (int i = 0; i < numberOfOutputs; i++) {
-			collector = (StreamOutputWrapper<OUT>) setPartitioner(i, collector);
+			collector = (StreamOutputWrapper<OUT>) addStreamOutput(i, collector);
 		}
 
 		return collector;
 	}
 
 	public Collector<OUT> getCollector() {
-		return endCollector;
+		return outerCollector;
 	}
 
-	void setSerializers() {
-		outSerializer = configuration.getTypeSerializerOut1(streamVertex.userClassLoader);
+	void createOutSerializer() {
+		outSerializer = configuration.getTypeSerializerOut1(vertex.userClassLoader);
 		if (outSerializer != null) {
 			outSerializationDelegate = new SerializationDelegate<StreamRecord<OUT>>(outSerializer);
 			outSerializationDelegate.setInstance(outSerializer.createInstance());
 		}
 	}
 
-	Collector<OUT> setPartitioner(int outputNumber, StreamOutputWrapper<OUT> endCollector) {
-		StreamPartitioner<OUT> outputPartitioner = null;
+	Collector<OUT> addStreamOutput(int outputNumber, StreamOutputWrapper<OUT> networkCollector) {
+
+		StreamPartitioner<OUT> outputPartitioner;
 
 		try {
-			outputPartitioner = configuration.getPartitioner(streamVertex.userClassLoader,
+			outputPartitioner = configuration.getPartitioner(vertex.userClassLoader,
 					outputNumber);
-
 		} catch (Exception e) {
 			throw new StreamVertexException("Cannot deserialize partitioner for "
-					+ streamVertex.getName() + " with " + outputNumber + " outputs", e);
+					+ vertex.getName() + " with " + outputNumber + " outputs", e);
 		}
 
 		RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output;
@@ -162,37 +156,33 @@ public class OutputHandler<OUT> {
 		long bufferTimeout = configuration.getBufferTimeout();
 
 		if (bufferTimeout >= 0) {
-			output = new StreamRecordWriter<SerializationDelegate<StreamRecord<OUT>>>(streamVertex
+			output = new StreamRecordWriter<SerializationDelegate<StreamRecord<OUT>>>(vertex
 					.getEnvironment().getWriter(outputNumber), outputPartitioner, bufferTimeout);
 
 			if (LOG.isTraceEnabled()) {
 				LOG.trace("StreamRecordWriter initiated with {} bufferTimeout for {}",
-						bufferTimeout, streamVertex.getClass().getSimpleName());
+						bufferTimeout, vertex.getClass().getSimpleName());
 			}
 		} else {
-			output = new RecordWriter<SerializationDelegate<StreamRecord<OUT>>>(streamVertex
+			output = new RecordWriter<SerializationDelegate<StreamRecord<OUT>>>(vertex
 					.getEnvironment().getWriter(outputNumber), outputPartitioner);
 
 			if (LOG.isTraceEnabled()) {
-				LOG.trace("RecordWriter initiated for {}", streamVertex.getClass().getSimpleName());
+				LOG.trace("RecordWriter initiated for {}", vertex.getClass().getSimpleName());
 			}
 		}
 
 		outputs.add(output);
-		List<String> outputNames = configuration.getOutputNames(outputNumber);
-		boolean isSelectAllOutput = configuration.getSelectAll(outputNumber);
 
-		if (endCollector != null) {
-			endCollector.addOutput(new StreamOutput<OUT>(output, isSelectAllOutput ? null
-					: outputNames));
-		}
+		networkCollector.addOutput(new StreamOutput<OUT>(output, configuration
+				.isSelectAll(outputNumber) ? null : configuration.getOutputNames(outputNumber)));
 
 		if (LOG.isTraceEnabled()) {
 			LOG.trace("Partitioner set: {} with {} outputs for {}", outputPartitioner.getClass()
-					.getSimpleName(), outputNumber, streamVertex.getClass().getSimpleName());
+					.getSimpleName(), outputNumber, vertex.getClass().getSimpleName());
 		}
 
-		return endCollector;
+		return networkCollector;
 	}
 
 	public void flushOutputs() throws IOException, InterruptedException {
@@ -205,17 +195,15 @@ public class OutputHandler<OUT> {
 		}
 	}
 
-	long startTime;
-
 	public void invokeUserFunction(String componentTypeName, StreamInvokable<?, OUT> userInvokable)
 			throws IOException, InterruptedException {
 		if (LOG.isDebugEnabled()) {
 			LOG.debug("{} {} invoked with instance id {}", componentTypeName,
-					streamVertex.getName(), streamVertex.getInstanceID());
+					vertex.getName(), vertex.getInstanceID());
 		}
 
 		try {
-			streamVertex.invokeUserFunction(userInvokable);
+			vertex.invokeUserFunction(userInvokable);
 		} catch (Exception e) {
 			flushOutputs();
 			throw new RuntimeException(e);
@@ -223,7 +211,7 @@ public class OutputHandler<OUT> {
 
 		if (LOG.isDebugEnabled()) {
 			LOG.debug("{} {} invoke finished instance id {}", componentTypeName,
-					streamVertex.getName(), streamVertex.getInstanceID());
+					vertex.getName(), vertex.getInstanceID());
 		}
 
 		flushOutputs();


[2/8] flink git commit: [FLINK-1345] [streaming] Basic operator chaining added

Posted by mb...@apache.org.
[FLINK-1345] [streaming] Basic operator chaining added


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/26535c48
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/26535c48
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/26535c48

Branch: refs/heads/master
Commit: 26535c487753975556aafc19c2ce4f9c24cc677b
Parents: d9b942b
Author: Gyula Fora <gy...@apache.org>
Authored: Thu Jan 15 22:32:13 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Wed Jan 21 16:06:33 2015 +0100

----------------------------------------------------------------------
 .../flink/streaming/api/JobGraphBuilder.java    | 222 +++++++++++++------
 .../flink/streaming/api/StreamConfig.java       |  56 +++--
 .../api/collector/CollectorWrapper.java         |   2 +-
 .../api/datastream/IterativeDataStream.java     |   2 +-
 .../datastream/SingleOutputStreamOperator.java  |   5 +
 .../environment/StreamExecutionEnvironment.java |   2 +-
 .../api/invokable/ChainableInvokable.java       |  38 ++++
 .../streaming/api/invokable/SinkInvokable.java  |  10 +-
 .../api/invokable/StreamInvokable.java          |  23 +-
 .../invokable/operator/CounterInvokable.java    |  10 +-
 .../api/invokable/operator/FilterInvokable.java |  18 +-
 .../invokable/operator/FlatMapInvokable.java    |  12 +-
 .../operator/GroupedReduceInvokable.java        |   7 +-
 .../api/invokable/operator/MapInvokable.java    |  12 +-
 .../operator/StreamReduceInvokable.java         |  15 +-
 .../operator/co/CoWindowInvokable.java          |   2 +-
 .../api/streamvertex/OutputHandler.java         |  86 +++++--
 .../api/streamvertex/StreamVertex.java          |  24 +-
 .../examples/iteration/IterateExample.java      |   3 +-
 19 files changed, 406 insertions(+), 143 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
index 649072f..35e145e 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
@@ -22,7 +22,9 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.flink.api.common.io.InputFormat;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.runtime.jobgraph.AbstractJobVertex;
@@ -32,8 +34,7 @@ import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
 import org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup;
 import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
 import org.apache.flink.streaming.api.collector.OutputSelector;
-import org.apache.flink.streaming.api.function.source.SourceFunction;
-import org.apache.flink.streaming.api.invokable.SourceInvokable;
+import org.apache.flink.streaming.api.invokable.ChainableInvokable;
 import org.apache.flink.streaming.api.invokable.StreamInvokable;
 import org.apache.flink.streaming.api.invokable.operator.co.CoInvokable;
 import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
@@ -56,6 +57,8 @@ public class JobGraphBuilder {
 	private final static String DEAFULT_JOB_NAME = "Streaming Job";
 	private JobGraph jobGraph;
 
+	private boolean chaining = true;
+
 	// Graph attributes
 	private Map<String, AbstractJobVertex> streamVertices;
 	private Map<String, Integer> vertexParallelism;
@@ -65,7 +68,7 @@ public class JobGraphBuilder {
 	private Map<String, List<List<String>>> outEdgeNames;
 	private Map<String, List<Boolean>> outEdgeSelectAll;
 	private Map<String, List<String>> inEdgeList;
-	private Map<String, List<StreamPartitioner<?>>> connectionTypes;
+	private Map<String, List<StreamPartitioner<?>>> outPartitioning;
 	private Map<String, String> operatorNames;
 	private Map<String, StreamInvokable<?, ?>> invokableObjects;
 	private Map<String, StreamRecordSerializer<?>> typeSerializersIn1;
@@ -81,6 +84,12 @@ public class JobGraphBuilder {
 	private Map<String, Long> iterationWaitTime;
 	private Map<String, Map<String, OperatorState<?>>> operatorStates;
 	private Map<String, InputFormat<String, ?>> inputFormatList;
+	private Map<String, List<String>> chainedVertices;
+	private Map<String, List<ChainableInvokable<?, ?>>> chainedInvokable;
+	private Map<String, List<StreamRecordSerializer<?>>> chainedSerializer;
+
+	private Set<String> sources;
+	private Set<String> builtVertices;
 
 	/**
 	 * Creates an new {@link JobGraph} with the given name. A JobGraph is a DAG
@@ -96,7 +105,7 @@ public class JobGraphBuilder {
 		outEdgeNames = new HashMap<String, List<List<String>>>();
 		outEdgeSelectAll = new HashMap<String, List<Boolean>>();
 		inEdgeList = new HashMap<String, List<String>>();
-		connectionTypes = new HashMap<String, List<StreamPartitioner<?>>>();
+		outPartitioning = new HashMap<String, List<StreamPartitioner<?>>>();
 		operatorNames = new HashMap<String, String>();
 		invokableObjects = new HashMap<String, StreamInvokable<?, ?>>();
 		typeSerializersIn1 = new HashMap<String, StreamRecordSerializer<?>>();
@@ -112,6 +121,12 @@ public class JobGraphBuilder {
 		iterationWaitTime = new HashMap<String, Long>();
 		operatorStates = new HashMap<String, Map<String, OperatorState<?>>>();
 		inputFormatList = new HashMap<String, InputFormat<String, ?>>();
+		chainedInvokable = new HashMap<String, List<ChainableInvokable<?, ?>>>();
+		chainedSerializer = new HashMap<String, List<StreamRecordSerializer<?>>>();
+		chainedVertices = new HashMap<String, List<String>>();
+
+		sources = new HashSet<String>();
+		builtVertices = new HashSet<String>();
 
 		if (LOG.isDebugEnabled()) {
 			LOG.debug("JobGraph created");
@@ -152,32 +167,12 @@ public class JobGraphBuilder {
 		}
 	}
 
-	/**
-	 * Adds a source vertex to the streaming JobGraph with the given parameters
-	 * 
-	 * @param vertexName
-	 *            Name of the vertex
-	 * @param function
-	 *            User defined function
-	 * @param inTypeInfo
-	 *            Input type for serialization
-	 * @param outTypeInfo
-	 *            Output type for serialization
-	 * @param operatorName
-	 *            Operator type
-	 * @param parallelism
-	 *            Number of parallel instances created
-	 */
-	public <IN, OUT> void addSourceVertex(String vertexName, SourceFunction<OUT> function,
-			TypeInformation<IN> inTypeInfo, TypeInformation<OUT> outTypeInfo, String operatorName,
-			byte[] serializedFunction, int parallelism) {
-
-		@SuppressWarnings("unchecked")
-		StreamInvokable<IN, OUT> invokableObject = (StreamInvokable<IN, OUT>) new SourceInvokable<OUT>(
-				function);
-
+	public <IN, OUT> void addSourceVertex(String vertexName,
+			StreamInvokable<IN, OUT> invokableObject, TypeInformation<IN> inTypeInfo,
+			TypeInformation<OUT> outTypeInfo, String operatorName, int parallelism) {
 		addStreamVertex(vertexName, invokableObject, inTypeInfo, outTypeInfo, operatorName,
 				parallelism);
+		sources.add(vertexName);
 	}
 
 	/**
@@ -205,7 +200,7 @@ public class JobGraphBuilder {
 
 		setBytesFrom(iterationHead, vertexName);
 
-		setEdge(vertexName, iterationHead, connectionTypes
+		setEdge(vertexName, iterationHead, outPartitioning
 				.get(inEdgeList.get(iterationHead).get(0)).get(0), 0, new ArrayList<String>(),
 				false);
 
@@ -214,6 +209,8 @@ public class JobGraphBuilder {
 		if (LOG.isDebugEnabled()) {
 			LOG.debug("ITERATION SOURCE: {}", vertexName);
 		}
+
+		sources.add(vertexName);
 	}
 
 	/**
@@ -295,7 +292,7 @@ public class JobGraphBuilder {
 		outEdgeNames.put(vertexName, new ArrayList<List<String>>());
 		outEdgeSelectAll.put(vertexName, new ArrayList<Boolean>());
 		inEdgeList.put(vertexName, new ArrayList<String>());
-		connectionTypes.put(vertexName, new ArrayList<StreamPartitioner<?>>());
+		outPartitioning.put(vertexName, new ArrayList<StreamPartitioner<?>>());
 		iterationTailCount.put(vertexName, 0);
 	}
 
@@ -317,50 +314,134 @@ public class JobGraphBuilder {
 	 */
 	private void createVertex(String vertexName) {
 
-		// Get vertex attributes
-		Class<? extends AbstractInvokable> vertexClass = vertexClasses.get(vertexName);
-		StreamInvokable<?, ?> invokableObject = invokableObjects.get(vertexName);
-		int parallelism = vertexParallelism.get(vertexName);
-		byte[] outputSelector = outputSelectors.get(vertexName);
-		Map<String, OperatorState<?>> state = operatorStates.get(vertexName);
+		if (!builtVertices.contains(vertexName)) {
+			if (!outEdgeList.get(vertexName).isEmpty()) {
 
-		// Create vertex object
-		AbstractJobVertex vertex = new AbstractJobVertex(vertexName);
+				for (String outName : outEdgeList.get(vertexName)) {
+					if (isChainable(vertexName, outName)) {
+						chainRecursively(vertexName, vertexName, outName);
+					} else {
+						createVertex(outName);
+					}
 
-		this.jobGraph.addVertex(vertex);
+				}
+			}
+
+			// Get vertex attributes
+			Class<? extends AbstractInvokable> vertexClass = vertexClasses.get(vertexName);
+			StreamInvokable<?, ?> invokableObject = invokableObjects.get(vertexName);
+			int parallelism = vertexParallelism.get(vertexName);
+			byte[] outputSelector = outputSelectors.get(vertexName);
+			Map<String, OperatorState<?>> state = operatorStates.get(vertexName);
+
+			// Create vertex object
+			String cname = chainedVertices.get(vertexName) == null ? "" : " => "
+					+ StringUtils.join(chainedVertices.get(vertexName), " => ");
+			AbstractJobVertex vertex = new AbstractJobVertex(vertexName + cname);
+
+			this.jobGraph.addVertex(vertex);
+
+			vertex.setInvokableClass(vertexClass);
+			vertex.setParallelism(parallelism);
+			if (LOG.isDebugEnabled()) {
+				LOG.debug("Parallelism set: {} for {}", parallelism, vertexName);
+			}
+
+			StreamConfig config = new StreamConfig(vertex.getConfiguration());
+
+			config.setBufferTimeout(bufferTimeout.get(vertexName));
+
+			config.setTypeSerializerIn1(typeSerializersIn1.get(vertexName));
+			config.setTypeSerializerIn2(typeSerializersIn2.get(vertexName));
+			config.setTypeSerializerOut1(typeSerializersOut1.get(vertexName));
+			config.setTypeSerializerOut2(typeSerializersOut2.get(vertexName));
+
+			// Set vertex config
+			config.setUserInvokable(invokableObject);
+			config.setOutputSelector(outputSelector);
+			config.setOperatorStates(state);
+
+			if (vertexClass.equals(StreamIterationHead.class)
+					|| vertexClass.equals(StreamIterationTail.class)) {
+				config.setIterationId(iterationIds.get(vertexName));
+				config.setIterationWaitTime(iterationWaitTime.get(vertexName));
+			}
+
+			if (inputFormatList.containsKey(vertexName)) {
+				vertex.setInputSplitSource(inputFormatList.get(vertexName));
+			}
+
+			List<ChainableInvokable<?, ?>> chainedInvokables = chainedInvokable.get(vertexName);
+			List<StreamRecordSerializer<?>> chainedSerializers = chainedSerializer.get(vertexName);
+
+			int numChained = chainedInvokables == null ? 0 : chainedInvokables.size();
+			config.setNumberofChainedTasks(numChained);
+
+			for (int i = 0; i < numChained; i++) {
+				config.setChainedInvokable(chainedInvokables.get(i), i);
+				config.setChainedSerializer(chainedSerializers.get(i), i);
+			}
+
+			streamVertices.put(vertexName, vertex);
+			builtVertices.add(vertexName);
 
-		vertex.setInvokableClass(vertexClass);
-		vertex.setParallelism(parallelism);
-		if (LOG.isDebugEnabled()) {
-			LOG.debug("Parallelism set: {} for {}", parallelism, vertexName);
 		}
 
-		StreamConfig config = new StreamConfig(vertex.getConfiguration());
+	}
 
-		config.setBufferTimeout(bufferTimeout.get(vertexName));
+	private void chainRecursively(String chainStart, String current, String next) {
+		chainTasks(chainStart, next);
+		// Add multiple chaining here
+		for (String output : outEdgeList.get(next)) {
+			if (isChainable(next, output)) {
+				chainRecursively(chainStart, next, output);
+			} else {
+				createVertex(output);
+			}
+		}
+	}
 
-		config.setTypeSerializerIn1(typeSerializersIn1.get(vertexName));
-		config.setTypeSerializerIn2(typeSerializersIn2.get(vertexName));
-		config.setTypeSerializerOut1(typeSerializersOut1.get(vertexName));
-		config.setTypeSerializerOut2(typeSerializersOut2.get(vertexName));
+	private boolean isChainable(String vertexName, String outName) {
+		return outEdgeList.get(vertexName).size() == 1
+				&& inEdgeList.get(outName).size() == 1
+				&& outputSelectors.get(vertexName) == null
+				&& invokableObjects.get(outName).isChainable()
+				&& outPartitioning.get(vertexName).get(0).getStrategy() == PartitioningStrategy.FORWARD
+				&& vertexParallelism.get(vertexName) == vertexParallelism.get(outName) && chaining;
+	}
 
-		// Set vertex config
-		config.setUserInvokable(invokableObject);
-		config.setVertexName(vertexName);
-		config.setOutputSelector(outputSelector);
-		config.setOperatorStates(state);
+	private void chainTasks(String first, String second) {
+		List<ChainableInvokable<?, ?>> chainedInvokables = chainedInvokable.get(first);
+		if (chainedInvokables == null) {
+			chainedInvokables = new ArrayList<ChainableInvokable<?, ?>>();
+		}
+		chainedInvokables.add((ChainableInvokable<?, ?>) invokableObjects.get(second));
+		chainedInvokable.put(first, chainedInvokables);
 
-		if (vertexClass.equals(StreamIterationHead.class)
-				|| vertexClass.equals(StreamIterationTail.class)) {
-			config.setIterationId(iterationIds.get(vertexName));
-			config.setIterationWaitTime(iterationWaitTime.get(vertexName));
+		List<StreamRecordSerializer<?>> chainedSerializers = chainedSerializer.get(first);
+		if (chainedSerializers == null) {
+			chainedSerializers = new ArrayList<StreamRecordSerializer<?>>();
 		}
+		chainedSerializers.add(typeSerializersIn1.get(second));
+		chainedSerializer.put(first, chainedSerializers);
 
-		if (inputFormatList.containsKey(vertexName)) {
-			vertex.setInputSplitSource(inputFormatList.get(vertexName));
+		List<String> chained = chainedVertices.get(first);
+		if (chained == null) {
+			chained = new ArrayList<String>();
 		}
+		chained.add(second);
+		chainedVertices.put(first, chained);
+
+		outEdgeList.put(first, outEdgeList.get(second));
+		typeSerializersOut1.put(first, typeSerializersOut1.get(second));
+		outPartitioning.put(first, outPartitioning.get(second));
+		outEdgeType.put(first, outEdgeType.get(second));
+		outEdgeNames.put(first, outEdgeNames.get(second));
+		outEdgeSelectAll.put(first, outEdgeSelectAll.get(second));
+		outPartitioning.put(first, outPartitioning.get(second));
+		bufferTimeout.put(first, bufferTimeout.get(second));
+		outputSelectors.put(first, outputSelectors.get(second));
 
-		streamVertices.put(vertexName, vertex);
 	}
 
 	/**
@@ -385,8 +466,8 @@ public class JobGraphBuilder {
 			downStreamVertex
 					.connectNewDataSetAsInput(upStreamVertex, DistributionPattern.POINTWISE);
 		} else {
-			downStreamVertex
-					.connectNewDataSetAsInput(upStreamVertex, DistributionPattern.ALL_TO_ALL);
+			downStreamVertex.connectNewDataSetAsInput(upStreamVertex,
+					DistributionPattern.ALL_TO_ALL);
 		}
 
 		if (LOG.isDebugEnabled()) {
@@ -468,7 +549,7 @@ public class JobGraphBuilder {
 		outEdgeList.get(upStreamVertexName).add(downStreamVertexName);
 		outEdgeType.get(upStreamVertexName).add(typeNumber);
 		inEdgeList.get(downStreamVertexName).add(upStreamVertexName);
-		connectionTypes.get(upStreamVertexName).add(partitionerObject);
+		outPartitioning.get(upStreamVertexName).add(partitionerObject);
 		outEdgeNames.get(upStreamVertexName).add(outputNames);
 		outEdgeSelectAll.get(upStreamVertexName).add(selectAll);
 	}
@@ -602,11 +683,12 @@ public class JobGraphBuilder {
 	 * provided.
 	 */
 	private void buildJobGraph() {
-		for (String vertexName : outEdgeList.keySet()) {
-			createVertex(vertexName);
+
+		for (String sourceName : sources) {
+			createVertex(sourceName);
 		}
 
-		for (String upStreamVertexName : outEdgeList.keySet()) {
+		for (String upStreamVertexName : builtVertices) {
 			int i = 0;
 
 			List<Integer> outEdgeTypeList = outEdgeType.get(upStreamVertexName);
@@ -621,7 +703,7 @@ public class JobGraphBuilder {
 				downStreamVertexConfig.setNumberOfInputs(inputNumber);
 
 				connect(upStreamVertexName, downStreamVertexName,
-						connectionTypes.get(upStreamVertexName).get(i));
+						outPartitioning.get(upStreamVertexName).get(i));
 				i++;
 			}
 		}
@@ -631,4 +713,8 @@ public class JobGraphBuilder {
 		setNumberOfJobOutputs();
 	}
 
+	public void setChaining(boolean chaining) {
+		this.chaining = chaining;
+	}
+
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
index 8837b85..22bcf92 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
@@ -27,6 +27,7 @@ import org.apache.commons.lang3.SerializationUtils;
 import org.apache.flink.api.common.functions.AbstractRichFunction;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.streaming.api.collector.OutputSelector;
+import org.apache.flink.streaming.api.invokable.ChainableInvokable;
 import org.apache.flink.streaming.api.invokable.StreamInvokable;
 import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
 import org.apache.flink.streaming.api.streamvertex.StreamVertexException;
@@ -39,6 +40,9 @@ public class StreamConfig {
 	private static final String INPUT_TYPE = "inputType_";
 	private static final String NUMBER_OF_OUTPUTS = "numberOfOutputs";
 	private static final String NUMBER_OF_INPUTS = "numberOfInputs";
+	private static final String NUMBER_OF_CHAINED_TASKS = "numOfChained";
+	private static final String CHAINED_IN_SERIALIZER = "chainedSerializer_";
+	private static final String CHAINED_INVOKABLE = "chainedInvokable_";
 	private static final String OUTPUT_NAME = "outputName_";
 	private static final String OUTPUT_SELECT_ALL = "outputSelectAll_";
 	private static final String PARTITIONER_OBJECT = "partitionerObject_";
@@ -46,8 +50,6 @@ public class StreamConfig {
 	private static final String ITERATION_ID = "iteration-id";
 	private static final String OUTPUT_SELECTOR = "outputSelector";
 	private static final String DIRECTED_EMIT = "directedEmit";
-	private static final String FUNCTION_NAME = "operatorName";
-	private static final String VERTEX_NAME = "vertexName";
 	private static final String SERIALIZEDUDF = "serializedudf";
 	private static final String USER_FUNCTION = "userfunction";
 	private static final String BUFFER_TIMEOUT = "bufferTimeout";
@@ -164,18 +166,6 @@ public class StreamConfig {
 		}
 	}
 
-	public void setVertexName(String vertexName) {
-		config.setString(VERTEX_NAME, vertexName);
-	}
-
-	public String getVertexName() {
-		return config.getString(VERTEX_NAME, null);
-	}
-
-	public String getFunctionName() {
-		return config.getString(FUNCTION_NAME, "");
-	}
-
 	public void setDirectedEmit(boolean directedEmit) {
 		config.setBoolean(DIRECTED_EMIT, directedEmit);
 	}
@@ -261,7 +251,7 @@ public class StreamConfig {
 	}
 
 	@SuppressWarnings("unchecked")
-	public List<String> getOutputName(int outputIndex) {
+	public List<String> getOutputNames(int outputIndex) {
 		return (List<String>) SerializationUtils.deserialize(config.getBytes(OUTPUT_NAME
 				+ outputIndex, null));
 	}
@@ -316,4 +306,40 @@ public class StreamConfig {
 		}
 	}
 
+	public int getNumberofChainedTasks() {
+		return config.getInteger(NUMBER_OF_CHAINED_TASKS, 0);
+	}
+
+	public void setNumberofChainedTasks(int n) {
+		config.setInteger(NUMBER_OF_CHAINED_TASKS, n);
+	}
+
+	public ChainableInvokable<?, ?> getChainedInvokable(int chainedTaskIndex, ClassLoader cl) {
+		try {
+			return (ChainableInvokable<?, ?>) InstantiationUtil.readObjectFromConfig(this.config,
+					CHAINED_INVOKABLE + chainedTaskIndex, cl);
+		} catch (Exception e) {
+			throw new RuntimeException("Could not instantiate invokable.");
+		}
+	}
+
+	public StreamRecordSerializer<?> getChainedInSerializer(int chainedTaskIndex, ClassLoader cl) {
+		try {
+			return (StreamRecordSerializer<?>) InstantiationUtil.readObjectFromConfig(this.config,
+					CHAINED_IN_SERIALIZER + chainedTaskIndex, cl);
+		} catch (Exception e) {
+			throw new RuntimeException("Could not instantiate serializer.");
+		}
+	}
+
+	public void setChainedSerializer(StreamRecordSerializer<?> typeWrapper, int chainedTaskIndex) {
+		config.setBytes(CHAINED_IN_SERIALIZER + chainedTaskIndex,
+				SerializationUtils.serialize(typeWrapper));
+	}
+
+	public void setChainedInvokable(ChainableInvokable<?, ?> invokable, int chainedTaskIndex) {
+		config.setBytes(CHAINED_INVOKABLE + chainedTaskIndex,
+				SerializationUtils.serialize(invokable));
+	}
+
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
index d782d08..b7e57e0 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
@@ -37,7 +37,7 @@ public class CollectorWrapper<OUT> implements Collector<OUT> {
 	@Override
 	public void collect(OUT record) {
 		for(Collector<OUT> output: outputs){
-			output.collect(record);;
+			output.collect(record);
 		}
 	}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
index d7467d1..88feb5d 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
@@ -67,7 +67,7 @@ public class IterativeDataStream<IN> extends
 				iterationTail.getParallelism(), waitTime);
 
 		jobGraphBuilder.setIterationSourceSettings(iterationID.toString(), iterationTail.getId());
-		connectGraph(iterationTail, iterationSink.getId(), 0);
+		connectGraph(iterationTail.forward(), iterationSink.getId(), 0);
 		return iterationTail;
 	}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
index 5a8261e..7a0abe4 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
@@ -182,6 +182,11 @@ public class SingleOutputStreamOperator<OUT, O extends SingleOutputStreamOperato
 		return (SingleOutputStreamOperator<OUT, O>) super.distribute();
 	}
 
+	@SuppressWarnings("unchecked")
+	public SingleOutputStreamOperator<OUT, O> global() {
+		return (SingleOutputStreamOperator<OUT, O>) super.global();
+	}
+
 	@Override
 	public SingleOutputStreamOperator<OUT, O> copy() {
 		return new SingleOutputStreamOperator<OUT, O>(this);

http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
index 87f2287..08fd19b 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
@@ -434,7 +434,7 @@ public abstract class StreamExecutionEnvironment {
 		DataStreamSource<OUT> returnStream = new DataStreamSource<OUT>(this, sourceName,
 				outTypeInfo, isParallel);
 
-		jobGraphBuilder.addStreamVertex(returnStream.getId(), new SourceInvokable<OUT>(function),
+		jobGraphBuilder.addSourceVertex(returnStream.getId(), new SourceInvokable<OUT>(function),
 				null, outTypeInfo, sourceName, dop);
 
 		return returnStream;

http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/ChainableInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/ChainableInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/ChainableInvokable.java
new file mode 100644
index 0000000..373b4e8
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/ChainableInvokable.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable;
+
+import org.apache.flink.api.common.functions.Function;
+import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
+import org.apache.flink.util.Collector;
+
+public abstract class ChainableInvokable<IN, OUT> extends StreamInvokable<IN, OUT> implements
+		Collector<IN> {
+
+	private static final long serialVersionUID = 1L;
+
+	public ChainableInvokable(Function userFunction) {
+		super(userFunction);
+	}
+
+	public void setup(Collector<OUT> collector, StreamRecordSerializer<IN> inSerializer) {
+		this.collector = collector;
+		this.inSerializer = inSerializer;
+		this.objectSerializer = inSerializer.getObjectSerializer();
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/SinkInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/SinkInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/SinkInvokable.java
index 74591a8..13a6ba1 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/SinkInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/SinkInvokable.java
@@ -19,7 +19,7 @@ package org.apache.flink.streaming.api.invokable;
 
 import org.apache.flink.streaming.api.function.sink.SinkFunction;
 
-public class SinkInvokable<IN> extends StreamInvokable<IN, IN> {
+public class SinkInvokable<IN> extends ChainableInvokable<IN, IN> {
 	private static final long serialVersionUID = 1L;
 
 	private SinkFunction<IN> sinkFunction;
@@ -38,7 +38,13 @@ public class SinkInvokable<IN> extends StreamInvokable<IN, IN> {
 
 	@Override
 	protected void callUserFunction() throws Exception {
-		sinkFunction.invoke((IN) nextRecord.getObject());
+		sinkFunction.invoke(nextObject);
+	}
+
+	@Override
+	public void collect(IN record) {
+		nextObject = copy(record);
+		callUserFunctionAndLogException();
 	}
 
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java
index 87ad4e0..25e9221 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java
@@ -37,7 +37,7 @@ import org.slf4j.LoggerFactory;
 /**
  * The StreamInvokable represents the base class for all invokables in the
  * streaming topology.
- *
+ * 
  * @param <OUT>
  *            The output type of the invokable
  */
@@ -52,6 +52,7 @@ public abstract class StreamInvokable<IN, OUT> implements Serializable {
 	protected StreamRecordSerializer<IN> inSerializer;
 	protected TypeSerializer<IN> objectSerializer;
 	protected StreamRecord<IN> nextRecord;
+	protected IN nextObject;
 	protected boolean isMutable;
 
 	protected Collector<OUT> collector;
@@ -92,7 +93,13 @@ public abstract class StreamInvokable<IN, OUT> implements Serializable {
 	protected StreamRecord<IN> readNext() {
 		this.nextRecord = inSerializer.createInstance();
 		try {
-			return nextRecord = recordIterator.next(nextRecord);
+			nextRecord = recordIterator.next(nextRecord);
+			try {
+				nextObject = nextRecord.getObject();
+			} catch (NullPointerException e) {
+				// end of stream
+			}
+			return nextRecord;
 		} catch (IOException e) {
 			throw new RuntimeException("Could not read next record.");
 		}
@@ -135,10 +142,14 @@ public abstract class StreamInvokable<IN, OUT> implements Serializable {
 	 * RichFunction class
 	 * 
 	 */
-	public void close() throws Exception {
+	public void close() {
 		isRunning = false;
 		collector.close();
-		FunctionUtils.closeFunction(userFunction);
+		try {
+			FunctionUtils.closeFunction(userFunction);
+		} catch (Exception e) {
+			throw new RuntimeException("Error when closing the function: " + e.getMessage());
+		}
 	}
 
 	public void setRuntimeContext(RuntimeContext t) {
@@ -148,4 +159,8 @@ public abstract class StreamInvokable<IN, OUT> implements Serializable {
 	protected IN copy(IN record) {
 		return objectSerializer.copy(record);
 	}
+
+	public boolean isChainable() {
+		return this instanceof ChainableInvokable;
+	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/CounterInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/CounterInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/CounterInvokable.java
index 0267253..3fc314c 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/CounterInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/CounterInvokable.java
@@ -17,9 +17,9 @@
 
 package org.apache.flink.streaming.api.invokable.operator;
 
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.invokable.ChainableInvokable;
 
-public class CounterInvokable<IN> extends StreamInvokable<IN, Long> {
+public class CounterInvokable<IN> extends ChainableInvokable<IN, Long> {
 	private static final long serialVersionUID = 1L;
 
 	Long count = 0L;
@@ -34,4 +34,10 @@ public class CounterInvokable<IN> extends StreamInvokable<IN, Long> {
 			collector.collect(++count);
 		}
 	}
+
+	@Override
+	public void collect(IN record) {
+		collector.collect(++count);
+	}
+
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FilterInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FilterInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FilterInvokable.java
index 48b8ad0..0c8298e 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FilterInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FilterInvokable.java
@@ -18,9 +18,9 @@
 package org.apache.flink.streaming.api.invokable.operator;
 
 import org.apache.flink.api.common.functions.FilterFunction;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.invokable.ChainableInvokable;
 
-public class FilterInvokable<IN> extends StreamInvokable<IN, IN> {
+public class FilterInvokable<IN> extends ChainableInvokable<IN, IN> {
 
 	private static final long serialVersionUID = 1L;
 
@@ -36,14 +36,20 @@ public class FilterInvokable<IN> extends StreamInvokable<IN, IN> {
 	public void invoke() throws Exception {
 		while (readNext() != null) {
 			callUserFunctionAndLogException();
-			if (collect) {
-				collector.collect(nextRecord.getObject());
-			}
 		}
 	}
 
 	@Override
 	protected void callUserFunction() throws Exception {
-		collect = filterFunction.filter(copy(nextRecord.getObject()));
+		collect = filterFunction.filter(copy(nextObject));
+		if (collect) {
+			collector.collect(nextObject);
+		}
+	}
+
+	@Override
+	public void collect(IN record) {
+		nextObject = copy(record);
+		callUserFunctionAndLogException();
 	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FlatMapInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FlatMapInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FlatMapInvokable.java
index 8ff78eb..2a4081b 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FlatMapInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FlatMapInvokable.java
@@ -18,9 +18,9 @@
 package org.apache.flink.streaming.api.invokable.operator;
 
 import org.apache.flink.api.common.functions.FlatMapFunction;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.invokable.ChainableInvokable;
 
-public class FlatMapInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
+public class FlatMapInvokable<IN, OUT> extends ChainableInvokable<IN, OUT> {
 	private static final long serialVersionUID = 1L;
 
 	private FlatMapFunction<IN, OUT> flatMapper;
@@ -39,7 +39,13 @@ public class FlatMapInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
 
 	@Override
 	protected void callUserFunction() throws Exception {
-		flatMapper.flatMap(nextRecord.getObject(), collector);
+		flatMapper.flatMap(nextObject, collector);
+	}
+
+	@Override
+	public void collect(IN record) {
+		nextObject = copy(record);
+		callUserFunctionAndLogException();
 	}
 
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java
index fdcf520..84258d6 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java
@@ -40,7 +40,7 @@ public class GroupedReduceInvokable<IN> extends StreamReduceInvokable<IN> {
 	protected void reduce() throws Exception {
 		Object key = nextRecord.getKey(keySelector);
 		currentValue = values.get(key);
-		nextValue = nextRecord.getObject();
+		nextValue = nextObject;
 		if (currentValue != null) {
 			callUserFunctionAndLogException();
 			values.put(key, reduced);
@@ -56,4 +56,9 @@ public class GroupedReduceInvokable<IN> extends StreamReduceInvokable<IN> {
 		reduced = reducer.reduce(currentValue, nextValue);
 	}
 
+	@Override
+	public boolean isChainable() {
+		return false;
+	}
+
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/MapInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/MapInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/MapInvokable.java
index 6be96ec..7c8e577 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/MapInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/MapInvokable.java
@@ -18,9 +18,9 @@
 package org.apache.flink.streaming.api.invokable.operator;
 
 import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.invokable.ChainableInvokable;
 
-public class MapInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
+public class MapInvokable<IN, OUT> extends ChainableInvokable<IN, OUT> {
 	private static final long serialVersionUID = 1L;
 
 	private MapFunction<IN, OUT> mapper;
@@ -39,6 +39,12 @@ public class MapInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
 
 	@Override
 	protected void callUserFunction() throws Exception {
-		collector.collect(mapper.map(nextRecord.getObject()));
+		collector.collect(mapper.map(nextObject));
+	}
+
+	@Override
+	public void collect(IN record) {
+		nextObject = copy(record);
+		callUserFunctionAndLogException();
 	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceInvokable.java
index 5f5cb12..e1a56cc 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceInvokable.java
@@ -18,9 +18,9 @@
 package org.apache.flink.streaming.api.invokable.operator;
 
 import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.invokable.ChainableInvokable;
 
-public class StreamReduceInvokable<IN> extends StreamInvokable<IN, IN> {
+public class StreamReduceInvokable<IN> extends ChainableInvokable<IN, IN> {
 	private static final long serialVersionUID = 1L;
 
 	protected ReduceFunction<IN> reducer;
@@ -41,13 +41,15 @@ public class StreamReduceInvokable<IN> extends StreamInvokable<IN, IN> {
 	}
 
 	protected void reduce() throws Exception {
-		nextValue = nextRecord.getObject();
 		callUserFunctionAndLogException();
 
 	}
 
 	@Override
 	protected void callUserFunction() throws Exception {
+
+		nextValue = nextObject;
+
 		if (currentValue != null) {
 			currentValue = reducer.reduce(currentValue, nextValue);
 		} else {
@@ -57,4 +59,11 @@ public class StreamReduceInvokable<IN> extends StreamInvokable<IN, IN> {
 		collector.collect(currentValue);
 
 	}
+
+	@Override
+	public void collect(IN record) {
+		nextObject = copy(record);
+		callUserFunctionAndLogException();
+	}
+
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowInvokable.java
index 03219b7..93f597f 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowInvokable.java
@@ -175,7 +175,7 @@ public class CoWindowInvokable<IN1, IN2, OUT> extends CoInvokable<IN1, IN2, OUT>
 	}
 
 	@Override
-	public void close() throws Exception {
+	public void close() {
 		if (!window.miniBatchEnd()) {
 			callUserFunctionAndLogException();
 		}

http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
index 61a6eb4..457f3f8 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
@@ -17,26 +17,30 @@
 
 package org.apache.flink.streaming.api.streamvertex;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
 import org.apache.flink.runtime.plugable.SerializationDelegate;
 import org.apache.flink.streaming.api.StreamConfig;
+import org.apache.flink.streaming.api.collector.CollectorWrapper;
 import org.apache.flink.streaming.api.collector.DirectedOutputWrapper;
 import org.apache.flink.streaming.api.collector.OutputSelector;
-import org.apache.flink.streaming.api.collector.StreamOutputWrapper;
 import org.apache.flink.streaming.api.collector.StreamOutput;
+import org.apache.flink.streaming.api.collector.StreamOutputWrapper;
+import org.apache.flink.streaming.api.invokable.ChainableInvokable;
 import org.apache.flink.streaming.api.invokable.StreamInvokable;
 import org.apache.flink.streaming.api.streamrecord.StreamRecord;
 import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
 import org.apache.flink.streaming.io.StreamRecordWriter;
 import org.apache.flink.streaming.partitioner.StreamPartitioner;
+import org.apache.flink.util.Collector;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
-
 public class OutputHandler<OUT> {
 	private static final Logger LOG = LoggerFactory.getLogger(OutputHandler.class);
 
@@ -44,17 +48,21 @@ public class OutputHandler<OUT> {
 	private StreamConfig configuration;
 
 	private List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> outputs;
-	private StreamOutputWrapper<OUT> collector;
-	private long bufferTimeout;
+	private Collector<OUT> endCollector;
 
 	TypeInformation<OUT> outTypeInfo = null;
 	StreamRecordSerializer<OUT> outSerializer = null;
 	SerializationDelegate<StreamRecord<OUT>> outSerializationDelegate = null;
 
+	public List<ChainableInvokable<?, ?>> chainedInvokables;
+
+	private int numberOfChainedTasks;
+
 	public OutputHandler(StreamVertex<?, OUT> streamComponent) {
 		this.streamVertex = streamComponent;
 		this.outputs = new LinkedList<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>>();
 		this.configuration = new StreamConfig(streamComponent.getTaskConfiguration());
+		this.chainedInvokables = new ArrayList<ChainableInvokable<?, ?>>();
 
 		try {
 			setConfigOutputs();
@@ -62,6 +70,7 @@ public class OutputHandler<OUT> {
 			throw new StreamVertexException("Cannot register outputs for "
 					+ streamComponent.getClass().getSimpleName(), e);
 		}
+
 	}
 
 	public List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> getOutputs() {
@@ -69,18 +78,42 @@ public class OutputHandler<OUT> {
 	}
 
 	private void setConfigOutputs() {
-		setSerializers();
-		setCollector();
+		numberOfChainedTasks = configuration.getNumberofChainedTasks();
+		endCollector = createChainedOutputs(0);
+	}
 
-		int numberOfOutputs = configuration.getNumberOfOutputs();
-		bufferTimeout = configuration.getBufferTimeout();
+	@SuppressWarnings("unchecked")
+	private Collector<OUT> createChainedOutputs(int chainedTaskIndex) {
 
-		for (int i = 0; i < numberOfOutputs; i++) {
-			setPartitioner(i, outputs);
+		if (numberOfChainedTasks == chainedTaskIndex) {
+			return createEndCollector();
+		} else {
+			CollectorWrapper<OUT> chainedCollector = new CollectorWrapper<OUT>();
+
+			@SuppressWarnings("rawtypes")
+			ChainableInvokable chainableInvokable = configuration.getChainedInvokable(
+					chainedTaskIndex, streamVertex.getUserCodeClassLoader());
+
+			chainableInvokable.setup(
+					createChainedOutputs(chainedTaskIndex + 1),
+					configuration.getChainedInSerializer(chainedTaskIndex,
+							streamVertex.getUserCodeClassLoader()));
+
+			chainedInvokables.add(chainableInvokable);
+
+			chainedCollector.addChainedOutput((Collector<OUT>) chainableInvokable);
+
+			return chainedCollector;
 		}
+
 	}
 
-	private StreamOutputWrapper<OUT> setCollector() {
+	private Collector<OUT> createEndCollector() {
+
+		setSerializers();
+
+		StreamOutputWrapper<OUT> collector;
+
 		if (streamVertex.configuration.getDirectedEmit()) {
 			OutputSelector<OUT> outputSelector = streamVertex.configuration
 					.getOutputSelector(streamVertex.userClassLoader);
@@ -91,11 +124,17 @@ public class OutputHandler<OUT> {
 			collector = new StreamOutputWrapper<OUT>(streamVertex.getInstanceID(),
 					outSerializationDelegate);
 		}
+
+		int numberOfOutputs = configuration.getNumberOfOutputs();
+		for (int i = 0; i < numberOfOutputs; i++) {
+			collector = (StreamOutputWrapper<OUT>) setPartitioner(i, collector);
+		}
+
 		return collector;
 	}
 
-	public StreamOutputWrapper<OUT> getCollector() {
-		return collector;
+	public Collector<OUT> getCollector() {
+		return endCollector;
 	}
 
 	void setSerializers() {
@@ -106,8 +145,7 @@ public class OutputHandler<OUT> {
 		}
 	}
 
-	void setPartitioner(int outputNumber,
-			List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> outputs) {
+	Collector<OUT> setPartitioner(int outputNumber, StreamOutputWrapper<OUT> endCollector) {
 		StreamPartitioner<OUT> outputPartitioner = null;
 
 		try {
@@ -121,6 +159,8 @@ public class OutputHandler<OUT> {
 
 		RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output;
 
+		long bufferTimeout = configuration.getBufferTimeout();
+
 		if (bufferTimeout >= 0) {
 			output = new StreamRecordWriter<SerializationDelegate<StreamRecord<OUT>>>(streamVertex
 					.getEnvironment().getWriter(outputNumber), outputPartitioner, bufferTimeout);
@@ -139,18 +179,20 @@ public class OutputHandler<OUT> {
 		}
 
 		outputs.add(output);
-		List<String> outputName = configuration.getOutputName(outputNumber);
+		List<String> outputNames = configuration.getOutputNames(outputNumber);
 		boolean isSelectAllOutput = configuration.getSelectAll(outputNumber);
 
-		if (collector != null) {
-			collector
-					.addOutput(new StreamOutput<OUT>(output, isSelectAllOutput ? null : outputName));
+		if (endCollector != null) {
+			endCollector.addOutput(new StreamOutput<OUT>(output, isSelectAllOutput ? null
+					: outputNames));
 		}
 
 		if (LOG.isTraceEnabled()) {
 			LOG.trace("Partitioner set: {} with {} outputs for {}", outputPartitioner.getClass()
 					.getSimpleName(), outputNumber, streamVertex.getClass().getSimpleName());
 		}
+
+		return endCollector;
 	}
 
 	public void flushOutputs() throws IOException, InterruptedException {

http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertex.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertex.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertex.java
index 93e290f..994b1fa 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertex.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertex.java
@@ -22,6 +22,7 @@ import java.util.Map;
 import org.apache.flink.runtime.execution.Environment;
 import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
 import org.apache.flink.streaming.api.StreamConfig;
+import org.apache.flink.streaming.api.invokable.ChainableInvokable;
 import org.apache.flink.streaming.api.invokable.StreamInvokable;
 import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
 import org.apache.flink.streaming.io.CoReaderIterator;
@@ -35,11 +36,8 @@ public class StreamVertex<IN, OUT> extends AbstractInvokable implements StreamTa
 
 	protected StreamConfig configuration;
 	protected int instanceID;
-	protected String name;
 	private static int numVertices = 0;
 
-	protected String functionName;
-
 	private InputHandler<IN> inputHandler;
 	protected OutputHandler<OUT> outputHandler;
 	private StreamInvokable<IN, OUT> userInvokable;
@@ -70,17 +68,26 @@ public class StreamVertex<IN, OUT> extends AbstractInvokable implements StreamTa
 	protected void initialize() {
 		this.userClassLoader = getUserCodeClassLoader();
 		this.configuration = new StreamConfig(getTaskConfiguration());
-		this.name = configuration.getVertexName();
-		this.functionName = configuration.getFunctionName();
 		this.states = configuration.getOperatorStates(userClassLoader);
-		this.context = createRuntimeContext(name, this.states);
+		this.context = createRuntimeContext(getEnvironment().getTaskName(), this.states);
 	}
 
 	protected <T> void invokeUserFunction(StreamInvokable<?, T> userInvokable) throws Exception {
 		userInvokable.setRuntimeContext(context);
 		userInvokable.open(getTaskConfiguration());
+
+		for (ChainableInvokable<?, ?> invokable : outputHandler.chainedInvokables) {
+			invokable.setRuntimeContext(context);
+			invokable.open(getTaskConfiguration());
+		}
+
 		userInvokable.invoke();
 		userInvokable.close();
+
+		for (ChainableInvokable<?, ?> invokable : outputHandler.chainedInvokables) {
+			invokable.close();
+		}
+
 	}
 
 	public void setInputsOutputs() {
@@ -94,14 +101,15 @@ public class StreamVertex<IN, OUT> extends AbstractInvokable implements StreamTa
 	}
 
 	public String getName() {
-		return name;
+		return getEnvironment().getTaskName();
 	}
 
 	public int getInstanceID() {
 		return instanceID;
 	}
 
-	public StreamingRuntimeContext createRuntimeContext(String taskName, Map<String, OperatorState<?>> states) {
+	public StreamingRuntimeContext createRuntimeContext(String taskName,
+			Map<String, OperatorState<?>> states) {
 		Environment env = getEnvironment();
 		return new StreamingRuntimeContext(taskName, env, getUserCodeClassLoader(), states);
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/iteration/IterateExample.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/iteration/IterateExample.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/iteration/IterateExample.java
index a5dc68a..998e818 100644
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/iteration/IterateExample.java
+++ b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/iteration/IterateExample.java
@@ -76,8 +76,7 @@ public class IterateExample {
 
 		// apply the step function to add new random value to the tuple and to
 		// increment the counter and split the output with the output selector
-		SplitDataStream<Tuple2<Double, Integer>> step = it.map(new Step()).shuffle()
-				.split(new MySelector());
+		SplitDataStream<Tuple2<Double, Integer>> step = it.map(new Step()).split(new MySelector());
 
 		// close the iteration by selecting the tuples that were directed to the
 		// 'iterate' channel in the output selector


[6/8] flink git commit: [streaming] JobGraphbuilder separated to StreamGraph and StreamingJobGraphGenerator

Posted by mb...@apache.org.
[streaming] JobGraphbuilder separated to StreamGraph and StreamingJobGraphGenerator


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/e3b608ce
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/e3b608ce
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/e3b608ce

Branch: refs/heads/master
Commit: e3b608ce20f2c06b6c9ce8f231b6f53ab73f1de5
Parents: 7dbb55e
Author: Gyula Fora <gy...@apache.org>
Authored: Wed Jan 21 00:34:11 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Wed Jan 21 16:06:34 2015 +0100

----------------------------------------------------------------------
 .../flink/streaming/api/JobGraphBuilder.java    | 675 -------------------
 .../flink/streaming/api/StreamConfig.java       |  28 +-
 .../apache/flink/streaming/api/StreamGraph.java | 545 +++++++++++++++
 .../api/StreamingJobGraphGenerator.java         | 286 ++++++++
 .../api/datastream/ConnectedDataStream.java     |   8 +-
 .../streaming/api/datastream/DataStream.java    |  18 +-
 .../api/datastream/IterativeDataStream.java     |   4 +-
 .../datastream/SingleOutputStreamOperator.java  |  20 +-
 .../temporaloperator/StreamCrossOperator.java   |   4 +-
 .../temporaloperator/StreamJoinOperator.java    |   2 +-
 .../api/environment/LocalStreamEnvironment.java |   4 +-
 .../environment/RemoteStreamEnvironment.java    |   4 +-
 .../environment/StreamContextEnvironment.java   |   4 +-
 .../environment/StreamExecutionEnvironment.java |  18 +-
 .../operator/GroupedReduceInvokable.java        |   1 -
 .../operator/GroupedWindowInvokable.java        |   3 +-
 .../api/invokable/operator/WindowInvokable.java |   3 +-
 .../api/streamvertex/OutputHandler.java         |  64 +-
 .../streaming/util/TestStreamEnvironment.java   |   2 +-
 .../api/scala/StreamCrossOperator.scala         |   4 +-
 .../api/scala/StreamJoinOperator.scala          |   2 +-
 21 files changed, 914 insertions(+), 785 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
deleted file mode 100644
index 6ae97c9..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
+++ /dev/null
@@ -1,675 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.flink.api.common.io.InputFormat;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.jobgraph.AbstractJobVertex;
-import org.apache.flink.runtime.jobgraph.DistributionPattern;
-import org.apache.flink.runtime.jobgraph.JobGraph;
-import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
-import org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup;
-import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
-import org.apache.flink.streaming.api.collector.OutputSelector;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
-import org.apache.flink.streaming.api.invokable.StreamInvokable.ChainingStrategy;
-import org.apache.flink.streaming.api.invokable.operator.co.CoInvokable;
-import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
-import org.apache.flink.streaming.api.streamvertex.CoStreamVertex;
-import org.apache.flink.streaming.api.streamvertex.StreamIterationHead;
-import org.apache.flink.streaming.api.streamvertex.StreamIterationTail;
-import org.apache.flink.streaming.api.streamvertex.StreamVertex;
-import org.apache.flink.streaming.partitioner.StreamPartitioner;
-import org.apache.flink.streaming.partitioner.StreamPartitioner.PartitioningStrategy;
-import org.apache.flink.streaming.state.OperatorState;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Object for building Apache Flink stream processing job graphs
- */
-public class JobGraphBuilder {
-
-	private static final Logger LOG = LoggerFactory.getLogger(JobGraphBuilder.class);
-	private final static String DEAFULT_JOB_NAME = "Streaming Job";
-	private JobGraph jobGraph;
-
-	private boolean chaining = true;
-
-	// Graph attributes
-	private Map<String, AbstractJobVertex> streamVertices;
-	private Map<String, Integer> vertexParallelism;
-	private Map<String, Long> bufferTimeout;
-	private Map<String, List<String>> outEdgeList;
-	private Map<String, List<Integer>> outEdgeIndex;
-	private Map<String, List<List<String>>> outEdgeNames;
-	private Map<String, List<Boolean>> outEdgeSelectAll;
-	private Map<String, List<String>> inEdgeList;
-	private Map<String, List<StreamPartitioner<?>>> outPartitioning;
-	private Map<String, String> operatorNames;
-	private Map<String, StreamInvokable<?, ?>> invokableObjects;
-	private Map<String, StreamRecordSerializer<?>> typeSerializersIn1;
-	private Map<String, StreamRecordSerializer<?>> typeSerializersIn2;
-	private Map<String, StreamRecordSerializer<?>> typeSerializersOut1;
-	private Map<String, StreamRecordSerializer<?>> typeSerializersOut2;
-	private Map<String, byte[]> outputSelectors;
-	private Map<String, Class<? extends AbstractInvokable>> vertexClasses;
-	private Map<String, Integer> iterationIds;
-	private Map<Integer, String> iterationIDtoHeadName;
-	private Map<Integer, String> iterationIDtoTailName;
-	private Map<String, Integer> iterationTailCount;
-	private Map<String, Long> iterationWaitTime;
-	private Map<String, Map<String, OperatorState<?>>> operatorStates;
-	private Map<String, InputFormat<String, ?>> inputFormatList;
-	private Map<String, Map<String, StreamConfig>> chainedConfigs;
-	private Map<String, StreamConfig> vertexConfigs;
-
-	private Set<String> sources;
-	private Set<String> builtNodes;
-
-	/**
-	 * Creates an new {@link JobGraph} with the given name. A JobGraph is a DAG
-	 * and consists of sources, tasks (intermediate vertices) and sinks.
-	 */
-	public JobGraphBuilder() {
-
-		initGraph();
-
-		if (LOG.isDebugEnabled()) {
-			LOG.debug("JobGraph created");
-		}
-	}
-
-	public void initGraph() {
-		streamVertices = new HashMap<String, AbstractJobVertex>();
-		vertexParallelism = new HashMap<String, Integer>();
-		bufferTimeout = new HashMap<String, Long>();
-		outEdgeList = new HashMap<String, List<String>>();
-		outEdgeIndex = new HashMap<String, List<Integer>>();
-		outEdgeNames = new HashMap<String, List<List<String>>>();
-		outEdgeSelectAll = new HashMap<String, List<Boolean>>();
-		inEdgeList = new HashMap<String, List<String>>();
-		outPartitioning = new HashMap<String, List<StreamPartitioner<?>>>();
-		operatorNames = new HashMap<String, String>();
-		invokableObjects = new HashMap<String, StreamInvokable<?, ?>>();
-		typeSerializersIn1 = new HashMap<String, StreamRecordSerializer<?>>();
-		typeSerializersIn2 = new HashMap<String, StreamRecordSerializer<?>>();
-		typeSerializersOut1 = new HashMap<String, StreamRecordSerializer<?>>();
-		typeSerializersOut2 = new HashMap<String, StreamRecordSerializer<?>>();
-		outputSelectors = new HashMap<String, byte[]>();
-		vertexClasses = new HashMap<String, Class<? extends AbstractInvokable>>();
-		iterationIds = new HashMap<String, Integer>();
-		iterationIDtoHeadName = new HashMap<Integer, String>();
-		iterationIDtoTailName = new HashMap<Integer, String>();
-		iterationTailCount = new HashMap<String, Integer>();
-		iterationWaitTime = new HashMap<String, Long>();
-		operatorStates = new HashMap<String, Map<String, OperatorState<?>>>();
-		inputFormatList = new HashMap<String, InputFormat<String, ?>>();
-		chainedConfigs = new HashMap<String, Map<String, StreamConfig>>();
-		vertexConfigs = new HashMap<String, StreamConfig>();
-
-		sources = new HashSet<String>();
-		builtNodes = new HashSet<String>();
-	}
-
-	/**
-	 * Adds a vertex to the streaming JobGraph with the given parameters
-	 * 
-	 * @param vertexName
-	 *            Name of the vertex
-	 * @param invokableObject
-	 *            User defined operator
-	 * @param inTypeInfo
-	 *            Input type for serialization
-	 * @param outTypeInfo
-	 *            Output type for serialization
-	 * @param operatorName
-	 *            Operator type
-	 * @param parallelism
-	 *            Number of parallel instances created
-	 */
-	public <IN, OUT> void addStreamVertex(String vertexName,
-			StreamInvokable<IN, OUT> invokableObject, TypeInformation<IN> inTypeInfo,
-			TypeInformation<OUT> outTypeInfo, String operatorName, int parallelism) {
-
-		addVertex(vertexName, StreamVertex.class, invokableObject, operatorName, parallelism);
-
-		StreamRecordSerializer<IN> inSerializer = inTypeInfo != null ? new StreamRecordSerializer<IN>(
-				inTypeInfo) : null;
-		StreamRecordSerializer<OUT> outSerializer = outTypeInfo != null ? new StreamRecordSerializer<OUT>(
-				outTypeInfo) : null;
-
-		addTypeSerializers(vertexName, inSerializer, null, outSerializer, null);
-
-		if (LOG.isDebugEnabled()) {
-			LOG.debug("Vertex: {}", vertexName);
-		}
-	}
-
-	public <IN, OUT> void addSourceVertex(String vertexName,
-			StreamInvokable<IN, OUT> invokableObject, TypeInformation<IN> inTypeInfo,
-			TypeInformation<OUT> outTypeInfo, String operatorName, int parallelism) {
-		addStreamVertex(vertexName, invokableObject, inTypeInfo, outTypeInfo, operatorName,
-				parallelism);
-		sources.add(vertexName);
-	}
-
-	/**
-	 * Adds a vertex for the iteration head to the {@link JobGraph}. The
-	 * iterated values will be fed from this vertex back to the graph.
-	 * 
-	 * @param vertexName
-	 *            Name of the vertex
-	 * @param iterationHead
-	 *            Id of the iteration head
-	 * @param iterationID
-	 *            ID of iteration for multiple iterations
-	 * @param parallelism
-	 *            Number of parallel instances created
-	 * @param waitTime
-	 *            Max wait time for next record
-	 */
-	public void addIterationHead(String vertexName, String iterationHead, Integer iterationID,
-			int parallelism, long waitTime) {
-
-		addVertex(vertexName, StreamIterationHead.class, null, null, parallelism);
-
-		chaining = false;
-
-		iterationIds.put(vertexName, iterationID);
-		iterationIDtoHeadName.put(iterationID, vertexName);
-
-		setSerializersFrom(iterationHead, vertexName);
-
-		setEdge(vertexName, iterationHead, outPartitioning
-				.get(inEdgeList.get(iterationHead).get(0)).get(0), 0, new ArrayList<String>(),
-				false);
-
-		iterationWaitTime.put(iterationIDtoHeadName.get(iterationID), waitTime);
-
-		if (LOG.isDebugEnabled()) {
-			LOG.debug("ITERATION SOURCE: {}", vertexName);
-		}
-
-		sources.add(vertexName);
-	}
-
-	/**
-	 * Adds a vertex for the iteration tail to the {@link JobGraph}. The values
-	 * intended to be iterated will be sent to this sink from the iteration
-	 * head.
-	 * 
-	 * @param vertexName
-	 *            Name of the vertex
-	 * @param iterationTail
-	 *            Id of the iteration tail
-	 * @param iterationID
-	 *            ID of iteration for mulitple iterations
-	 * @param parallelism
-	 *            Number of parallel instances created
-	 * @param waitTime
-	 *            Max waiting time for next record
-	 */
-	public void addIterationTail(String vertexName, String iterationTail, Integer iterationID,
-			int parallelism, long waitTime) {
-
-		if (bufferTimeout.get(iterationTail) == 0) {
-			throw new RuntimeException("Buffer timeout 0 at iteration tail is not supported.");
-		}
-
-		addVertex(vertexName, StreamIterationTail.class, null, null, parallelism);
-
-		iterationIds.put(vertexName, iterationID);
-		iterationIDtoTailName.put(iterationID, vertexName);
-
-		setSerializersFrom(iterationTail, vertexName);
-		iterationWaitTime.put(iterationIDtoTailName.get(iterationID), waitTime);
-
-		if (LOG.isDebugEnabled()) {
-			LOG.debug("ITERATION SINK: {}", vertexName);
-		}
-
-	}
-
-	public <IN1, IN2, OUT> void addCoTask(String vertexName,
-			CoInvokable<IN1, IN2, OUT> taskInvokableObject, TypeInformation<IN1> in1TypeInfo,
-			TypeInformation<IN2> in2TypeInfo, TypeInformation<OUT> outTypeInfo,
-			String operatorName, int parallelism) {
-
-		addVertex(vertexName, CoStreamVertex.class, taskInvokableObject, operatorName, parallelism);
-
-		addTypeSerializers(vertexName, new StreamRecordSerializer<IN1>(in1TypeInfo),
-				new StreamRecordSerializer<IN2>(in2TypeInfo), new StreamRecordSerializer<OUT>(
-						outTypeInfo), null);
-
-		if (LOG.isDebugEnabled()) {
-			LOG.debug("CO-TASK: {}", vertexName);
-		}
-	}
-
-	/**
-	 * Sets vertex parameters in the JobGraph
-	 * 
-	 * @param vertexName
-	 *            Name of the vertex
-	 * @param vertexClass
-	 *            The class of the vertex
-	 * @param invokableObjectject
-	 *            The user defined invokable object
-	 * @param operatorName
-	 *            Type of the user defined operator
-	 * @param parallelism
-	 *            Number of parallel instances created
-	 */
-	private void addVertex(String vertexName, Class<? extends AbstractInvokable> vertexClass,
-			StreamInvokable<?, ?> invokableObject, String operatorName, int parallelism) {
-
-		vertexClasses.put(vertexName, vertexClass);
-		setParallelism(vertexName, parallelism);
-		invokableObjects.put(vertexName, invokableObject);
-		operatorNames.put(vertexName, operatorName);
-		outEdgeList.put(vertexName, new ArrayList<String>());
-		outEdgeIndex.put(vertexName, new ArrayList<Integer>());
-		outEdgeNames.put(vertexName, new ArrayList<List<String>>());
-		outEdgeSelectAll.put(vertexName, new ArrayList<Boolean>());
-		inEdgeList.put(vertexName, new ArrayList<String>());
-		outPartitioning.put(vertexName, new ArrayList<StreamPartitioner<?>>());
-		iterationTailCount.put(vertexName, 0);
-	}
-
-	private void addTypeSerializers(String vertexName, StreamRecordSerializer<?> in1,
-			StreamRecordSerializer<?> in2, StreamRecordSerializer<?> out1,
-			StreamRecordSerializer<?> out2) {
-		typeSerializersIn1.put(vertexName, in1);
-		typeSerializersIn2.put(vertexName, in2);
-		typeSerializersOut1.put(vertexName, out1);
-		typeSerializersOut2.put(vertexName, out2);
-	}
-
-	private List<Tuple2<String, String>> createChain(String startNode, String current) {
-
-		if (!builtNodes.contains(startNode)) {
-
-			List<Tuple2<String, String>> transitiveOutEdges = new ArrayList<Tuple2<String, String>>();
-			List<String> chainableOutputs = new ArrayList<String>();
-			List<String> nonChainableOutputs = new ArrayList<String>();
-
-			for (String outName : outEdgeList.get(current)) {
-				if (isChainable(current, outName)) {
-					chainableOutputs.add(outName);
-				} else {
-					nonChainableOutputs.add(outName);
-				}
-
-			}
-
-			for (String chainable : chainableOutputs) {
-				transitiveOutEdges.addAll(createChain(startNode, chainable));
-			}
-
-			for (String nonChainable : nonChainableOutputs) {
-				transitiveOutEdges.add(new Tuple2<String, String>(current, nonChainable));
-				transitiveOutEdges.addAll(createChain(nonChainable, nonChainable));
-			}
-
-			StreamConfig config = current.equals(startNode) ? createProcessingVertex(startNode)
-					: new StreamConfig(new Configuration());
-
-			setVertexConfig(current, config, chainableOutputs, nonChainableOutputs);
-
-			if (current.equals(startNode)) {
-
-				config.setChainStart();
-				config.setRecordWriterOrder(transitiveOutEdges);
-
-				for (Tuple2<String, String> edge : transitiveOutEdges) {
-					connect(startNode, edge);
-				}
-
-				vertexConfigs.get(startNode).setTransitiveChainedTaskConfigs(
-						chainedConfigs.get(startNode));
-
-			} else {
-
-				Map<String, StreamConfig> chainedConfs = chainedConfigs.get(startNode);
-
-				if (chainedConfs == null) {
-					chainedConfigs.put(startNode, new HashMap<String, StreamConfig>());
-				}
-				chainedConfigs.get(startNode).put(current, config);
-			}
-
-			return transitiveOutEdges;
-
-		} else {
-			return new ArrayList<Tuple2<String, String>>();
-		}
-	}
-
-	private StreamConfig createProcessingVertex(String vertexName) {
-
-		AbstractJobVertex vertex = new AbstractJobVertex(vertexName);
-
-		this.jobGraph.addVertex(vertex);
-
-		int parallelism = vertexParallelism.get(vertexName);
-
-		vertex.setInvokableClass(vertexClasses.get(vertexName));
-		vertex.setParallelism(parallelism);
-		if (LOG.isDebugEnabled()) {
-			LOG.debug("Parallelism set: {} for {}", parallelism, vertexName);
-		}
-
-		if (inputFormatList.containsKey(vertexName)) {
-			vertex.setInputSplitSource(inputFormatList.get(vertexName));
-		}
-
-		streamVertices.put(vertexName, vertex);
-		builtNodes.add(vertexName);
-
-		return new StreamConfig(vertex.getConfiguration());
-	}
-
-	private void setVertexConfig(String vertexName, StreamConfig config,
-			List<String> chainableOutputs, List<String> nonChainableOutputs) {
-
-		StreamInvokable<?, ?> invokableObject = invokableObjects.get(vertexName);
-		byte[] outputSelector = outputSelectors.get(vertexName);
-		Class<? extends AbstractInvokable> vertexClass = vertexClasses.get(vertexName);
-		Map<String, OperatorState<?>> state = operatorStates.get(vertexName);
-
-		config.setVertexName(vertexName);
-
-		config.setBufferTimeout(bufferTimeout.get(vertexName));
-
-		config.setTypeSerializerIn1(typeSerializersIn1.get(vertexName));
-		config.setTypeSerializerIn2(typeSerializersIn2.get(vertexName));
-		config.setTypeSerializerOut1(typeSerializersOut1.get(vertexName));
-		config.setTypeSerializerOut2(typeSerializersOut2.get(vertexName));
-
-		config.setUserInvokable(invokableObject);
-		config.setOutputSelector(outputSelector);
-		config.setOperatorStates(state);
-
-		config.setNumberOfOutputs(nonChainableOutputs.size());
-		config.setOutputs(nonChainableOutputs);
-		config.setChainedOutputs(chainableOutputs);
-
-		if (vertexClass.equals(StreamIterationHead.class)
-				|| vertexClass.equals(StreamIterationTail.class)) {
-			config.setIterationId(iterationIds.get(vertexName));
-			config.setIterationWaitTime(iterationWaitTime.get(vertexName));
-		}
-
-		vertexConfigs.put(vertexName, config);
-	}
-
-	private boolean isChainable(String vertexName, String outName) {
-		return inEdgeList.get(outName).size() == 1
-				&& invokableObjects.get(outName) != null
-				&& outputSelectors.get(vertexName) == null
-				&& invokableObjects.get(outName).getChainingStrategy() == ChainingStrategy.ALWAYS
-				&& (invokableObjects.get(vertexName).getChainingStrategy() == ChainingStrategy.HEAD || invokableObjects
-						.get(vertexName).getChainingStrategy() == ChainingStrategy.ALWAYS)
-				&& outPartitioning.get(vertexName)
-						.get(outEdgeList.get(vertexName).indexOf(outName)).getStrategy() == PartitioningStrategy.FORWARD
-				&& vertexParallelism.get(vertexName) == vertexParallelism.get(outName) && chaining;
-	}
-
-	private <T> void connect(String headOfChain, Tuple2<String, String> edge) {
-
-		String upStreamVertexName = edge.f0;
-		String downStreamVertexName = edge.f1;
-
-		int outputIndex = outEdgeList.get(upStreamVertexName).indexOf(downStreamVertexName);
-
-		AbstractJobVertex headVertex = streamVertices.get(headOfChain);
-		AbstractJobVertex downStreamVertex = streamVertices.get(downStreamVertexName);
-
-		StreamConfig downStreamConfig = new StreamConfig(downStreamVertex.getConfiguration());
-		StreamConfig upStreamConfig = new StreamConfig(headVertex.getConfiguration());
-
-		List<Integer> outEdgeIndexList = outEdgeIndex.get(upStreamVertexName);
-		int numOfInputs = downStreamConfig.getNumberOfInputs();
-
-		downStreamConfig.setInputIndex(numOfInputs++, outEdgeIndexList.get(outputIndex));
-		downStreamConfig.setNumberOfInputs(numOfInputs);
-
-		StreamPartitioner<?> partitionerObject = outPartitioning.get(upStreamVertexName).get(
-				outputIndex);
-
-		upStreamConfig.setPartitioner(downStreamVertexName, partitionerObject);
-
-		if (partitionerObject.getStrategy() == PartitioningStrategy.FORWARD) {
-			downStreamVertex.connectNewDataSetAsInput(headVertex, DistributionPattern.POINTWISE);
-		} else {
-			downStreamVertex.connectNewDataSetAsInput(headVertex, DistributionPattern.ALL_TO_ALL);
-		}
-
-		if (LOG.isDebugEnabled()) {
-			LOG.debug("CONNECTED: {} - {} -> {}", partitionerObject.getClass().getSimpleName(),
-					headOfChain, downStreamVertexName);
-		}
-
-		upStreamConfig.setOutputNames(downStreamVertexName, outEdgeNames.get(upStreamVertexName)
-				.get(outputIndex));
-		upStreamConfig.setSelectAll(downStreamVertexName, outEdgeSelectAll.get(upStreamVertexName)
-				.get(outputIndex));
-	}
-
-	/**
-	 * Sets the number of parallel instances created for the given vertex.
-	 * 
-	 * @param vertexName
-	 *            Name of the vertex
-	 * @param parallelism
-	 *            Number of parallel instances created
-	 */
-	public void setParallelism(String vertexName, int parallelism) {
-		vertexParallelism.put(vertexName, parallelism);
-	}
-
-	/**
-	 * Sets the input format for the given vertex.
-	 * 
-	 * @param vertexName
-	 *            Name of the vertex
-	 * @param inputFormat
-	 *            input format of the file source associated with the given
-	 *            vertex
-	 */
-	public void setInputFormat(String vertexName, InputFormat<String, ?> inputFormat) {
-		inputFormatList.put(vertexName, inputFormat);
-	}
-
-	public void setBufferTimeout(String vertexName, long bufferTimeout) {
-		this.bufferTimeout.put(vertexName, bufferTimeout);
-	}
-
-	public void addOperatorState(String veretxName, String stateName, OperatorState<?> state) {
-		Map<String, OperatorState<?>> states = operatorStates.get(veretxName);
-		if (states == null) {
-			states = new HashMap<String, OperatorState<?>>();
-			states.put(stateName, state);
-		} else {
-			if (states.containsKey(stateName)) {
-				throw new RuntimeException("State has already been registered with this name: "
-						+ stateName);
-			} else {
-				states.put(stateName, state);
-			}
-		}
-		operatorStates.put(veretxName, states);
-	}
-
-	/**
-	 * Connects two vertices in the JobGraph using the selected partitioner
-	 * settings
-	 * 
-	 * @param upStreamVertexName
-	 *            Name of the upstream(output) vertex
-	 * @param downStreamVertexName
-	 *            Name of the downstream(input) vertex
-	 * @param partitionerObject
-	 *            Partitioner object
-	 * @param typeNumber
-	 *            Number of the type (used at co-functions)
-	 * @param outputNames
-	 *            User defined names of the out edge
-	 */
-	public void setEdge(String upStreamVertexName, String downStreamVertexName,
-			StreamPartitioner<?> partitionerObject, int typeNumber, List<String> outputNames,
-			boolean selectAll) {
-		outEdgeList.get(upStreamVertexName).add(downStreamVertexName);
-		outEdgeIndex.get(upStreamVertexName).add(typeNumber);
-		inEdgeList.get(downStreamVertexName).add(upStreamVertexName);
-		outPartitioning.get(upStreamVertexName).add(partitionerObject);
-		outEdgeNames.get(upStreamVertexName).add(outputNames);
-		outEdgeSelectAll.get(upStreamVertexName).add(selectAll);
-	}
-
-	/**
-	 * Sets the parallelism and buffertimeout of the iteration head of the given
-	 * iteration id to the parallelism given.
-	 * 
-	 * @param iterationID
-	 *            ID of the iteration
-	 * @param iterationTail
-	 *            ID of the iteration tail
-	 */
-	public void setIterationSourceSettings(String iterationID, String iterationTail) {
-		setParallelism(iterationIDtoHeadName.get(iterationID), vertexParallelism.get(iterationTail));
-		setBufferTimeout(iterationIDtoHeadName.get(iterationID), bufferTimeout.get(iterationTail));
-	}
-
-	/**
-	 * Sets a user defined {@link OutputSelector} for the given vertex. Used for
-	 * directed emits.
-	 * 
-	 * @param vertexName
-	 *            Name of the vertex for which the output selector will be set
-	 * @param serializedOutputSelector
-	 *            Byte array representing the serialized output selector.
-	 */
-	public <T> void setOutputSelector(String vertexName, byte[] serializedOutputSelector) {
-		outputSelectors.put(vertexName, serializedOutputSelector);
-
-		if (LOG.isDebugEnabled()) {
-			LOG.debug("Outputselector set for {}", vertexName);
-		}
-
-	}
-
-	public <IN, OUT> void setInvokable(String id, StreamInvokable<IN, OUT> invokableObject) {
-		invokableObjects.put(id, invokableObject);
-	}
-
-	public StreamInvokable<?, ?> getInvokable(String id) {
-		return invokableObjects.get(id);
-	}
-
-	public <OUT> void setOutType(String id, TypeInformation<OUT> outType) {
-		StreamRecordSerializer<OUT> serializer = new StreamRecordSerializer<OUT>(outType);
-		typeSerializersOut1.put(id, serializer);
-	}
-
-	/**
-	 * Sets TypeSerializerWrapper from one vertex to another, used with some
-	 * sinks.
-	 * 
-	 * @param from
-	 *            from
-	 * @param to
-	 *            to
-	 */
-	public void setSerializersFrom(String from, String to) {
-		operatorNames.put(to, operatorNames.get(from));
-
-		typeSerializersIn1.put(to, typeSerializersOut1.get(from));
-		typeSerializersIn2.put(to, typeSerializersOut2.get(from));
-		typeSerializersOut1.put(to, typeSerializersOut1.get(from));
-		typeSerializersOut2.put(to, typeSerializersOut2.get(from));
-	}
-
-	/**
-	 * Sets slot sharing for the vertices.
-	 */
-	private void setSlotSharing() {
-		SlotSharingGroup shareGroup = new SlotSharingGroup();
-
-		for (AbstractJobVertex vertex : streamVertices.values()) {
-			vertex.setSlotSharingGroup(shareGroup);
-		}
-
-		for (Integer iterID : new HashSet<Integer>(iterationIds.values())) {
-			CoLocationGroup ccg = new CoLocationGroup();
-			AbstractJobVertex tail = streamVertices.get(iterationIDtoTailName.get(iterID));
-			AbstractJobVertex head = streamVertices.get(iterationIDtoHeadName.get(iterID));
-
-			ccg.addVertex(head);
-			ccg.addVertex(tail);
-		}
-	}
-
-	/**
-	 * Gets the assembled {@link JobGraph} and adds a default name for it.
-	 */
-	public JobGraph getJobGraph() {
-		return getJobGraph(DEAFULT_JOB_NAME);
-	}
-
-	/**
-	 * Gets the assembled {@link JobGraph} and adds a user specified name for
-	 * it.
-	 * 
-	 * @param jobGraphName
-	 *            name of the jobGraph
-	 */
-	public JobGraph getJobGraph(String jobGraphName) {
-		jobGraph = new JobGraph(jobGraphName);
-		buildJobGraph();
-		return jobGraph;
-	}
-
-	/**
-	 * Builds the {@link JobGraph} from the vertices with the edges and settings
-	 * provided.
-	 */
-	private void buildJobGraph() {
-
-		for (String sourceName : sources) {
-			createChain(sourceName, sourceName);
-		}
-
-		setSlotSharing();
-	}
-
-	public void setChaining(boolean chaining) {
-		this.chaining = chaining;
-	}
-
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
index 6fffaa6..213a892 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
@@ -18,7 +18,7 @@
 package org.apache.flink.streaming.api;
 
 import java.io.Serializable;
-import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -62,7 +62,7 @@ public class StreamConfig implements Serializable {
 	private static final String TYPE_SERIALIZER_OUT_2 = "typeSerializer_out_2";
 	private static final String ITERATON_WAIT = "iterationWait";
 	private static final String OUTPUTS = "outVertexNames";
-	private static final String RW_ORDER = "rwOrder";
+	private static final String EDGES_IN_ORDER = "rwOrder";
 
 	// DEFAULT VALUES
 
@@ -186,10 +186,10 @@ public class StreamConfig implements Serializable {
 		return config.getBoolean(DIRECTED_EMIT, false);
 	}
 
-	public void setOutputSelector(byte[] outputSelector) {
+	public void setOutputSelector(OutputSelector<?> outputSelector) {
 		if (outputSelector != null) {
 			setDirectedEmit(true);
-			config.setBytes(OUTPUT_SELECTOR, outputSelector);
+			config.setBytes(OUTPUT_SELECTOR, SerializationUtils.serialize(outputSelector));
 		}
 	}
 
@@ -293,20 +293,16 @@ public class StreamConfig implements Serializable {
 		}
 	}
 
-	public void setRecordWriterOrder(List<Tuple2<String, String>> outEdgeList) {
+	public void setOutEdgesInOrder(List<Tuple2<String, String>> outEdgeList) {
 
-		List<String> outVertices = new ArrayList<String>();
-		for (Tuple2<String, String> edge : outEdgeList) {
-			outVertices.add(edge.f1);
-		}
-
-		config.setBytes(RW_ORDER, SerializationUtils.serialize((Serializable) outVertices));
+		config.setBytes(EDGES_IN_ORDER, SerializationUtils.serialize((Serializable) outEdgeList));
 	}
 
 	@SuppressWarnings("unchecked")
-	public List<String> getRecordWriterOrder(ClassLoader cl) {
+	public List<Tuple2<String, String>> getOutEdgesInOrder(ClassLoader cl) {
 		try {
-			return (List<String>) InstantiationUtil.readObjectFromConfig(this.config, RW_ORDER, cl);
+			return (List<Tuple2<String, String>>) InstantiationUtil.readObjectFromConfig(
+					this.config, EDGES_IN_ORDER, cl);
 		} catch (Exception e) {
 			throw new RuntimeException("Could not instantiate outputs.");
 		}
@@ -358,8 +354,10 @@ public class StreamConfig implements Serializable {
 	public Map<String, StreamConfig> getTransitiveChainedTaskConfigs(ClassLoader cl) {
 		try {
 
-			return (Map<String, StreamConfig>) InstantiationUtil.readObjectFromConfig(this.config,
-					CHAINED_TASK_CONFIG, cl);
+			Map<String, StreamConfig> confs = (Map<String, StreamConfig>) InstantiationUtil
+					.readObjectFromConfig(this.config, CHAINED_TASK_CONFIG, cl);
+
+			return confs == null ? new HashMap<String, StreamConfig>() : confs;
 		} catch (Exception e) {
 			throw new RuntimeException("Could not instantiate configuration.");
 		}

http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamGraph.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamGraph.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamGraph.java
new file mode 100644
index 0000000..3dd1bdb
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamGraph.java
@@ -0,0 +1,545 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.flink.api.common.io.InputFormat;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
+import org.apache.flink.streaming.api.collector.OutputSelector;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.invokable.operator.co.CoInvokable;
+import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
+import org.apache.flink.streaming.api.streamvertex.CoStreamVertex;
+import org.apache.flink.streaming.api.streamvertex.StreamIterationHead;
+import org.apache.flink.streaming.api.streamvertex.StreamIterationTail;
+import org.apache.flink.streaming.api.streamvertex.StreamVertex;
+import org.apache.flink.streaming.partitioner.StreamPartitioner;
+import org.apache.flink.streaming.state.OperatorState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Object for building Apache Flink stream processing graphs
+ */
+public class StreamGraph {
+
+	private static final Logger LOG = LoggerFactory.getLogger(StreamGraph.class);
+	private final static String DEAFULT_JOB_NAME = "Flink Streaming Job";
+
+	protected boolean chaining = true;
+
+	// Graph attributes
+	private Map<String, Integer> operatorParallelisms;
+	private Map<String, Long> bufferTimeouts;
+	private Map<String, List<String>> outEdgeLists;
+	private Map<String, List<Integer>> outEdgeTypes;
+	private Map<String, List<List<String>>> selectedNames;
+	private Map<String, List<Boolean>> outEdgeSelectAlls;
+	private Map<String, List<String>> inEdgeLists;
+	private Map<String, List<StreamPartitioner<?>>> outputPartitioners;
+	private Map<String, String> operatorNames;
+	private Map<String, StreamInvokable<?, ?>> invokableObjects;
+	private Map<String, StreamRecordSerializer<?>> typeSerializersIn1;
+	private Map<String, StreamRecordSerializer<?>> typeSerializersIn2;
+	private Map<String, StreamRecordSerializer<?>> typeSerializersOut1;
+	private Map<String, StreamRecordSerializer<?>> typeSerializersOut2;
+	private Map<String, OutputSelector<?>> outputSelectors;
+	private Map<String, Class<? extends AbstractInvokable>> jobVertexClasses;
+	private Map<String, Integer> iterationIds;
+	private Map<Integer, String> iterationIDtoHeadName;
+	private Map<Integer, String> iterationIDtoTailName;
+	private Map<String, Integer> iterationTailCount;
+	private Map<String, Long> iterationTimeouts;
+	private Map<String, Map<String, OperatorState<?>>> operatorStates;
+	private Map<String, InputFormat<String, ?>> inputFormatLists;
+
+	private Set<String> sources;
+
+	public StreamGraph() {
+
+		initGraph();
+
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("StreamGraph created");
+		}
+	}
+
+	public void initGraph() {
+		operatorParallelisms = new HashMap<String, Integer>();
+		bufferTimeouts = new HashMap<String, Long>();
+		outEdgeLists = new HashMap<String, List<String>>();
+		outEdgeTypes = new HashMap<String, List<Integer>>();
+		selectedNames = new HashMap<String, List<List<String>>>();
+		outEdgeSelectAlls = new HashMap<String, List<Boolean>>();
+		inEdgeLists = new HashMap<String, List<String>>();
+		outputPartitioners = new HashMap<String, List<StreamPartitioner<?>>>();
+		operatorNames = new HashMap<String, String>();
+		invokableObjects = new HashMap<String, StreamInvokable<?, ?>>();
+		typeSerializersIn1 = new HashMap<String, StreamRecordSerializer<?>>();
+		typeSerializersIn2 = new HashMap<String, StreamRecordSerializer<?>>();
+		typeSerializersOut1 = new HashMap<String, StreamRecordSerializer<?>>();
+		typeSerializersOut2 = new HashMap<String, StreamRecordSerializer<?>>();
+		outputSelectors = new HashMap<String, OutputSelector<?>>();
+		jobVertexClasses = new HashMap<String, Class<? extends AbstractInvokable>>();
+		iterationIds = new HashMap<String, Integer>();
+		iterationIDtoHeadName = new HashMap<Integer, String>();
+		iterationIDtoTailName = new HashMap<Integer, String>();
+		iterationTailCount = new HashMap<String, Integer>();
+		iterationTimeouts = new HashMap<String, Long>();
+		operatorStates = new HashMap<String, Map<String, OperatorState<?>>>();
+		inputFormatLists = new HashMap<String, InputFormat<String, ?>>();
+		sources = new HashSet<String>();
+	}
+
+	/**
+	 * Adds a vertex to the streaming graph with the given parameters
+	 * 
+	 * @param vertexName
+	 *            Name of the vertex
+	 * @param invokableObject
+	 *            User defined operator
+	 * @param inTypeInfo
+	 *            Input type for serialization
+	 * @param outTypeInfo
+	 *            Output type for serialization
+	 * @param operatorName
+	 *            Operator type
+	 * @param parallelism
+	 *            Number of parallel instances created
+	 */
+	public <IN, OUT> void addStreamVertex(String vertexName,
+			StreamInvokable<IN, OUT> invokableObject, TypeInformation<IN> inTypeInfo,
+			TypeInformation<OUT> outTypeInfo, String operatorName, int parallelism) {
+
+		addVertex(vertexName, StreamVertex.class, invokableObject, operatorName, parallelism);
+
+		StreamRecordSerializer<IN> inSerializer = inTypeInfo != null ? new StreamRecordSerializer<IN>(
+				inTypeInfo) : null;
+		StreamRecordSerializer<OUT> outSerializer = outTypeInfo != null ? new StreamRecordSerializer<OUT>(
+				outTypeInfo) : null;
+
+		addTypeSerializers(vertexName, inSerializer, null, outSerializer, null);
+
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("Vertex: {}", vertexName);
+		}
+	}
+
+	public <IN, OUT> void addSourceVertex(String vertexName,
+			StreamInvokable<IN, OUT> invokableObject, TypeInformation<IN> inTypeInfo,
+			TypeInformation<OUT> outTypeInfo, String operatorName, int parallelism) {
+		addStreamVertex(vertexName, invokableObject, inTypeInfo, outTypeInfo, operatorName,
+				parallelism);
+		sources.add(vertexName);
+	}
+
+	/**
+	 * Adds a vertex for the iteration head to the {@link JobGraph}. The
+	 * iterated values will be fed from this vertex back to the graph.
+	 * 
+	 * @param vertexName
+	 *            Name of the vertex
+	 * @param iterationHead
+	 *            Id of the iteration head
+	 * @param iterationID
+	 *            ID of iteration for multiple iterations
+	 * @param parallelism
+	 *            Number of parallel instances created
+	 * @param waitTime
+	 *            Max wait time for next record
+	 */
+	public void addIterationHead(String vertexName, String iterationHead, Integer iterationID,
+			int parallelism, long waitTime) {
+
+		addVertex(vertexName, StreamIterationHead.class, null, null, parallelism);
+
+		chaining = false;
+
+		iterationIds.put(vertexName, iterationID);
+		iterationIDtoHeadName.put(iterationID, vertexName);
+
+		setSerializersFrom(iterationHead, vertexName);
+
+		setEdge(vertexName, iterationHead,
+				outputPartitioners.get(inEdgeLists.get(iterationHead).get(0)).get(0), 0,
+				new ArrayList<String>(), false);
+
+		iterationTimeouts.put(iterationIDtoHeadName.get(iterationID), waitTime);
+
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("ITERATION SOURCE: {}", vertexName);
+		}
+
+		sources.add(vertexName);
+	}
+
+	/**
+	 * Adds a vertex for the iteration tail to the {@link JobGraph}. The values
+	 * intended to be iterated will be sent to this sink from the iteration
+	 * head.
+	 * 
+	 * @param vertexName
+	 *            Name of the vertex
+	 * @param iterationTail
+	 *            Id of the iteration tail
+	 * @param iterationID
+	 *            ID of iteration for mulitple iterations
+	 * @param parallelism
+	 *            Number of parallel instances created
+	 * @param waitTime
+	 *            Max waiting time for next record
+	 */
+	public void addIterationTail(String vertexName, String iterationTail, Integer iterationID,
+			int parallelism, long waitTime) {
+
+		if (bufferTimeouts.get(iterationTail) == 0) {
+			throw new RuntimeException("Buffer timeout 0 at iteration tail is not supported.");
+		}
+
+		addVertex(vertexName, StreamIterationTail.class, null, null, parallelism);
+
+		iterationIds.put(vertexName, iterationID);
+		iterationIDtoTailName.put(iterationID, vertexName);
+
+		setSerializersFrom(iterationTail, vertexName);
+		iterationTimeouts.put(iterationIDtoTailName.get(iterationID), waitTime);
+
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("ITERATION SINK: {}", vertexName);
+		}
+
+	}
+
+	public <IN1, IN2, OUT> void addCoTask(String vertexName,
+			CoInvokable<IN1, IN2, OUT> taskInvokableObject, TypeInformation<IN1> in1TypeInfo,
+			TypeInformation<IN2> in2TypeInfo, TypeInformation<OUT> outTypeInfo,
+			String operatorName, int parallelism) {
+
+		addVertex(vertexName, CoStreamVertex.class, taskInvokableObject, operatorName, parallelism);
+
+		addTypeSerializers(vertexName, new StreamRecordSerializer<IN1>(in1TypeInfo),
+				new StreamRecordSerializer<IN2>(in2TypeInfo), new StreamRecordSerializer<OUT>(
+						outTypeInfo), null);
+
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("CO-TASK: {}", vertexName);
+		}
+	}
+
+	/**
+	 * Sets vertex parameters in the JobGraph
+	 * 
+	 * @param vertexName
+	 *            Name of the vertex
+	 * @param vertexClass
+	 *            The class of the vertex
+	 * @param invokableObjectject
+	 *            The user defined invokable object
+	 * @param operatorName
+	 *            Type of the user defined operator
+	 * @param parallelism
+	 *            Number of parallel instances created
+	 */
+	private void addVertex(String vertexName, Class<? extends AbstractInvokable> vertexClass,
+			StreamInvokable<?, ?> invokableObject, String operatorName, int parallelism) {
+
+		jobVertexClasses.put(vertexName, vertexClass);
+		setParallelism(vertexName, parallelism);
+		invokableObjects.put(vertexName, invokableObject);
+		operatorNames.put(vertexName, operatorName);
+		outEdgeLists.put(vertexName, new ArrayList<String>());
+		outEdgeTypes.put(vertexName, new ArrayList<Integer>());
+		selectedNames.put(vertexName, new ArrayList<List<String>>());
+		outEdgeSelectAlls.put(vertexName, new ArrayList<Boolean>());
+		inEdgeLists.put(vertexName, new ArrayList<String>());
+		outputPartitioners.put(vertexName, new ArrayList<StreamPartitioner<?>>());
+		iterationTailCount.put(vertexName, 0);
+	}
+
+	/**
+	 * Connects two vertices in the JobGraph using the selected partitioner
+	 * settings
+	 * 
+	 * @param upStreamVertexName
+	 *            Name of the upstream(output) vertex
+	 * @param downStreamVertexName
+	 *            Name of the downstream(input) vertex
+	 * @param partitionerObject
+	 *            Partitioner object
+	 * @param typeNumber
+	 *            Number of the type (used at co-functions)
+	 * @param outputNames
+	 *            User defined names of the out edge
+	 */
+	public void setEdge(String upStreamVertexName, String downStreamVertexName,
+			StreamPartitioner<?> partitionerObject, int typeNumber, List<String> outputNames,
+			boolean selectAll) {
+		outEdgeLists.get(upStreamVertexName).add(downStreamVertexName);
+		outEdgeTypes.get(upStreamVertexName).add(typeNumber);
+		inEdgeLists.get(downStreamVertexName).add(upStreamVertexName);
+		outputPartitioners.get(upStreamVertexName).add(partitionerObject);
+		selectedNames.get(upStreamVertexName).add(outputNames);
+		outEdgeSelectAlls.get(upStreamVertexName).add(selectAll);
+	}
+
+	private void addTypeSerializers(String vertexName, StreamRecordSerializer<?> in1,
+			StreamRecordSerializer<?> in2, StreamRecordSerializer<?> out1,
+			StreamRecordSerializer<?> out2) {
+		typeSerializersIn1.put(vertexName, in1);
+		typeSerializersIn2.put(vertexName, in2);
+		typeSerializersOut1.put(vertexName, out1);
+		typeSerializersOut2.put(vertexName, out2);
+	}
+
+	/**
+	 * Sets the number of parallel instances created for the given vertex.
+	 * 
+	 * @param vertexName
+	 *            Name of the vertex
+	 * @param parallelism
+	 *            Number of parallel instances created
+	 */
+	public void setParallelism(String vertexName, int parallelism) {
+		operatorParallelisms.put(vertexName, parallelism);
+	}
+
+	public int getParallelism(String vertexName) {
+		return operatorParallelisms.get(vertexName);
+	}
+
+	/**
+	 * Sets the input format for the given vertex.
+	 * 
+	 * @param vertexName
+	 *            Name of the vertex
+	 * @param inputFormat
+	 *            input format of the file source associated with the given
+	 *            vertex
+	 */
+	public void setInputFormat(String vertexName, InputFormat<String, ?> inputFormat) {
+		inputFormatLists.put(vertexName, inputFormat);
+	}
+
+	public void setBufferTimeout(String vertexName, long bufferTimeout) {
+		this.bufferTimeouts.put(vertexName, bufferTimeout);
+	}
+
+	public long getBufferTimeout(String vertexName) {
+		return this.bufferTimeouts.get(vertexName);
+	}
+
+	public void addOperatorState(String veretxName, String stateName, OperatorState<?> state) {
+		Map<String, OperatorState<?>> states = operatorStates.get(veretxName);
+		if (states == null) {
+			states = new HashMap<String, OperatorState<?>>();
+			states.put(stateName, state);
+		} else {
+			if (states.containsKey(stateName)) {
+				throw new RuntimeException("State has already been registered with this name: "
+						+ stateName);
+			} else {
+				states.put(stateName, state);
+			}
+		}
+		operatorStates.put(veretxName, states);
+	}
+
+	/**
+	 * Sets the parallelism and buffertimeout of the iteration head of the given
+	 * iteration id to the parallelism given.
+	 * 
+	 * @param iterationID
+	 *            ID of the iteration
+	 * @param iterationTail
+	 *            ID of the iteration tail
+	 */
+	public void setIterationSourceSettings(String iterationID, String iterationTail) {
+		setParallelism(iterationIDtoHeadName.get(iterationID),
+				operatorParallelisms.get(iterationTail));
+		setBufferTimeout(iterationIDtoHeadName.get(iterationID), bufferTimeouts.get(iterationTail));
+	}
+
+	/**
+	 * Sets a user defined {@link OutputSelector} for the given operator. Used
+	 * for directed emits.
+	 * 
+	 * @param vertexName
+	 *            Name of the vertex for which the output selector will be set
+	 * @param outputSelector
+	 *            The outputselector object
+	 */
+	public void setOutputSelector(String vertexName, OutputSelector<?> outputSelector) {
+		outputSelectors.put(vertexName, outputSelector);
+
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("Outputselector set for {}", vertexName);
+		}
+
+	}
+
+	public <IN, OUT> void setInvokable(String vertexName, StreamInvokable<IN, OUT> invokableObject) {
+		invokableObjects.put(vertexName, invokableObject);
+	}
+
+	public <OUT> void setOutType(String id, TypeInformation<OUT> outType) {
+		StreamRecordSerializer<OUT> serializer = new StreamRecordSerializer<OUT>(outType);
+		typeSerializersOut1.put(id, serializer);
+	}
+
+	public StreamInvokable<?, ?> getInvokable(String vertexName) {
+		return invokableObjects.get(vertexName);
+	}
+
+	@SuppressWarnings("unchecked")
+	public <OUT> StreamRecordSerializer<OUT> getOutSerializer1(String vertexName) {
+		return (StreamRecordSerializer<OUT>) typeSerializersOut1.get(vertexName);
+	}
+
+	@SuppressWarnings("unchecked")
+	public <OUT> StreamRecordSerializer<OUT> getOutSerializer2(String vertexName) {
+		return (StreamRecordSerializer<OUT>) typeSerializersOut2.get(vertexName);
+	}
+
+	@SuppressWarnings("unchecked")
+	public <IN> StreamRecordSerializer<IN> getInSerializer1(String vertexName) {
+		return (StreamRecordSerializer<IN>) typeSerializersIn1.get(vertexName);
+	}
+
+	@SuppressWarnings("unchecked")
+	public <IN> StreamRecordSerializer<IN> getInSerializer2(String vertexName) {
+		return (StreamRecordSerializer<IN>) typeSerializersIn2.get(vertexName);
+	}
+
+	/**
+	 * Sets TypeSerializerWrapper from one vertex to another, used with some
+	 * sinks.
+	 * 
+	 * @param from
+	 *            from
+	 * @param to
+	 *            to
+	 */
+	public void setSerializersFrom(String from, String to) {
+		operatorNames.put(to, operatorNames.get(from));
+
+		typeSerializersIn1.put(to, typeSerializersOut1.get(from));
+		typeSerializersIn2.put(to, typeSerializersOut2.get(from));
+		typeSerializersOut1.put(to, typeSerializersOut1.get(from));
+		typeSerializersOut2.put(to, typeSerializersOut2.get(from));
+	}
+
+	/**
+	 * Gets the assembled {@link JobGraph} and adds a default name for it.
+	 */
+	public JobGraph getJobGraph() {
+		return getJobGraph(DEAFULT_JOB_NAME);
+	}
+
+	/**
+	 * Gets the assembled {@link JobGraph} and adds a user specified name for
+	 * it.
+	 * 
+	 * @param jobGraphName
+	 *            name of the jobGraph
+	 */
+	public JobGraph getJobGraph(String jobGraphName) {
+
+		StreamingJobGraphGenerator optimizer = new StreamingJobGraphGenerator(this);
+
+		return optimizer.createJobGraph(jobGraphName);
+	}
+
+	public void setChaining(boolean chaining) {
+		this.chaining = chaining;
+	}
+
+	public Collection<String> getSources() {
+		return sources;
+	}
+
+	public List<String> getOutEdges(String vertexName) {
+		return outEdgeLists.get(vertexName);
+	}
+
+	public List<String> getInEdges(String vertexName) {
+		return inEdgeLists.get(vertexName);
+	}
+
+	public List<Integer> getOutEdgeTypes(String vertexName) {
+
+		return outEdgeTypes.get(vertexName);
+	}
+
+	public StreamPartitioner<?> getOutPartitioner(String vertexName, int outputIndex) {
+		return outputPartitioners.get(vertexName).get(outputIndex);
+	}
+
+	public List<String> getSelectedNames(String vertexName, int outputIndex) {
+		return selectedNames.get(vertexName).get(outputIndex);
+	}
+
+	public Boolean isSelectAll(String vertexName, int outputIndex) {
+		return outEdgeSelectAlls.get(vertexName).get(outputIndex);
+	}
+
+	public Collection<Integer> getIterationIDs() {
+		return new HashSet<Integer>(iterationIds.values());
+	}
+
+	public String getIterationTail(int iterID) {
+		return iterationIDtoTailName.get(iterID);
+	}
+
+	public String getIterationHead(int iterID) {
+		return iterationIDtoHeadName.get(iterID);
+	}
+
+	public Class<? extends AbstractInvokable> getJobVertexClass(String vertexName) {
+		return jobVertexClasses.get(vertexName);
+	}
+
+	public InputFormat<String, ?> getInputFormat(String vertexName) {
+		return inputFormatLists.get(vertexName);
+	}
+
+	public OutputSelector<?> getOutputSelector(String vertexName) {
+		return outputSelectors.get(vertexName);
+	}
+
+	public Map<String, OperatorState<?>> getState(String vertexName) {
+		return operatorStates.get(vertexName);
+	}
+
+	public Integer getIterationID(String vertexName) {
+		return iterationIds.get(vertexName);
+	}
+
+	public long getIterationTimeout(String vertexName) {
+		return iterationTimeouts.get(vertexName);
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamingJobGraphGenerator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamingJobGraphGenerator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamingJobGraphGenerator.java
new file mode 100644
index 0000000..6c0cc20
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamingJobGraphGenerator.java
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.jobgraph.AbstractJobVertex;
+import org.apache.flink.runtime.jobgraph.DistributionPattern;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
+import org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup;
+import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.invokable.StreamInvokable.ChainingStrategy;
+import org.apache.flink.streaming.api.streamvertex.StreamIterationHead;
+import org.apache.flink.streaming.api.streamvertex.StreamIterationTail;
+import org.apache.flink.streaming.partitioner.StreamPartitioner;
+import org.apache.flink.streaming.partitioner.StreamPartitioner.PartitioningStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class StreamingJobGraphGenerator {
+
+	private static final Logger LOG = LoggerFactory.getLogger(StreamingJobGraphGenerator.class);
+
+	private StreamGraph streamGraph;
+
+	private Map<String, AbstractJobVertex> streamVertices;
+	private JobGraph jobGraph;
+	private Collection<String> builtNodes;
+
+	private Map<String, Map<String, StreamConfig>> chainedConfigs;
+	private Map<String, StreamConfig> vertexConfigs;
+	private Map<String, String> chainedNames;
+
+	public StreamingJobGraphGenerator(StreamGraph streamGraph) {
+		this.streamGraph = streamGraph;
+	}
+
+	private void init() {
+		this.streamVertices = new HashMap<String, AbstractJobVertex>();
+		this.builtNodes = new HashSet<String>();
+		this.chainedConfigs = new HashMap<String, Map<String, StreamConfig>>();
+		this.vertexConfigs = new HashMap<String, StreamConfig>();
+		this.chainedNames = new HashMap<String, String>();
+	}
+
+	public JobGraph createJobGraph(String jobName) {
+		jobGraph = new JobGraph(jobName);
+		init();
+
+		for (String sourceName : streamGraph.getSources()) {
+			createChain(sourceName, sourceName);
+		}
+
+		setSlotSharing();
+
+		return jobGraph;
+	}
+
+	private List<Tuple2<String, String>> createChain(String startNode, String current) {
+
+		if (!builtNodes.contains(startNode)) {
+
+			List<Tuple2<String, String>> transitiveOutEdges = new ArrayList<Tuple2<String, String>>();
+			List<String> chainableOutputs = new ArrayList<String>();
+			List<String> nonChainableOutputs = new ArrayList<String>();
+
+			for (String outName : streamGraph.getOutEdges(current)) {
+				if (isChainable(current, outName)) {
+					chainableOutputs.add(outName);
+				} else {
+					nonChainableOutputs.add(outName);
+				}
+			}
+
+			for (String chainable : chainableOutputs) {
+				transitiveOutEdges.addAll(createChain(startNode, chainable));
+			}
+
+			for (String nonChainable : nonChainableOutputs) {
+				transitiveOutEdges.add(new Tuple2<String, String>(current, nonChainable));
+				createChain(nonChainable, nonChainable);
+			}
+
+			chainedNames.put(current, createChainedName(current, chainableOutputs));
+
+			StreamConfig config = current.equals(startNode) ? createProcessingVertex(startNode)
+					: new StreamConfig(new Configuration());
+
+			setVertexConfig(current, config, chainableOutputs, nonChainableOutputs);
+
+			if (current.equals(startNode)) {
+
+				config.setChainStart();
+				config.setOutEdgesInOrder(transitiveOutEdges);
+
+				for (Tuple2<String, String> edge : transitiveOutEdges) {
+					connect(startNode, edge);
+				}
+
+				config.setTransitiveChainedTaskConfigs(chainedConfigs.get(startNode));
+
+			} else {
+
+				Map<String, StreamConfig> chainedConfs = chainedConfigs.get(startNode);
+
+				if (chainedConfs == null) {
+					chainedConfigs.put(startNode, new HashMap<String, StreamConfig>());
+				}
+				chainedConfigs.get(startNode).put(current, config);
+			}
+
+			return transitiveOutEdges;
+
+		} else {
+			return new ArrayList<Tuple2<String, String>>();
+		}
+	}
+
+	private String createChainedName(String vertexName, List<String> chainedOutputs) {
+		if (chainedOutputs.size() > 1) {
+			List<String> outputChainedNames = new ArrayList<String>();
+			for (String chainable : chainedOutputs) {
+				outputChainedNames.add(chainedNames.get(chainable));
+			}
+
+			return vertexName + " -> (" + StringUtils.join(outputChainedNames, ", ") + ")";
+		} else if (chainedOutputs.size() == 1) {
+			return vertexName + " -> " + chainedNames.get(chainedOutputs.get(0));
+		} else {
+			return vertexName;
+		}
+
+	}
+
+	private StreamConfig createProcessingVertex(String vertexName) {
+
+		AbstractJobVertex vertex = new AbstractJobVertex(chainedNames.get(vertexName));
+
+		vertex.setInvokableClass(streamGraph.getJobVertexClass(vertexName));
+		vertex.setParallelism(streamGraph.getParallelism(vertexName));
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("Parallelism set: {} for {}", streamGraph.getParallelism(vertexName),
+					vertexName);
+		}
+
+		if (streamGraph.getInputFormat(vertexName) != null) {
+			vertex.setInputSplitSource(streamGraph.getInputFormat(vertexName));
+		}
+
+		streamVertices.put(vertexName, vertex);
+		builtNodes.add(vertexName);
+		jobGraph.addVertex(vertex);
+
+		return new StreamConfig(vertex.getConfiguration());
+	}
+
+	private void setVertexConfig(String vertexName, StreamConfig config,
+			List<String> chainableOutputs, List<String> nonChainableOutputs) {
+
+		config.setVertexName(vertexName);
+		config.setBufferTimeout(streamGraph.getBufferTimeout(vertexName));
+
+		config.setTypeSerializerIn1(streamGraph.getInSerializer1(vertexName));
+		config.setTypeSerializerIn2(streamGraph.getInSerializer2(vertexName));
+		config.setTypeSerializerOut1(streamGraph.getOutSerializer1(vertexName));
+		config.setTypeSerializerOut2(streamGraph.getOutSerializer2(vertexName));
+
+		config.setUserInvokable(streamGraph.getInvokable(vertexName));
+		config.setOutputSelector(streamGraph.getOutputSelector(vertexName));
+		config.setOperatorStates(streamGraph.getState(vertexName));
+
+		config.setNumberOfOutputs(nonChainableOutputs.size());
+		config.setOutputs(nonChainableOutputs);
+		config.setChainedOutputs(chainableOutputs);
+
+		Class<? extends AbstractInvokable> vertexClass = streamGraph.getJobVertexClass(vertexName);
+
+		if (vertexClass.equals(StreamIterationHead.class)
+				|| vertexClass.equals(StreamIterationTail.class)) {
+			config.setIterationId(streamGraph.getIterationID(vertexName));
+			config.setIterationWaitTime(streamGraph.getIterationTimeout(vertexName));
+		}
+
+		vertexConfigs.put(vertexName, config);
+	}
+
+	private <T> void connect(String headOfChain, Tuple2<String, String> edge) {
+
+		String upStreamVertexName = edge.f0;
+		String downStreamVertexName = edge.f1;
+
+		int outputIndex = streamGraph.getOutEdges(upStreamVertexName).indexOf(downStreamVertexName);
+
+		AbstractJobVertex headVertex = streamVertices.get(headOfChain);
+		AbstractJobVertex downStreamVertex = streamVertices.get(downStreamVertexName);
+
+		StreamConfig downStreamConfig = new StreamConfig(downStreamVertex.getConfiguration());
+		StreamConfig upStreamConfig = new StreamConfig(headVertex.getConfiguration());
+
+		List<Integer> outEdgeIndexList = streamGraph.getOutEdgeTypes(upStreamVertexName);
+		int numOfInputs = downStreamConfig.getNumberOfInputs();
+
+		downStreamConfig.setInputIndex(numOfInputs++, outEdgeIndexList.get(outputIndex));
+		downStreamConfig.setNumberOfInputs(numOfInputs);
+
+		StreamPartitioner<?> partitioner = streamGraph.getOutPartitioner(upStreamVertexName,
+				outputIndex);
+
+		upStreamConfig.setPartitioner(downStreamVertexName, partitioner);
+
+		if (partitioner.getStrategy() == PartitioningStrategy.FORWARD) {
+			downStreamVertex.connectNewDataSetAsInput(headVertex, DistributionPattern.POINTWISE);
+		} else {
+			downStreamVertex.connectNewDataSetAsInput(headVertex, DistributionPattern.ALL_TO_ALL);
+		}
+
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("CONNECTED: {} - {} -> {}", partitioner.getClass().getSimpleName(),
+					headOfChain, downStreamVertexName);
+		}
+
+		upStreamConfig.setOutputNames(downStreamVertexName,
+				streamGraph.getSelectedNames(upStreamVertexName, outputIndex));
+		upStreamConfig.setSelectAll(downStreamVertexName,
+				streamGraph.isSelectAll(upStreamVertexName, outputIndex));
+	}
+
+	private boolean isChainable(String vertexName, String outName) {
+
+		StreamInvokable<?, ?> headInvokable = streamGraph.getInvokable(vertexName);
+		StreamInvokable<?, ?> outInvokable = streamGraph.getInvokable(outName);
+
+		return streamGraph.getInEdges(outName).size() == 1
+				&& outInvokable != null
+				&& streamGraph.getOutputSelector(vertexName) == null
+				&& outInvokable.getChainingStrategy() == ChainingStrategy.ALWAYS
+				&& (headInvokable.getChainingStrategy() == ChainingStrategy.HEAD || headInvokable
+						.getChainingStrategy() == ChainingStrategy.ALWAYS)
+				&& streamGraph.getOutPartitioner(vertexName,
+						streamGraph.getOutEdges(vertexName).indexOf(outName)).getStrategy() == PartitioningStrategy.FORWARD
+				&& streamGraph.getParallelism(vertexName) == streamGraph.getParallelism(outName)
+				&& streamGraph.chaining;
+	}
+
+	private void setSlotSharing() {
+		SlotSharingGroup shareGroup = new SlotSharingGroup();
+
+		for (AbstractJobVertex vertex : streamVertices.values()) {
+			vertex.setSlotSharingGroup(shareGroup);
+		}
+
+		for (Integer iterID : streamGraph.getIterationIDs()) {
+			CoLocationGroup ccg = new CoLocationGroup();
+			AbstractJobVertex tail = streamVertices.get(streamGraph.getIterationTail(iterID));
+			AbstractJobVertex head = streamVertices.get(streamGraph.getIterationHead(iterID));
+
+			ccg.addVertex(head);
+			ccg.addVertex(tail);
+		}
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
index 80ea970..db8649b 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
@@ -22,7 +22,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.java.ClosureCleaner;
 import org.apache.flink.api.java.functions.KeySelector;
 import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.apache.flink.streaming.api.JobGraphBuilder;
+import org.apache.flink.streaming.api.StreamGraph;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.streaming.api.function.co.CoFlatMapFunction;
 import org.apache.flink.streaming.api.function.co.CoMapFunction;
@@ -52,7 +52,7 @@ import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
 public class ConnectedDataStream<IN1, IN2> {
 
 	protected StreamExecutionEnvironment environment;
-	protected JobGraphBuilder jobGraphBuilder;
+	protected StreamGraph jobGraphBuilder;
 	protected DataStream<IN1> dataStream1;
 	protected DataStream<IN2> dataStream2;
 
@@ -61,7 +61,7 @@ public class ConnectedDataStream<IN1, IN2> {
 	protected KeySelector<IN2, ?> keySelector2;
 
 	protected ConnectedDataStream(DataStream<IN1> input1, DataStream<IN2> input2) {
-		this.jobGraphBuilder = input1.jobGraphBuilder;
+		this.jobGraphBuilder = input1.streamGraph;
 		this.environment = input1.environment;
 		this.dataStream1 = input1.copy();
 		this.dataStream2 = input2.copy();
@@ -402,7 +402,7 @@ public class ConnectedDataStream<IN1, IN2> {
 		SingleOutputStreamOperator<OUT, ?> returnStream = new SingleOutputStreamOperator(
 				environment, functionName, outTypeInfo, functionInvokable);
 
-		dataStream1.jobGraphBuilder.addCoTask(returnStream.getId(), functionInvokable,
+		dataStream1.streamGraph.addCoTask(returnStream.getId(), functionInvokable,
 				getInputType1(), getInputType2(), outTypeInfo, functionName,
 				environment.getDegreeOfParallelism());
 

http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index f68ab68..8e7d823 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -43,7 +43,7 @@ import org.apache.flink.api.java.typeutils.TupleTypeInfo;
 import org.apache.flink.api.java.typeutils.TypeExtractor;
 import org.apache.flink.core.fs.FileSystem.WriteMode;
 import org.apache.flink.core.fs.Path;
-import org.apache.flink.streaming.api.JobGraphBuilder;
+import org.apache.flink.streaming.api.StreamGraph;
 import org.apache.flink.streaming.api.datastream.temporaloperator.StreamCrossOperator;
 import org.apache.flink.streaming.api.datastream.temporaloperator.StreamJoinOperator;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
@@ -102,7 +102,7 @@ public class DataStream<OUT> {
 	protected TypeInformation typeInfo;
 	protected List<DataStream<OUT>> mergedStreams;
 
-	protected final JobGraphBuilder jobGraphBuilder;
+	protected final StreamGraph streamGraph;
 
 	/**
 	 * Create a new {@link DataStream} in the given execution environment with
@@ -125,7 +125,7 @@ public class DataStream<OUT> {
 		this.id = operatorType + "-" + counter.toString();
 		this.environment = environment;
 		this.degreeOfParallelism = environment.getDegreeOfParallelism();
-		this.jobGraphBuilder = environment.getJobGraphBuilder();
+		this.streamGraph = environment.getStreamGraph();
 		this.userDefinedNames = new ArrayList<String>();
 		this.selectAll = true;
 		this.partitioner = new DistributePartitioner<OUT>(true);
@@ -147,7 +147,7 @@ public class DataStream<OUT> {
 		this.userDefinedNames = new ArrayList<String>(dataStream.userDefinedNames);
 		this.selectAll = dataStream.selectAll;
 		this.partitioner = dataStream.partitioner;
-		this.jobGraphBuilder = dataStream.jobGraphBuilder;
+		this.streamGraph = dataStream.streamGraph;
 		this.typeInfo = dataStream.typeInfo;
 		this.mergedStreams = new ArrayList<DataStream<OUT>>();
 		this.mergedStreams.add(this);
@@ -189,7 +189,7 @@ public class DataStream<OUT> {
 
 	@SuppressWarnings("unchecked")
 	public <R> DataStream<R> setType(TypeInformation<R> outType) {
-		jobGraphBuilder.setOutType(id, outType);
+		streamGraph.setOutType(id, outType);
 		typeInfo = outType;
 		return (DataStream<R>) this;
 	}
@@ -1085,7 +1085,7 @@ public class DataStream<OUT> {
 		DataStream<R> returnStream = new DataStreamSource<R>(environment, "iterationSource", null,
 				null, true);
 
-		jobGraphBuilder.addIterationHead(returnStream.getId(), this.getId(), iterationID,
+		streamGraph.addIterationHead(returnStream.getId(), this.getId(), iterationID,
 				degreeOfParallelism, waitTime);
 
 		return this.copy();
@@ -1112,7 +1112,7 @@ public class DataStream<OUT> {
 		SingleOutputStreamOperator<R, ?> returnStream = new SingleOutputStreamOperator(environment,
 				operatorName, outTypeInfo, invokable);
 
-		jobGraphBuilder.addStreamVertex(returnStream.getId(), invokable, getType(), outTypeInfo,
+		streamGraph.addStreamVertex(returnStream.getId(), invokable, getType(), outTypeInfo,
 				operatorName, degreeOfParallelism);
 
 		connectGraph(inputStream, returnStream.getId(), 0);
@@ -1157,7 +1157,7 @@ public class DataStream<OUT> {
 	 */
 	protected <X> void connectGraph(DataStream<X> inputStream, String outputID, int typeNumber) {
 		for (DataStream<X> stream : inputStream.mergedStreams) {
-			jobGraphBuilder.setEdge(stream.getId(), outputID, stream.partitioner, typeNumber,
+			streamGraph.setEdge(stream.getId(), outputID, stream.partitioner, typeNumber,
 					inputStream.userDefinedNames, inputStream.selectAll);
 		}
 
@@ -1179,7 +1179,7 @@ public class DataStream<OUT> {
 		DataStreamSink<OUT> returnStream = new DataStreamSink<OUT>(environment, "sink", getType(),
 				sinkInvokable);
 
-		jobGraphBuilder.addStreamVertex(returnStream.getId(), sinkInvokable, getType(), null,
+		streamGraph.addStreamVertex(returnStream.getId(), sinkInvokable, getType(), null,
 				"sink", degreeOfParallelism);
 
 		this.connectGraph(this.copy(), returnStream.getId(), 0);

http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
index c4cd1e1..6f66b2c 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
@@ -64,10 +64,10 @@ public class IterativeDataStream<IN> extends
 		DataStream<IN> iterationSink = new DataStreamSink<IN>(environment, "iterationSink", null,
 				null);
 
-		jobGraphBuilder.addIterationTail(iterationSink.getId(), iterationTail.getId(), iterationID,
+		streamGraph.addIterationTail(iterationSink.getId(), iterationTail.getId(), iterationID,
 				iterationTail.getParallelism(), waitTime);
 
-		jobGraphBuilder.setIterationSourceSettings(iterationID.toString(), iterationTail.getId());
+		streamGraph.setIterationSourceSettings(iterationID.toString(), iterationTail.getId());
 		connectGraph(iterationTail.forward(), iterationSink.getId(), 0);
 		return iterationTail;
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
index b8ca42c..dbfbc48 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
@@ -20,8 +20,6 @@ package org.apache.flink.streaming.api.datastream;
 import java.util.Map;
 import java.util.Map.Entry;
 
-import org.apache.commons.lang3.SerializationException;
-import org.apache.commons.lang3.SerializationUtils;
 import org.apache.flink.api.common.functions.RichFunction;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.streaming.api.collector.OutputSelector;
@@ -65,7 +63,7 @@ public class SingleOutputStreamOperator<OUT, O extends SingleOutputStreamOperato
 
 	@SuppressWarnings("unchecked")
 	public <R> SingleOutputStreamOperator<R, ?> setType(TypeInformation<R> outType) {
-		jobGraphBuilder.setOutType(id, outType);
+		streamGraph.setOutType(id, outType);
 		typeInfo = outType;
 		return (SingleOutputStreamOperator<R, ?>) this;
 	}
@@ -84,7 +82,7 @@ public class SingleOutputStreamOperator<OUT, O extends SingleOutputStreamOperato
 		}
 		this.degreeOfParallelism = dop;
 
-		jobGraphBuilder.setParallelism(id, degreeOfParallelism);
+		streamGraph.setParallelism(id, degreeOfParallelism);
 
 		return this;
 	}
@@ -98,7 +96,7 @@ public class SingleOutputStreamOperator<OUT, O extends SingleOutputStreamOperato
 	 * @return The operator with buffer timeout set.
 	 */
 	public SingleOutputStreamOperator<OUT, O> setBufferTimeout(long timeoutMillis) {
-		jobGraphBuilder.setBufferTimeout(id, timeoutMillis);
+		streamGraph.setBufferTimeout(id, timeoutMillis);
 		return this;
 	}
 
@@ -115,13 +113,7 @@ public class SingleOutputStreamOperator<OUT, O extends SingleOutputStreamOperato
 	public SplitDataStream<OUT> split(OutputSelector<OUT> outputSelector) {
 		if (!isSplit) {
 			this.isSplit = true;
-			try {
-				jobGraphBuilder.setOutputSelector(id,
-						SerializationUtils.serialize(clean(outputSelector)));
-
-			} catch (SerializationException e) {
-				throw new RuntimeException("Cannot serialize OutputSelector");
-			}
+			streamGraph.setOutputSelector(id, clean(outputSelector));
 
 			return new SplitDataStream<OUT>(this);
 		} else {
@@ -144,7 +136,7 @@ public class SingleOutputStreamOperator<OUT, O extends SingleOutputStreamOperato
 	 * @return The data stream with state registered.
 	 */
 	protected SingleOutputStreamOperator<OUT, O> registerState(String name, OperatorState<?> state) {
-		jobGraphBuilder.addOperatorState(getId(), name, state);
+		streamGraph.addOperatorState(getId(), name, state);
 		return this;
 	}
 
@@ -161,7 +153,7 @@ public class SingleOutputStreamOperator<OUT, O extends SingleOutputStreamOperato
 	 */
 	protected SingleOutputStreamOperator<OUT, O> registerState(Map<String, OperatorState<?>> states) {
 		for (Entry<String, OperatorState<?>> entry : states.entrySet()) {
-			jobGraphBuilder.addOperatorState(getId(), entry.getKey(), entry.getValue());
+			streamGraph.addOperatorState(getId(), entry.getKey(), entry.getValue());
 		}
 
 		return this;

http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamCrossOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamCrossOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamCrossOperator.java
index 8422400..03160c2 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamCrossOperator.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamCrossOperator.java
@@ -67,7 +67,7 @@ public class StreamCrossOperator<I1, I2> extends
 
 		@SuppressWarnings("unchecked")
 		public CrossWindow<I1, I2> every(long length) {
-			((CoWindowInvokable<I1, I2, ?>) jobGraphBuilder.getInvokable(id)).setSlideSize(length);
+			((CoWindowInvokable<I1, I2, ?>) streamGraph.getInvokable(id)).setSlideSize(length);
 			return this;
 		}
 
@@ -90,7 +90,7 @@ public class StreamCrossOperator<I1, I2> extends
 					new CrossWindowFunction<I1, I2, R>(clean(function)), op.windowSize,
 					op.slideInterval, op.timeStamp1, op.timeStamp2);
 
-			jobGraphBuilder.setInvokable(id, invokable);
+			streamGraph.setInvokable(id, invokable);
 
 			return setType(outTypeInfo);
 

http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamJoinOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamJoinOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamJoinOperator.java
index 626b9f1..d2b2032 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamJoinOperator.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamJoinOperator.java
@@ -245,7 +245,7 @@ public class StreamJoinOperator<I1, I2> extends
 					getJoinWindowFunction(joinFunction, predicate), predicate.op.windowSize,
 					predicate.op.slideInterval, predicate.op.timeStamp1, predicate.op.timeStamp2);
 
-			jobGraphBuilder.setInvokable(id, invokable);
+			streamGraph.setInvokable(id, invokable);
 
 			return setType(outType);
 		}

http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java
index 3e44012..4824fca 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java
@@ -29,7 +29,7 @@ public class LocalStreamEnvironment extends StreamExecutionEnvironment {
 	 */
 	@Override
 	public void execute() throws Exception {
-		ClusterUtil.runOnMiniCluster(this.jobGraphBuilder.getJobGraph(), getDegreeOfParallelism());
+		ClusterUtil.runOnMiniCluster(this.streamGraph.getJobGraph(), getDegreeOfParallelism());
 	}
 
 	/**
@@ -41,7 +41,7 @@ public class LocalStreamEnvironment extends StreamExecutionEnvironment {
 	 */
 	@Override
 	public void execute(String jobName) throws Exception {
-		ClusterUtil.runOnMiniCluster(this.jobGraphBuilder.getJobGraph(jobName),
+		ClusterUtil.runOnMiniCluster(this.streamGraph.getJobGraph(jobName),
 				getDegreeOfParallelism());
 	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/RemoteStreamEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/RemoteStreamEnvironment.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/RemoteStreamEnvironment.java
index d833c8e..2eb05ad 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/RemoteStreamEnvironment.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/RemoteStreamEnvironment.java
@@ -81,14 +81,14 @@ public class RemoteStreamEnvironment extends StreamExecutionEnvironment {
 	@Override
 	public void execute() {
 
-		JobGraph jobGraph = jobGraphBuilder.getJobGraph();
+		JobGraph jobGraph = streamGraph.getJobGraph();
 		executeRemotely(jobGraph);
 	}
 
 	@Override
 	public void execute(String jobName) {
 
-		JobGraph jobGraph = jobGraphBuilder.getJobGraph(jobName);
+		JobGraph jobGraph = streamGraph.getJobGraph(jobName);
 		executeRemotely(jobGraph);
 	}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamContextEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamContextEnvironment.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamContextEnvironment.java
index c157435..a9a5bd3 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamContextEnvironment.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamContextEnvironment.java
@@ -54,9 +54,9 @@ public class StreamContextEnvironment extends StreamExecutionEnvironment {
 
 		JobGraph jobGraph;
 		if (jobName == null) {
-			jobGraph = this.jobGraphBuilder.getJobGraph();
+			jobGraph = this.streamGraph.getJobGraph();
 		} else {
-			jobGraph = this.jobGraphBuilder.getJobGraph(jobName);
+			jobGraph = this.streamGraph.getJobGraph(jobName);
 		}
 
 		for (File file : jars) {

http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
index 4194864..51dc0ae 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
@@ -33,7 +33,7 @@ import org.apache.flink.api.java.typeutils.TypeExtractor;
 import org.apache.flink.client.program.Client;
 import org.apache.flink.client.program.ContextEnvironment;
 import org.apache.flink.core.fs.Path;
-import org.apache.flink.streaming.api.JobGraphBuilder;
+import org.apache.flink.streaming.api.StreamGraph;
 import org.apache.flink.streaming.api.datastream.DataStream;
 import org.apache.flink.streaming.api.datastream.DataStreamSource;
 import org.apache.flink.streaming.api.function.source.FileSourceFunction;
@@ -61,7 +61,7 @@ public abstract class StreamExecutionEnvironment {
 
 	private ExecutionConfig config = new ExecutionConfig();
 
-	protected JobGraphBuilder jobGraphBuilder;
+	protected StreamGraph streamGraph;
 
 	// --------------------------------------------------------------------------------------------
 	// Constructor and Properties
@@ -71,7 +71,7 @@ public abstract class StreamExecutionEnvironment {
 	 * Constructor for creating StreamExecutionEnvironment
 	 */
 	protected StreamExecutionEnvironment() {
-		jobGraphBuilder = new JobGraphBuilder();
+		streamGraph = new StreamGraph();
 	}
 
 	/**
@@ -352,7 +352,7 @@ public abstract class StreamExecutionEnvironment {
 			TypeInformation<String> typeInfo) {
 		FileSourceFunction function = new FileSourceFunction(inputFormat, typeInfo);
 		DataStreamSource<String> returnStream = addSource(function, null, "fileSource");
-		jobGraphBuilder.setInputFormat(returnStream.getId(), inputFormat);
+		streamGraph.setInputFormat(returnStream.getId(), inputFormat);
 		return returnStream;
 	}
 
@@ -437,7 +437,7 @@ public abstract class StreamExecutionEnvironment {
 		DataStreamSource<OUT> returnStream = new DataStreamSource<OUT>(this, sourceName,
 				outTypeInfo, sourceInvokable, isParallel);
 
-		jobGraphBuilder.addSourceVertex(returnStream.getId(), sourceInvokable, null, outTypeInfo,
+		streamGraph.addSourceVertex(returnStream.getId(), sourceInvokable, null, outTypeInfo,
 				sourceName, dop);
 
 		return returnStream;
@@ -584,12 +584,12 @@ public abstract class StreamExecutionEnvironment {
 	public abstract void execute(String jobName) throws Exception;
 
 	/**
-	 * Getter of the {@link JobGraphBuilder} of the streaming job.
+	 * Getter of the {@link StreamGraph} of the streaming job.
 	 * 
-	 * @return jobGraphBuilder
+	 * @return The streamgraph representing the transformations
 	 */
-	public JobGraphBuilder getJobGraphBuilder() {
-		return jobGraphBuilder;
+	public StreamGraph getStreamGraph() {
+		return streamGraph;
 	}
 
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java
index 01c0545..c2177fa 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java
@@ -34,7 +34,6 @@ public class GroupedReduceInvokable<IN> extends StreamReduceInvokable<IN> {
 		super(reducer);
 		this.keySelector = keySelector;
 		values = new HashMap<Object, IN>();
-		setChainingStrategy(ChainingStrategy.NEVER);
 	}
 
 	@Override

http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
index bbd7b0c..a46fa96 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
@@ -67,7 +67,7 @@ import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
  * method. Fake elements created on prenotification will be forwarded to all
  * groups. The {@link ActiveTriggerCallback} is also implemented in a way, that
  * it forwards/distributed calls all groups.
- *
+ * 
  * @param <IN>
  *            The type of input elements handled by this operator invokable.
  */
@@ -154,7 +154,6 @@ public class GroupedWindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
 			LinkedList<EvictionPolicy<IN>> centralEvictionPolicies) {
 
 		super(userFunction);
-		setChainingStrategy(ChainingStrategy.NEVER);
 
 		this.keySelector = keySelector;
 


[5/8] flink git commit: [streaming] JobGraphbuilder separated to StreamGraph and StreamingJobGraphGenerator

Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
index de9c664..ea891c9 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
@@ -61,8 +61,7 @@ public abstract class WindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT>
 	public WindowInvokable(Function userFunction, LinkedList<TriggerPolicy<IN>> triggerPolicies,
 			LinkedList<EvictionPolicy<IN>> evictionPolicies) {
 		super(userFunction);
-		setChainingStrategy(ChainingStrategy.NEVER);
-		
+
 		this.triggerPolicies = triggerPolicies;
 		this.evictionPolicies = evictionPolicies;
 

http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
index 99f826d..135f742 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
@@ -24,6 +24,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
 import org.apache.flink.runtime.plugable.SerializationDelegate;
 import org.apache.flink.streaming.api.StreamConfig;
@@ -54,7 +55,7 @@ public class OutputHandler<OUT> {
 
 	private Map<String, StreamOutput<?>> outputMap;
 	private Map<String, StreamConfig> chainedConfigs;
-	private List<String> recordWriterOrder;
+	private List<Tuple2<String, String>> outEdgesInOrder;
 
 	public OutputHandler(StreamVertex<?, OUT> vertex) {
 
@@ -68,24 +69,16 @@ public class OutputHandler<OUT> {
 		// We read the chained configs, and the order of record writer
 		// registrations by outputname
 		this.chainedConfigs = configuration.getTransitiveChainedTaskConfigs(cl);
-		this.recordWriterOrder = configuration.getRecordWriterOrder(cl);
-
-		// For the network outputs of the chain head we create the stream
-		// outputs
-		for (String outName : configuration.getOutputs(cl)) {
-			StreamOutput<?> streamOutput = createStreamOutput(outName, configuration);
-			outputMap.put(outName, streamOutput);
-		}
-
-		// If we have chained tasks we iterate through them and create the
-		// stream outputs for the network outputs
-		if (chainedConfigs != null) {
-			for (StreamConfig config : chainedConfigs.values()) {
-				for (String outName : config.getOutputs(cl)) {
-					StreamOutput<?> streamOutput = createStreamOutput(outName, config);
-					outputMap.put(outName, streamOutput);
-				}
-			}
+		this.chainedConfigs.put(configuration.getTaskName(), configuration);
+				
+		this.outEdgesInOrder = configuration.getOutEdgesInOrder(cl);
+
+		// We iterate through all the out edges from this job vertex and create
+		// a stream output
+		for (Tuple2<String, String> outEdge : outEdgesInOrder) {
+			StreamOutput<?> streamOutput = createStreamOutput(outEdge.f1,
+					chainedConfigs.get(outEdge.f0), outEdgesInOrder.indexOf(outEdge));
+			outputMap.put(outEdge.f1, streamOutput);
 		}
 
 		// We create the outer collector that will be passed to the first task
@@ -196,40 +189,32 @@ public class OutputHandler<OUT> {
 	 * We create the StreamOutput for the specific output given by the name, and
 	 * the configuration of its source task
 	 * 
-	 * @param name
+	 * @param outputVertex
 	 *            Name of the output to which the streamoutput will be set up
 	 * @param configuration
 	 *            The config of upStream task
 	 * @return
 	 */
-	private <T> StreamOutput<T> createStreamOutput(String name, StreamConfig configuration) {
-
-		int outputNumber = recordWriterOrder.indexOf(name);
-
-		StreamPartitioner<T> outputPartitioner;
-
-		try {
-			outputPartitioner = configuration.getPartitioner(vertex.userClassLoader, name);
-		} catch (Exception e) {
-			throw new StreamVertexException("Cannot deserialize partitioner for "
-					+ vertex.getName() + " with " + name + " outputs", e);
-		}
+	private <T> StreamOutput<T> createStreamOutput(String outputVertex, StreamConfig configuration,
+			int outputIndex) {
+		
+		StreamPartitioner<T> outputPartitioner = configuration.getPartitioner(cl, outputVertex);
 
 		RecordWriter<SerializationDelegate<StreamRecord<T>>> output;
 
-		long bufferTimeout = configuration.getBufferTimeout();
+		if (configuration.getBufferTimeout() >= 0) {
 
-		if (bufferTimeout >= 0) {
 			output = new StreamRecordWriter<SerializationDelegate<StreamRecord<T>>>(vertex
-					.getEnvironment().getWriter(outputNumber), outputPartitioner, bufferTimeout);
+					.getEnvironment().getWriter(outputIndex), outputPartitioner,
+					configuration.getBufferTimeout());
 
 			if (LOG.isTraceEnabled()) {
 				LOG.trace("StreamRecordWriter initiated with {} bufferTimeout for {}",
-						bufferTimeout, vertex.getClass().getSimpleName());
+						configuration.getBufferTimeout(), vertex.getClass().getSimpleName());
 			}
 		} else {
 			output = new RecordWriter<SerializationDelegate<StreamRecord<T>>>(vertex
-					.getEnvironment().getWriter(outputNumber), outputPartitioner);
+					.getEnvironment().getWriter(outputIndex), outputPartitioner);
 
 			if (LOG.isTraceEnabled()) {
 				LOG.trace("RecordWriter initiated for {}", vertex.getClass().getSimpleName());
@@ -237,11 +222,12 @@ public class OutputHandler<OUT> {
 		}
 
 		StreamOutput<T> streamOutput = new StreamOutput<T>(output,
-				configuration.isSelectAll(name) ? null : configuration.getOutputNames(name));
+				configuration.isSelectAll(outputVertex) ? null
+						: configuration.getOutputNames(outputVertex));
 
 		if (LOG.isTraceEnabled()) {
 			LOG.trace("Partitioner set: {} with {} outputs for {}", outputPartitioner.getClass()
-					.getSimpleName(), outputNumber, vertex.getClass().getSimpleName());
+					.getSimpleName(), outputIndex, vertex.getClass().getSimpleName());
 		}
 
 		return streamOutput;

http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/TestStreamEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/TestStreamEnvironment.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/TestStreamEnvironment.java
index e7f15b6..95a5d9b 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/TestStreamEnvironment.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/TestStreamEnvironment.java
@@ -46,7 +46,7 @@ public class TestStreamEnvironment extends StreamExecutionEnvironment {
 
 	@Override
 	public void execute(String jobName) throws Exception {
-		JobGraph jobGraph = jobGraphBuilder.getJobGraph(jobName);
+		JobGraph jobGraph = streamGraph.getJobGraph(jobName);
 
 		Configuration configuration = jobGraph.getJobConfiguration();
 

http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamCrossOperator.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamCrossOperator.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamCrossOperator.scala
index a69454c..a408ec0 100644
--- a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamCrossOperator.scala
+++ b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamCrossOperator.scala
@@ -83,7 +83,7 @@ object StreamCrossOperator {
         clean(getCrossWindowFunction(op, fun)), op.windowSize, op.slideInterval, op.timeStamp1,
         op.timeStamp2)
 
-      javaStream.getExecutionEnvironment().getJobGraphBuilder().setInvokable(javaStream.getId(),
+      javaStream.getExecutionEnvironment().getStreamGraph().setInvokable(javaStream.getId(),
         invokable)
 
       javaStream.setType(implicitly[TypeInformation[R]])
@@ -94,7 +94,7 @@ object StreamCrossOperator {
     }
 
     override def every(length: Long): CrossWindow[I1, I2] = {
-      val builder = javaStream.getExecutionEnvironment().getJobGraphBuilder()
+      val builder = javaStream.getExecutionEnvironment().getStreamGraph()
       val invokable = builder.getInvokable(javaStream.getId())
       invokable.asInstanceOf[CoWindowInvokable[_,_,_]].setSlideSize(length)
       this

http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamJoinOperator.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamJoinOperator.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamJoinOperator.scala
index 7ecd79a..1bd1bfb 100644
--- a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamJoinOperator.scala
+++ b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamJoinOperator.scala
@@ -186,7 +186,7 @@ object StreamJoinOperator {
         clean(getJoinWindowFunction(jp, fun)), op.windowSize, op.slideInterval, op.timeStamp1,
         op.timeStamp2)
 
-      javaStream.getExecutionEnvironment().getJobGraphBuilder().setInvokable(javaStream.getId(),
+      javaStream.getExecutionEnvironment().getStreamGraph().setInvokable(javaStream.getId(),
         invokable)
 
       javaStream.setType(implicitly[TypeInformation[R]])