You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by mb...@apache.org on 2015/01/21 18:53:55 UTC
[1/8] flink git commit: [FLINK-1403] [streaming] Distributed
filesystem support for streaming filesinks
Repository: flink
Updated Branches:
refs/heads/master 7df6a3d72 -> 92947f0d8
[FLINK-1403] [streaming] Distributed filesystem support for streaming filesinks
Now streaming filesinks support the same filesystems as the batch ones
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/dc0d81bb
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/dc0d81bb
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/dc0d81bb
Branch: refs/heads/master
Commit: dc0d81bb8bc056f9c8def5ce701c1aa121d2a13f
Parents: 7df6a3d
Author: szape <ne...@gmail.com>
Authored: Tue Jan 13 14:54:01 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Wed Jan 21 16:05:25 2015 +0100
----------------------------------------------------------------------
.../streaming/api/datastream/DataStream.java | 163 +++++++++++++++----
.../api/function/sink/FileSinkFunction.java | 118 ++++++++++++++
.../function/sink/FileSinkFunctionByMillis.java | 59 +++++++
.../api/function/sink/RichSinkFunction.java | 2 +-
.../api/function/sink/SinkFunction.java | 2 +-
5 files changed, 306 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/dc0d81bb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index 7d1659f..db644e9 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -20,6 +20,7 @@ package org.apache.flink.streaming.api.datastream;
import java.util.ArrayList;
import java.util.List;
+import org.apache.commons.lang3.Validate;
import org.apache.flink.api.common.functions.FilterFunction;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.functions.MapFunction;
@@ -28,15 +29,20 @@ import org.apache.flink.api.common.functions.RichFilterFunction;
import org.apache.flink.api.common.functions.RichFlatMapFunction;
import org.apache.flink.api.common.functions.RichMapFunction;
import org.apache.flink.api.common.functions.RichReduceFunction;
+import org.apache.flink.api.common.io.OutputFormat;
import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo;
import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.ClosureCleaner;
import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.io.CsvOutputFormat;
+import org.apache.flink.api.java.io.TextOutputFormat;
import org.apache.flink.api.java.operators.Keys;
import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.core.fs.FileSystem.WriteMode;
+import org.apache.flink.core.fs.Path;
import org.apache.flink.streaming.api.JobGraphBuilder;
import org.apache.flink.streaming.api.datastream.temporaloperator.StreamCrossOperator;
import org.apache.flink.streaming.api.datastream.temporaloperator.StreamJoinOperator;
@@ -45,12 +51,9 @@ import org.apache.flink.streaming.api.function.aggregation.AggregationFunction;
import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.AggregationType;
import org.apache.flink.streaming.api.function.aggregation.ComparableAggregator;
import org.apache.flink.streaming.api.function.aggregation.SumAggregator;
+import org.apache.flink.streaming.api.function.sink.FileSinkFunctionByMillis;
import org.apache.flink.streaming.api.function.sink.PrintSinkFunction;
import org.apache.flink.streaming.api.function.sink.SinkFunction;
-import org.apache.flink.streaming.api.function.sink.WriteFormat;
-import org.apache.flink.streaming.api.function.sink.WriteFormatAsCsv;
-import org.apache.flink.streaming.api.function.sink.WriteFormatAsText;
-import org.apache.flink.streaming.api.function.sink.WriteSinkFunctionByMillis;
import org.apache.flink.streaming.api.invokable.SinkInvokable;
import org.apache.flink.streaming.api.invokable.StreamInvokable;
import org.apache.flink.streaming.api.invokable.operator.CounterInvokable;
@@ -897,20 +900,35 @@ public class DataStream<OUT> {
}
/**
+ * Writes a DataStream to the file specified by path in text format. For
+ * every element of the DataStream the result of {@link Object#toString()}
+ * is written.
+ *
+ * @param path
+ * the path pointing to the location the text file is written to
+ *
+ * @return the closed DataStream.
+ */
+ public DataStreamSink<OUT> writeAsText(String path) {
+ return writeToFile(new TextOutputFormat<OUT>(new Path(path)), 0L);
+ }
+
+ /**
* Writes a DataStream to the file specified by path in text format. The
* writing is performed periodically, in every millis milliseconds. For
* every element of the DataStream the result of {@link Object#toString()}
* is written.
*
* @param path
- * is the path to the location where the tuples are written
+ * the path pointing to the location the text file is written to
* @param millis
- * is the file update frequency
+ * the file update frequency
*
- * @return The closed DataStream
+ * @return the closed DataStream
*/
public DataStreamSink<OUT> writeAsText(String path, long millis) {
- return writeAsText(path, new WriteFormatAsText<OUT>(), millis);
+ TextOutputFormat<OUT> tof = new TextOutputFormat<OUT>(new Path(path));
+ return writeToFile(tof, millis);
}
/**
@@ -919,64 +937,137 @@ public class DataStream<OUT> {
* is written.
*
* @param path
- * is the path to the location where the tuples are written
+ * the path pointing to the location the text file is written to
+ * @param writeMode
+ * Control the behavior for existing files. Options are
+ * NO_OVERWRITE and OVERWRITE.
*
- * @return The closed DataStream
+ * @return the closed DataStream.
*/
- public DataStreamSink<OUT> writeAsText(String path) {
- return writeAsText(path, 0);
+ public DataStreamSink<OUT> writeAsText(String path, WriteMode writeMode) {
+ TextOutputFormat<OUT> tof = new TextOutputFormat<OUT>(new Path(path));
+ tof.setWriteMode(writeMode);
+ return writeToFile(tof, 0L);
}
/**
- * Writes a DataStream to the file specified by path in text format. The
- * writing is performed periodically, in every millis milliseconds. For
+ * Writes a DataStream to the file specified by path in text format. For
* every element of the DataStream the result of {@link Object#toString()}
* is written.
*
* @param path
- * is the path to the location where the tuples are written
+ * the path pointing to the location the text file is written to
+ * @param writeMode
+ * Controls the behavior for existing files. Options are
+ * NO_OVERWRITE and OVERWRITE.
* @param millis
- * is the file update frequency
+ * the file update frequency
*
- * @return The closed DataStream
+ * @return the closed DataStream.
*/
- public DataStreamSink<OUT> writeAsCsv(String path, long millis) {
- if (!getType().isTupleType()) {
- throw new RuntimeException("Only tuple data streams can be written in csv format");
- }
- return writeAsText(path, new WriteFormatAsCsv<OUT>(), millis);
+ public DataStreamSink<OUT> writeAsText(String path, WriteMode writeMode, long millis) {
+ TextOutputFormat<OUT> tof = new TextOutputFormat<OUT>(new Path(path));
+ tof.setWriteMode(writeMode);
+ return writeToFile(tof, millis);
}
/**
- * Writes a DataStream to the file specified by path in text format. For
+ * Writes a DataStream to the file specified by path in csv format. For
* every element of the DataStream the result of {@link Object#toString()}
- * is written.
+ * is written. This method can only be used on data streams of tuples.
*
* @param path
- * is the path to the location where the tuples are written
+ * the path pointing to the location the text file is written to
*
- * @return The closed DataStream
+ * @return the closed DataStream
*/
- public DataStreamSink<OUT> writeAsCsv(String path) {
- return writeAsCsv(path, 0);
+ @SuppressWarnings("unchecked")
+ public <X extends Tuple> DataStreamSink<OUT> writeAsCsv(String path) {
+ Validate.isTrue(getType().isTupleType(),
+ "The writeAsCsv() method can only be used on data sets of tuples.");
+ CsvOutputFormat<X> of = new CsvOutputFormat<X>(new Path(path),
+ CsvOutputFormat.DEFAULT_LINE_DELIMITER, CsvOutputFormat.DEFAULT_FIELD_DELIMITER);
+ return writeToFile((OutputFormat<OUT>) of, 0L);
}
/**
- * Writes a DataStream to the file specified by path in text format. The
+ * Writes a DataStream to the file specified by path in csv format. The
* writing is performed periodically, in every millis milliseconds. For
* every element of the DataStream the result of {@link Object#toString()}
- * is written.
+ * is written. This method can only be used on data streams of tuples.
*
* @param path
- * is the path to the location where the tuples are written
+ * the path pointing to the location the text file is written to
* @param millis
- * is the file update frequency
+ * the file update frequency
*
- * @return the data stream constructed
+ * @return the closed DataStream
*/
- private DataStreamSink<OUT> writeAsText(String path, WriteFormat<OUT> format, long millis) {
- DataStreamSink<OUT> returnStream = addSink(new WriteSinkFunctionByMillis<OUT>(path, format,
- millis));
+ @SuppressWarnings("unchecked")
+ public <X extends Tuple> DataStreamSink<OUT> writeAsCsv(String path, long millis) {
+ Validate.isTrue(getType().isTupleType(),
+ "The writeAsCsv() method can only be used on data sets of tuples.");
+ CsvOutputFormat<X> of = new CsvOutputFormat<X>(new Path(path),
+ CsvOutputFormat.DEFAULT_LINE_DELIMITER, CsvOutputFormat.DEFAULT_FIELD_DELIMITER);
+ return writeToFile((OutputFormat<OUT>) of, millis);
+ }
+
+ /**
+ * Writes a DataStream to the file specified by path in csv format. For
+ * every element of the DataStream the result of {@link Object#toString()}
+ * is written. This method can only be used on data streams of tuples.
+ *
+ * @param path
+ * the path pointing to the location the text file is written to
+ * @param writeMode
+ * Controls the behavior for existing files. Options are
+ * NO_OVERWRITE and OVERWRITE.
+ *
+ * @return the closed DataStream
+ */
+ @SuppressWarnings("unchecked")
+ public <X extends Tuple> DataStreamSink<OUT> writeAsCsv(String path, WriteMode writeMode) {
+ Validate.isTrue(getType().isTupleType(),
+ "The writeAsCsv() method can only be used on data sets of tuples.");
+ CsvOutputFormat<X> of = new CsvOutputFormat<X>(new Path(path),
+ CsvOutputFormat.DEFAULT_LINE_DELIMITER, CsvOutputFormat.DEFAULT_FIELD_DELIMITER);
+ if (writeMode != null) {
+ of.setWriteMode(writeMode);
+ }
+ return writeToFile((OutputFormat<OUT>) of, 0L);
+ }
+
+ /**
+ * Writes a DataStream to the file specified by path in csv format. The
+ * writing is performed periodically, in every millis milliseconds. For
+ * every element of the DataStream the result of {@link Object#toString()}
+ * is written. This method can only be used on data streams of tuples.
+ *
+ * @param path
+ * the path pointing to the location the text file is written to
+ * @param writeMode
+ * Controls the behavior for existing files. Options are
+ * NO_OVERWRITE and OVERWRITE.
+ * @param millis
+ * the file update frequency
+ *
+ * @return the closed DataStream
+ */
+ @SuppressWarnings("unchecked")
+ public <X extends Tuple> DataStreamSink<OUT> writeAsCsv(String path, WriteMode writeMode,
+ long millis) {
+ Validate.isTrue(getType().isTupleType(),
+ "The writeAsCsv() method can only be used on data sets of tuples.");
+ CsvOutputFormat<X> of = new CsvOutputFormat<X>(new Path(path),
+ CsvOutputFormat.DEFAULT_LINE_DELIMITER, CsvOutputFormat.DEFAULT_FIELD_DELIMITER);
+ if (writeMode != null) {
+ of.setWriteMode(writeMode);
+ }
+ return writeToFile((OutputFormat<OUT>) of, millis);
+ }
+
+ private DataStreamSink<OUT> writeToFile(OutputFormat<OUT> format, long millis) {
+ DataStreamSink<OUT> returnStream = addSink(new FileSinkFunctionByMillis<OUT>(format, millis));
return returnStream;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/dc0d81bb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/FileSinkFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/FileSinkFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/FileSinkFunction.java
new file mode 100644
index 0000000..24beba1
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/FileSinkFunction.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.sink;
+
+import java.io.IOException;
+import java.util.ArrayList;
+
+import org.apache.flink.api.common.io.CleanupWhenUnsuccessful;
+import org.apache.flink.api.common.io.OutputFormat;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.streamvertex.StreamingRuntimeContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Simple implementation of the SinkFunction writing tuples in the specified
+ * OutputFormat format. Tuples are collected to a list and written to the file
+ * periodically. The target path and the overwrite mode are pre-packaged in
+ * format.
+ *
+ * @param <IN>
+ * Input type
+ */
+public abstract class FileSinkFunction<IN> extends RichSinkFunction<IN> {
+ private static final long serialVersionUID = 1L;
+ private static final Logger LOG = LoggerFactory.getLogger(FileSinkFunction.class);
+ protected ArrayList<IN> tupleList = new ArrayList<IN>();
+ protected volatile OutputFormat<IN> format;
+ protected volatile boolean cleanupCalled = false;
+ protected int indexInSubtaskGroup;
+ protected int currentNumberOfSubtasks;
+
+ public FileSinkFunction(OutputFormat<IN> format) {
+ this.format = format;
+ }
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ StreamingRuntimeContext context = (StreamingRuntimeContext) getRuntimeContext();
+ format.configure(context.getTaskStubParameters());
+ indexInSubtaskGroup = context.getIndexOfThisSubtask();
+ currentNumberOfSubtasks = context.getNumberOfParallelSubtasks();
+ format.open(indexInSubtaskGroup, currentNumberOfSubtasks);
+ }
+
+ @Override
+ public void invoke(IN record) throws Exception {
+ tupleList.add(record);
+ if (updateCondition()) {
+ flush();
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (!tupleList.isEmpty()) {
+ flush();
+ }
+ try {
+ format.close();
+ } catch (Exception ex) {
+ try {
+ if (!cleanupCalled && format instanceof CleanupWhenUnsuccessful) {
+ cleanupCalled = true;
+ ((CleanupWhenUnsuccessful) format).tryCleanupOnError();
+ }
+ } catch (Throwable t) {
+ LOG.error("Cleanup on error failed.", t);
+ }
+ }
+ }
+
+ protected void flush() {
+ try {
+ for (IN rec : tupleList) {
+ format.writeRecord(rec);
+ }
+ } catch (Exception ex) {
+ try {
+ if (!cleanupCalled && format instanceof CleanupWhenUnsuccessful) {
+ cleanupCalled = true;
+ ((CleanupWhenUnsuccessful) format).tryCleanupOnError();
+ }
+ } catch (Throwable t) {
+ LOG.error("Cleanup on error failed.", t);
+ }
+ }
+ resetParameters();
+ }
+
+ /**
+ * Condition for writing the contents of tupleList and clearing it.
+ *
+ * @return value of the updating condition
+ */
+ protected abstract boolean updateCondition();
+
+ /**
+ * Statements to be executed after writing a batch goes here.
+ */
+ protected abstract void resetParameters();
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/dc0d81bb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/FileSinkFunctionByMillis.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/FileSinkFunctionByMillis.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/FileSinkFunctionByMillis.java
new file mode 100644
index 0000000..f049a32
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/FileSinkFunctionByMillis.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.sink;
+
+import org.apache.flink.api.common.io.OutputFormat;
+
+/**
+ * Implementation of FileSinkFunction. Writes tuples to file in every millis
+ * milliseconds.
+ *
+ * @param <IN>
+ * Input type
+ */
+public class FileSinkFunctionByMillis<IN> extends FileSinkFunction<IN> {
+ private static final long serialVersionUID = 1L;
+
+ private final long millis;
+ private long lastTime;
+
+ public FileSinkFunctionByMillis(OutputFormat<IN> format, long millis) {
+ super(format);
+ this.millis = millis;
+ lastTime = System.currentTimeMillis();
+ }
+
+ /**
+ * Condition for writing the contents of tupleList and clearing it.
+ *
+ * @return value of the updating condition
+ */
+ @Override
+ protected boolean updateCondition() {
+ return System.currentTimeMillis() - lastTime >= millis;
+ }
+
+ /**
+ * Statements to be executed after writing a batch goes here.
+ */
+ @Override
+ protected void resetParameters() {
+ tupleList.clear();
+ lastTime = System.currentTimeMillis();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/dc0d81bb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/RichSinkFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/RichSinkFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/RichSinkFunction.java
index b89c169..3b8a4db 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/RichSinkFunction.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/RichSinkFunction.java
@@ -23,6 +23,6 @@ public abstract class RichSinkFunction<IN> extends AbstractRichFunction implemen
private static final long serialVersionUID = 1L;
- public abstract void invoke(IN value);
+ public abstract void invoke(IN value) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/dc0d81bb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/SinkFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/SinkFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/SinkFunction.java
index 9db268e..6097603 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/SinkFunction.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/SinkFunction.java
@@ -23,6 +23,6 @@ import org.apache.flink.api.common.functions.Function;
public interface SinkFunction<IN> extends Function, Serializable {
- public abstract void invoke(IN value);
+ public abstract void invoke(IN value) throws Exception;
}
[3/8] flink git commit: [streaming] Streaming runtime collector
rework for increased flexibility with directed outputs and operator chains
Posted by mb...@apache.org.
[streaming] Streaming runtime collector rework for increased flexibility with directed outputs and operator chains
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/d9b942b3
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/d9b942b3
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/d9b942b3
Branch: refs/heads/master
Commit: d9b942b38cea01bf78ffab22a164b6622d629bc5
Parents: dc0d81b
Author: Gyula Fora <gy...@apache.org>
Authored: Thu Jan 15 11:14:33 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Wed Jan 21 16:06:33 2015 +0100
----------------------------------------------------------------------
.../api/collector/CollectorWrapper.java | 48 ++++++
.../api/collector/DirectedOutputWrapper.java | 144 ++++++++++++++++++
.../api/collector/DirectedStreamCollector.java | 130 ----------------
.../api/collector/StreamCollector.java | 148 -------------------
.../streaming/api/collector/StreamOutput.java | 65 ++++++++
.../api/collector/StreamOutputWrapper.java | 122 +++++++++++++++
.../api/streamvertex/InputHandler.java | 1 -
.../api/streamvertex/OutputHandler.java | 26 ++--
.../flink/streaming/io/StreamRecordWriter.java | 2 +-
.../api/collector/StreamCollectorTest.java | 4 +-
.../runtime/operators/RegularPactTask.java | 2 -
11 files changed, 396 insertions(+), 296 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/d9b942b3/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
new file mode 100644
index 0000000..d782d08
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.collector;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.flink.util.Collector;
+
+public class CollectorWrapper<OUT> implements Collector<OUT> {
+
+ List<Collector<OUT>> outputs;
+
+ public CollectorWrapper() {
+ this.outputs = new LinkedList<Collector<OUT>>();
+ }
+
+ public void addChainedOutput(Collector<OUT> output) {
+ outputs.add(output);
+ }
+
+ @Override
+ public void collect(OUT record) {
+ for(Collector<OUT> output: outputs){
+ output.collect(record);;
+ }
+ }
+
+ @Override
+ public void close() {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/d9b942b3/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedOutputWrapper.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedOutputWrapper.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedOutputWrapper.java
new file mode 100755
index 0000000..6ff03c4
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedOutputWrapper.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.collector;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.flink.runtime.plugable.SerializationDelegate;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.apache.flink.util.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A StreamCollector that uses user defined output names and a user defined
+ * output selector to make directed emits.
+ *
+ * @param <OUT>
+ * Type of the Tuple collected.
+ */
+public class DirectedOutputWrapper<OUT> extends StreamOutputWrapper<OUT> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(DirectedOutputWrapper.class);
+
+ OutputSelector<OUT> outputSelector;
+
+ protected Map<String, List<StreamOutput<OUT>>> outputMap;
+
+ private List<StreamOutput<OUT>> selectAllOutputs;
+ private Set<StreamOutput<OUT>> emitted;
+
+ /**
+ * Creates a new DirectedStreamCollector
+ *
+ * @param channelID
+ * Channel ID of the Task
+ * @param serializationDelegate
+ * Serialization delegate used for serialization
+ * @param outputSelector
+ * User defined {@link OutputSelector}
+ */
+ public DirectedOutputWrapper(int channelID,
+ SerializationDelegate<StreamRecord<OUT>> serializationDelegate,
+ OutputSelector<OUT> outputSelector) {
+ super(channelID, serializationDelegate);
+ this.outputSelector = outputSelector;
+ this.emitted = new HashSet<StreamOutput<OUT>>();
+ this.selectAllOutputs = new LinkedList<StreamOutput<OUT>>();
+ this.outputMap = new HashMap<String, List<StreamOutput<OUT>>>();
+
+ }
+
+ @Override
+ public void addOutput(StreamOutput<OUT> output) {
+
+ super.addOutput(output);
+
+ if (output.isSelectAll()) {
+ selectAllOutputs.add(output);
+ } else {
+ for (String selectedName : output.getSelectedNames()) {
+ if (selectedName != null) {
+ if (!outputMap.containsKey(selectedName)) {
+ outputMap.put(selectedName, new LinkedList<StreamOutput<OUT>>());
+ outputMap.get(selectedName).add(output);
+ } else {
+ if (!outputMap.get(selectedName).contains(output)) {
+ outputMap.get(selectedName).add(output);
+ }
+ }
+
+ }
+ }
+ }
+ }
+
+ @Override
+ protected void emit() {
+ Iterable<String> outputNames = outputSelector.select(serializationDelegate.getInstance()
+ .getObject());
+ emitted.clear();
+
+ for (StreamOutput<OUT> output : selectAllOutputs) {
+ try {
+ output.collect(serializationDelegate);
+ } catch (Exception e) {
+ if (LOG.isErrorEnabled()) {
+ LOG.error("Emit to {} failed due to: {}", output,
+ StringUtils.stringifyException(e));
+ }
+ }
+ }
+
+ for (String outputName : outputNames) {
+ List<StreamOutput<OUT>> outputList = outputMap.get(outputName);
+ try {
+ if (outputList == null) {
+ if (LOG.isErrorEnabled()) {
+ String format = String.format(
+ "Cannot emit because no output is selected with the name: %s",
+ outputName);
+ LOG.error(format);
+
+ }
+ } else {
+
+ for (StreamOutput<OUT> output : outputList) {
+ if (!emitted.contains(output)) {
+ output.collect(serializationDelegate);
+ emitted.add(output);
+ }
+ }
+
+ }
+
+ } catch (Exception e) {
+ if (LOG.isErrorEnabled()) {
+ LOG.error("Emit to {} failed due to: {}", outputName,
+ StringUtils.stringifyException(e));
+ }
+ }
+
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/d9b942b3/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedStreamCollector.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedStreamCollector.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedStreamCollector.java
deleted file mode 100755
index f37761f..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedStreamCollector.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.collector;
-
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
-import org.apache.flink.runtime.plugable.SerializationDelegate;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.apache.flink.util.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * A StreamCollector that uses user defined output names and a user defined
- * output selector to make directed emits.
- *
- * @param <OUT>
- * Type of the Tuple collected.
- */
-public class DirectedStreamCollector<OUT> extends StreamCollector<OUT> {
-
- private static final Logger LOG = LoggerFactory.getLogger(DirectedStreamCollector.class);
-
- OutputSelector<OUT> outputSelector;
- private List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> selectAllOutputs;
- private Set<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> emitted;
-
- /**
- * Creates a new DirectedStreamCollector
- *
- * @param channelID
- * Channel ID of the Task
- * @param serializationDelegate
- * Serialization delegate used for serialization
- * @param outputSelector
- * User defined {@link OutputSelector}
- */
- public DirectedStreamCollector(int channelID,
- SerializationDelegate<StreamRecord<OUT>> serializationDelegate,
- OutputSelector<OUT> outputSelector) {
- super(channelID, serializationDelegate);
- this.outputSelector = outputSelector;
- this.emitted = new HashSet<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>>();
- this.selectAllOutputs = new LinkedList<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>>();
- }
-
- @Override
- public void addOutput(RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output,
- List<String> outputNames, boolean isSelectAllOutput) {
-
- if (isSelectAllOutput) {
- selectAllOutputs.add(output);
- } else {
- addOneOutput(output, outputNames, isSelectAllOutput);
- }
- }
-
- /**
- * Emits a StreamRecord to the outputs selected by the user defined
- * OutputSelector
- *
- */
- protected void emitToOutputs() {
- Iterable<String> outputNames = outputSelector.select(streamRecord.getObject());
- emitted.clear();
-
- for (RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output : selectAllOutputs) {
- try {
- output.emit(serializationDelegate);
- } catch (Exception e) {
- if (LOG.isErrorEnabled()) {
- LOG.error("Emit to {} failed due to: {}", output,
- StringUtils.stringifyException(e));
- }
- }
- }
- emitted.addAll(selectAllOutputs);
-
- for (String outputName : outputNames) {
- List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> outputList = outputMap
- .get(outputName);
- try {
- if (outputList == null) {
- if (LOG.isErrorEnabled()) {
- String format = String.format(
- "Cannot emit because no output is selected with the name: %s",
- outputName);
- LOG.error(format);
-
- }
- } else {
-
- for (RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output : outputList) {
- if (!emitted.contains(output)) {
- output.emit(serializationDelegate);
- emitted.add(output);
- }
- }
-
- }
-
- } catch (Exception e) {
- if (LOG.isErrorEnabled()) {
- LOG.error("Emit to {} failed due to: {}", outputName,
- StringUtils.stringifyException(e));
- }
- }
-
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/d9b942b3/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamCollector.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamCollector.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamCollector.java
deleted file mode 100755
index a1b9e66..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamCollector.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.collector;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
-import org.apache.flink.runtime.plugable.SerializationDelegate;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.apache.flink.util.Collector;
-import org.apache.flink.util.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Collector for tuples in Apache Flink stream processing. The collected values
- * will be wrapped with ID in a {@link StreamRecord} and then emitted to the
- * outputs.
- *
- * @param <OUT>
- * Type of the Tuples/Objects collected.
- */
-public class StreamCollector<OUT> implements Collector<OUT> {
-
- private static final Logger LOG = LoggerFactory.getLogger(StreamCollector.class);
-
- protected StreamRecord<OUT> streamRecord;
- protected int channelID;
- protected List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> outputs;
- protected Map<String, List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>>> outputMap;
- protected SerializationDelegate<StreamRecord<OUT>> serializationDelegate;
-
- /**
- * Creates a new StreamCollector
- *
- * @param channelID
- * Channel ID of the Task
- * @param serializationDelegate
- * Serialization delegate used for serialization
- */
- public StreamCollector(int channelID,
- SerializationDelegate<StreamRecord<OUT>> serializationDelegate) {
- this.serializationDelegate = serializationDelegate;
-
- if (serializationDelegate != null) {
- this.streamRecord = serializationDelegate.getInstance();
- } else {
- this.streamRecord = new StreamRecord<OUT>();
- }
- this.channelID = channelID;
- this.outputs = new ArrayList<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>>();
- this.outputMap = new HashMap<String, List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>>>();
- }
-
- /**
- * Adds an output with the given user defined name
- *
- * @param output
- * The RecordWriter object representing the output.
- * @param outputNames
- * User defined names of the output.
- * @param isSelectAllOutput
- * Marks whether all the outputs are selected.
- */
- public void addOutput(RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output,
- List<String> outputNames, boolean isSelectAllOutput) {
- addOneOutput(output, outputNames, isSelectAllOutput);
- }
-
- protected void addOneOutput(RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output,
- List<String> outputNames, boolean isSelectAllOutput) {
- outputs.add(output);
- for (String outputName : outputNames) {
- if (outputName != null) {
- if (!outputMap.containsKey(outputName)) {
- outputMap
- .put(outputName,
- new ArrayList<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>>());
- outputMap.get(outputName).add(output);
- } else {
- if (!outputMap.get(outputName).contains(output)) {
- outputMap.get(outputName).add(output);
- }
- }
-
- }
- }
- }
-
- /**
- * Collects and emits a tuple/object to the outputs by reusing a
- * StreamRecord object.
- *
- * @param outputObject
- * Object to be collected and emitted.
- */
- @Override
- public void collect(OUT outputObject) {
- streamRecord.setObject(outputObject);
- emit(streamRecord);
- }
-
- /**
- * Emits a StreamRecord to the outputs.
- *
- * @param streamRecord
- * StreamRecord to emit.
- */
- private void emit(StreamRecord<OUT> streamRecord) {
- streamRecord.newId(channelID);
- serializationDelegate.setInstance(streamRecord);
- emitToOutputs();
- }
-
- protected void emitToOutputs() {
- for (RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output : outputs) {
- try {
- output.emit(serializationDelegate);
- } catch (Exception e) {
- if (LOG.isErrorEnabled()) {
- LOG.error("Emit failed due to: {}", StringUtils.stringifyException(e));
- }
- }
- }
- }
-
- @Override
- public void close() {
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/d9b942b3/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java
new file mode 100644
index 0000000..4c21564
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.collector;
+
+import java.util.List;
+
+import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
+import org.apache.flink.runtime.plugable.SerializationDelegate;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.apache.flink.util.Collector;
+
+public class StreamOutput<OUT> implements Collector<SerializationDelegate<StreamRecord<OUT>>> {
+
+ private RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output;
+
+ private List<String> selectedNames;
+ private boolean selectAll = true;
+
+ public StreamOutput(RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output,
+ List<String> selectedNames) {
+
+ this.output = output;
+
+ if (selectedNames != null) {
+ this.selectedNames = selectedNames;
+ selectAll = false;
+ }
+ }
+
+ public void collect(SerializationDelegate<StreamRecord<OUT>> record) {
+ try {
+ output.emit(record);
+ } catch (Exception e) {
+ throw new RuntimeException("Could not emit record: " + record.getInstance());
+ }
+ }
+
+ @Override
+ public void close() {
+ }
+
+ public boolean isSelectAll() {
+ return selectAll;
+ }
+
+ public List<String> getSelectedNames() {
+ return selectedNames;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/d9b942b3/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutputWrapper.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutputWrapper.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutputWrapper.java
new file mode 100755
index 0000000..fa374b1
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutputWrapper.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.collector;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
+import org.apache.flink.runtime.plugable.SerializationDelegate;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.apache.flink.util.Collector;
+import org.apache.flink.util.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Collector for tuples in Apache Flink stream processing. The collected values
+ * will be wrapped with ID in a {@link StreamRecord} and then emitted to the
+ * outputs.
+ *
+ * @param <OUT>
+ * Type of the Tuples/Objects collected.
+ */
+public class StreamOutputWrapper<OUT> implements Collector<OUT> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(StreamOutputWrapper.class);
+
+ protected StreamRecord<OUT> streamRecord;
+ protected int channelID;
+ protected List<StreamOutput<OUT>> outputs;
+ protected SerializationDelegate<StreamRecord<OUT>> serializationDelegate;
+
+ /**
+ * Creates a new StreamCollector
+ *
+ * @param channelID
+ * Channel ID of the Task
+ * @param serializationDelegate
+ * Serialization delegate used for serialization
+ */
+ public StreamOutputWrapper(int channelID,
+ SerializationDelegate<StreamRecord<OUT>> serializationDelegate) {
+ this.serializationDelegate = serializationDelegate;
+
+ if (serializationDelegate != null) {
+ this.streamRecord = serializationDelegate.getInstance();
+ } else {
+ this.streamRecord = new StreamRecord<OUT>();
+ }
+ this.channelID = channelID;
+ this.outputs = new LinkedList<StreamOutput<OUT>>();
+ }
+
+ /**
+ * Adds an output with the given user defined name
+ *
+ * @param output
+ * The RecordWriter object representing the output.
+ * @param outputNames
+ * User defined names of the output.
+ * @param isSelectAllOutput
+ * Marks whether all the outputs are selected.
+ */
+ public void addOutput(StreamOutput<OUT> output) {
+ outputs.add(output);
+ }
+
+ protected void addOneOutput(RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output,
+ List<String> outputNames, boolean isSelectAllOutput) {
+
+ }
+
+ /**
+ * Collects and emits a tuple/object to the outputs by reusing a
+ * StreamRecord object.
+ *
+ * @param outputObject
+ * Object to be collected and emitted.
+ */
+ @Override
+ public void collect(OUT outputObject) {
+ streamRecord.setObject(outputObject);
+ streamRecord.newId(channelID);
+ serializationDelegate.setInstance(streamRecord);
+
+ emit();
+ }
+
+ /**
+ * Emits the current streamrecord to the outputs.
+ */
+ protected void emit() {
+ for (StreamOutput<OUT> output : outputs) {
+ try {
+ output.collect(serializationDelegate);
+ } catch (Exception e) {
+ if (LOG.isErrorEnabled()) {
+ LOG.error("Emit failed due to: {}", StringUtils.stringifyException(e));
+ }
+ }
+ }
+ }
+
+ @Override
+ public void close() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/d9b942b3/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/InputHandler.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/InputHandler.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/InputHandler.java
index fdf23f7..ad6d948 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/InputHandler.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/InputHandler.java
@@ -49,7 +49,6 @@ public class InputHandler<IN> {
}
- @SuppressWarnings("unchecked")
protected void setConfigInputs() throws StreamVertexException {
inputSerializer = configuration.getTypeSerializerIn1(streamVertex.userClassLoader);
http://git-wip-us.apache.org/repos/asf/flink/blob/d9b942b3/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
index f9fe593..61a6eb4 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
@@ -21,9 +21,10 @@ import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
import org.apache.flink.runtime.plugable.SerializationDelegate;
import org.apache.flink.streaming.api.StreamConfig;
-import org.apache.flink.streaming.api.collector.DirectedStreamCollector;
+import org.apache.flink.streaming.api.collector.DirectedOutputWrapper;
import org.apache.flink.streaming.api.collector.OutputSelector;
-import org.apache.flink.streaming.api.collector.StreamCollector;
+import org.apache.flink.streaming.api.collector.StreamOutputWrapper;
+import org.apache.flink.streaming.api.collector.StreamOutput;
import org.apache.flink.streaming.api.invokable.StreamInvokable;
import org.apache.flink.streaming.api.streamrecord.StreamRecord;
import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
@@ -43,7 +44,7 @@ public class OutputHandler<OUT> {
private StreamConfig configuration;
private List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> outputs;
- private StreamCollector<OUT> collector;
+ private StreamOutputWrapper<OUT> collector;
private long bufferTimeout;
TypeInformation<OUT> outTypeInfo = null;
@@ -79,21 +80,21 @@ public class OutputHandler<OUT> {
}
}
- private StreamCollector<OUT> setCollector() {
+ private StreamOutputWrapper<OUT> setCollector() {
if (streamVertex.configuration.getDirectedEmit()) {
OutputSelector<OUT> outputSelector = streamVertex.configuration
.getOutputSelector(streamVertex.userClassLoader);
- collector = new DirectedStreamCollector<OUT>(streamVertex.getInstanceID(),
+ collector = new DirectedOutputWrapper<OUT>(streamVertex.getInstanceID(),
outSerializationDelegate, outputSelector);
} else {
- collector = new StreamCollector<OUT>(streamVertex.getInstanceID(),
+ collector = new StreamOutputWrapper<OUT>(streamVertex.getInstanceID(),
outSerializationDelegate);
}
return collector;
}
- public StreamCollector<OUT> getCollector() {
+ public StreamOutputWrapper<OUT> getCollector() {
return collector;
}
@@ -121,16 +122,16 @@ public class OutputHandler<OUT> {
RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output;
if (bufferTimeout >= 0) {
- output = new StreamRecordWriter<SerializationDelegate<StreamRecord<OUT>>>(streamVertex.getEnvironment().getWriter(outputNumber),
- outputPartitioner, bufferTimeout);
+ output = new StreamRecordWriter<SerializationDelegate<StreamRecord<OUT>>>(streamVertex
+ .getEnvironment().getWriter(outputNumber), outputPartitioner, bufferTimeout);
if (LOG.isTraceEnabled()) {
LOG.trace("StreamRecordWriter initiated with {} bufferTimeout for {}",
bufferTimeout, streamVertex.getClass().getSimpleName());
}
} else {
- output = new RecordWriter<SerializationDelegate<StreamRecord<OUT>>>(streamVertex.getEnvironment().getWriter(outputNumber),
- outputPartitioner);
+ output = new RecordWriter<SerializationDelegate<StreamRecord<OUT>>>(streamVertex
+ .getEnvironment().getWriter(outputNumber), outputPartitioner);
if (LOG.isTraceEnabled()) {
LOG.trace("RecordWriter initiated for {}", streamVertex.getClass().getSimpleName());
@@ -142,7 +143,8 @@ public class OutputHandler<OUT> {
boolean isSelectAllOutput = configuration.getSelectAll(outputNumber);
if (collector != null) {
- collector.addOutput(output, outputName, isSelectAllOutput);
+ collector
+ .addOutput(new StreamOutput<OUT>(output, isSelectAllOutput ? null : outputName));
}
if (LOG.isTraceEnabled()) {
http://git-wip-us.apache.org/repos/asf/flink/blob/d9b942b3/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/StreamRecordWriter.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/StreamRecordWriter.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/StreamRecordWriter.java
index fe9cce3..b7af589 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/StreamRecordWriter.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/io/StreamRecordWriter.java
@@ -25,7 +25,7 @@ import org.apache.flink.runtime.io.network.api.writer.RoundRobinChannelSelector;
import java.io.IOException;
-public class StreamRecordWriter<T extends IOReadableWritable> extends RecordWriter {
+public class StreamRecordWriter<T extends IOReadableWritable> extends RecordWriter<T> {
private long timeout;
http://git-wip-us.apache.org/repos/asf/flink/blob/d9b942b3/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/StreamCollectorTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/StreamCollectorTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/StreamCollectorTest.java
index 535e109..0bc3d7d 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/StreamCollectorTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/StreamCollectorTest.java
@@ -37,8 +37,8 @@ public class StreamCollectorTest {
null);
sd.setInstance(new StreamRecord<Tuple1<Integer>>().setObject(new Tuple1<Integer>()));
- StreamCollector<Tuple1<Integer>> collector = new StreamCollector<Tuple1<Integer>>(2, sd);
- collector.addOutput(recWriter, new ArrayList<String>(), false);
+ StreamOutputWrapper<Tuple1<Integer>> collector = new StreamOutputWrapper<Tuple1<Integer>>(2, sd);
+ collector.addOutput(new StreamOutput<Tuple1<Integer>>(recWriter, new ArrayList<String>()));
collector.collect(new Tuple1<Integer>(3));
collector.collect(new Tuple1<Integer>(4));
collector.collect(new Tuple1<Integer>(5));
http://git-wip-us.apache.org/repos/asf/flink/blob/d9b942b3/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java
index 2d36c1e..6b6918d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java
@@ -706,7 +706,6 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
*
* This method requires that the task configuration, the driver, and the user-code class loader are set.
*/
- @SuppressWarnings("unchecked")
protected void initInputReaders() throws Exception {
final int numInputs = getNumTaskInputs();
final MutableReader<?>[] inputReaders = new MutableReader[numInputs];
@@ -748,7 +747,6 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
*
* This method requires that the task configuration, the driver, and the user-code class loader are set.
*/
- @SuppressWarnings("unchecked")
protected void initBroadcastInputReaders() throws Exception {
final int numBroadcastInputs = this.config.getNumBroadcastInputs();
final MutableReader<?>[] broadcastInputReaders = new MutableReader[numBroadcastInputs];
[4/8] flink git commit: [FLINK-1345] [streaming] Advanced task
chaining added
Posted by mb...@apache.org.
[FLINK-1345] [streaming] Advanced task chaining added
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/7dbb55ec
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/7dbb55ec
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/7dbb55ec
Branch: refs/heads/master
Commit: 7dbb55ece0a9d9777c0e3254bc8f9f5cf566d535
Parents: 3e30c6f
Author: Gyula Fora <gy...@apache.org>
Authored: Sun Jan 18 18:23:57 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Wed Jan 21 16:06:34 2015 +0100
----------------------------------------------------------------------
.../flink/streaming/api/JobGraphBuilder.java | 298 ++++++++-----------
.../flink/streaming/api/StreamConfig.java | 180 +++++++----
.../api/collector/CollectorWrapper.java | 2 +-
.../streaming/api/collector/StreamOutput.java | 15 +
.../api/collector/StreamOutputWrapper.java | 6 -
.../api/streamvertex/OutputHandler.java | 204 ++++++++-----
.../api/streamvertex/StreamIterationHead.java | 23 +-
.../flink/streaming/api/scala/DataStream.scala | 10 +
8 files changed, 433 insertions(+), 305 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/7dbb55ec/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
index 0020d48..6ae97c9 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
@@ -24,9 +24,10 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.commons.lang3.StringUtils;
import org.apache.flink.api.common.io.InputFormat;
import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.jobgraph.AbstractJobVertex;
import org.apache.flink.runtime.jobgraph.DistributionPattern;
import org.apache.flink.runtime.jobgraph.JobGraph;
@@ -34,7 +35,6 @@ import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
import org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup;
import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
import org.apache.flink.streaming.api.collector.OutputSelector;
-import org.apache.flink.streaming.api.invokable.ChainableInvokable;
import org.apache.flink.streaming.api.invokable.StreamInvokable;
import org.apache.flink.streaming.api.invokable.StreamInvokable.ChainingStrategy;
import org.apache.flink.streaming.api.invokable.operator.co.CoInvokable;
@@ -85,11 +85,11 @@ public class JobGraphBuilder {
private Map<String, Long> iterationWaitTime;
private Map<String, Map<String, OperatorState<?>>> operatorStates;
private Map<String, InputFormat<String, ?>> inputFormatList;
- private Map<String, List<String>> chainedVertices;
- private Map<String, String> lastInChains;
+ private Map<String, Map<String, StreamConfig>> chainedConfigs;
+ private Map<String, StreamConfig> vertexConfigs;
private Set<String> sources;
- private Set<String> builtVertices;
+ private Set<String> builtNodes;
/**
* Creates an new {@link JobGraph} with the given name. A JobGraph is a DAG
@@ -129,11 +129,11 @@ public class JobGraphBuilder {
iterationWaitTime = new HashMap<String, Long>();
operatorStates = new HashMap<String, Map<String, OperatorState<?>>>();
inputFormatList = new HashMap<String, InputFormat<String, ?>>();
- chainedVertices = new HashMap<String, List<String>>();
- lastInChains = new HashMap<String, String>();
+ chainedConfigs = new HashMap<String, Map<String, StreamConfig>>();
+ vertexConfigs = new HashMap<String, StreamConfig>();
sources = new HashSet<String>();
- builtVertices = new HashSet<String>();
+ builtNodes = new HashSet<String>();
}
/**
@@ -198,6 +198,8 @@ public class JobGraphBuilder {
addVertex(vertexName, StreamIterationHead.class, null, null, parallelism);
+ chaining = false;
+
iterationIds.put(vertexName, iterationID);
iterationIDtoHeadName.put(iterationID, vertexName);
@@ -297,7 +299,6 @@ public class JobGraphBuilder {
inEdgeList.put(vertexName, new ArrayList<String>());
outPartitioning.put(vertexName, new ArrayList<StreamPartitioner<?>>());
iterationTailCount.put(vertexName, 0);
- lastInChains.put(vertexName, vertexName);
}
private void addTypeSerializers(String vertexName, StreamRecordSerializer<?> in1,
@@ -309,170 +310,175 @@ public class JobGraphBuilder {
typeSerializersOut2.put(vertexName, out2);
}
- /**
- * Creates an {@link AbstractJobVertex} in the {@link JobGraph} and sets its
- * config parameters using the ones set previously.
- *
- * @param vertexName
- * Name for which the vertex will be created.
- */
- private void createVertex(String vertexName) {
+ private List<Tuple2<String, String>> createChain(String startNode, String current) {
- if (!builtVertices.contains(vertexName)) {
- if (!outEdgeList.get(vertexName).isEmpty()) {
+ if (!builtNodes.contains(startNode)) {
- for (String outName : outEdgeList.get(vertexName)) {
- if (isChainable(vertexName, outName)) {
- chainRecursively(vertexName, vertexName, outName);
- } else {
- createVertex(outName);
- }
+ List<Tuple2<String, String>> transitiveOutEdges = new ArrayList<Tuple2<String, String>>();
+ List<String> chainableOutputs = new ArrayList<String>();
+ List<String> nonChainableOutputs = new ArrayList<String>();
+ for (String outName : outEdgeList.get(current)) {
+ if (isChainable(current, outName)) {
+ chainableOutputs.add(outName);
+ } else {
+ nonChainableOutputs.add(outName);
}
+
}
- List<String> chainedNames = chainedVertices.get(vertexName);
- boolean isChained = chainedNames != null;
- int numChained = isChained ? chainedNames.size() : 0;
- String lastInChain = lastInChains.get(vertexName);
-
- // Get vertex attributes
- Class<? extends AbstractInvokable> vertexClass = vertexClasses.get(vertexName);
- StreamInvokable<?, ?> invokableObject = invokableObjects.get(vertexName);
- int parallelism = vertexParallelism.get(vertexName);
- byte[] outputSelector = outputSelectors.get(lastInChain);
- Map<String, OperatorState<?>> state = operatorStates.get(vertexName);
-
- // Create vertex object
- String cname = chainedVertices.get(vertexName) == null ? "" : " => "
- + StringUtils.join(chainedVertices.get(vertexName), " => ");
- AbstractJobVertex vertex = new AbstractJobVertex(vertexName + cname);
-
- this.jobGraph.addVertex(vertex);
-
- vertex.setInvokableClass(vertexClass);
- vertex.setParallelism(parallelism);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Parallelism set: {} for {}", parallelism, vertexName);
+ for (String chainable : chainableOutputs) {
+ transitiveOutEdges.addAll(createChain(startNode, chainable));
}
- // Set vertex config
+ for (String nonChainable : nonChainableOutputs) {
+ transitiveOutEdges.add(new Tuple2<String, String>(current, nonChainable));
+ transitiveOutEdges.addAll(createChain(nonChainable, nonChainable));
+ }
- StreamConfig config = new StreamConfig(vertex.getConfiguration());
+ StreamConfig config = current.equals(startNode) ? createProcessingVertex(startNode)
+ : new StreamConfig(new Configuration());
- config.setBufferTimeout(bufferTimeout.get(lastInChain));
+ setVertexConfig(current, config, chainableOutputs, nonChainableOutputs);
- config.setTypeSerializerIn1(typeSerializersIn1.get(vertexName));
- config.setTypeSerializerIn2(typeSerializersIn2.get(vertexName));
- config.setTypeSerializerOut1(typeSerializersOut1.get(vertexName));
- config.setTypeSerializerOut2(typeSerializersOut2.get(vertexName));
+ if (current.equals(startNode)) {
- config.setUserInvokable(invokableObject);
- config.setOutputSelector(outputSelector);
- config.setOperatorStates(state);
+ config.setChainStart();
+ config.setRecordWriterOrder(transitiveOutEdges);
- if (vertexClass.equals(StreamIterationHead.class)
- || vertexClass.equals(StreamIterationTail.class)) {
- config.setIterationId(iterationIds.get(vertexName));
- config.setIterationWaitTime(iterationWaitTime.get(vertexName));
- }
+ for (Tuple2<String, String> edge : transitiveOutEdges) {
+ connect(startNode, edge);
+ }
- if (inputFormatList.containsKey(vertexName)) {
- vertex.setInputSplitSource(inputFormatList.get(vertexName));
- }
+ vertexConfigs.get(startNode).setTransitiveChainedTaskConfigs(
+ chainedConfigs.get(startNode));
+
+ } else {
- config.setNumberofChainedTasks(numChained);
+ Map<String, StreamConfig> chainedConfs = chainedConfigs.get(startNode);
- for (int i = 0; i < numChained; i++) {
- config.setChainedInvokable(
- (ChainableInvokable<?, ?>) invokableObjects.get(chainedNames.get(i)), i);
- config.setChainedSerializer(typeSerializersIn1.get(chainedNames.get(i)), i);
+ if (chainedConfs == null) {
+ chainedConfigs.put(startNode, new HashMap<String, StreamConfig>());
+ }
+ chainedConfigs.get(startNode).put(current, config);
}
- streamVertices.put(vertexName, vertex);
- builtVertices.add(vertexName);
+ return transitiveOutEdges;
+ } else {
+ return new ArrayList<Tuple2<String, String>>();
}
+ }
+
+ private StreamConfig createProcessingVertex(String vertexName) {
+
+ AbstractJobVertex vertex = new AbstractJobVertex(vertexName);
+
+ this.jobGraph.addVertex(vertex);
+ int parallelism = vertexParallelism.get(vertexName);
+
+ vertex.setInvokableClass(vertexClasses.get(vertexName));
+ vertex.setParallelism(parallelism);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Parallelism set: {} for {}", parallelism, vertexName);
+ }
+
+ if (inputFormatList.containsKey(vertexName)) {
+ vertex.setInputSplitSource(inputFormatList.get(vertexName));
+ }
+
+ streamVertices.put(vertexName, vertex);
+ builtNodes.add(vertexName);
+
+ return new StreamConfig(vertex.getConfiguration());
}
- private void chainRecursively(String chainStart, String current, String next) {
- // We chain the next operator to the start of this chain
- chainTasks(chainStart, next);
- // Now recursively chain the outputs of next (depth first)
- for (String output : outEdgeList.get(next)) {
- if (isChainable(next, output)) {
- // Recursive call
- chainRecursively(chainStart, next, output);
- } else {
- // If not chainable we continue building the jobgraph from there
- createVertex(output);
- }
+ private void setVertexConfig(String vertexName, StreamConfig config,
+ List<String> chainableOutputs, List<String> nonChainableOutputs) {
+
+ StreamInvokable<?, ?> invokableObject = invokableObjects.get(vertexName);
+ byte[] outputSelector = outputSelectors.get(vertexName);
+ Class<? extends AbstractInvokable> vertexClass = vertexClasses.get(vertexName);
+ Map<String, OperatorState<?>> state = operatorStates.get(vertexName);
+
+ config.setVertexName(vertexName);
+
+ config.setBufferTimeout(bufferTimeout.get(vertexName));
+
+ config.setTypeSerializerIn1(typeSerializersIn1.get(vertexName));
+ config.setTypeSerializerIn2(typeSerializersIn2.get(vertexName));
+ config.setTypeSerializerOut1(typeSerializersOut1.get(vertexName));
+ config.setTypeSerializerOut2(typeSerializersOut2.get(vertexName));
+
+ config.setUserInvokable(invokableObject);
+ config.setOutputSelector(outputSelector);
+ config.setOperatorStates(state);
+
+ config.setNumberOfOutputs(nonChainableOutputs.size());
+ config.setOutputs(nonChainableOutputs);
+ config.setChainedOutputs(chainableOutputs);
+
+ if (vertexClass.equals(StreamIterationHead.class)
+ || vertexClass.equals(StreamIterationTail.class)) {
+ config.setIterationId(iterationIds.get(vertexName));
+ config.setIterationWaitTime(iterationWaitTime.get(vertexName));
}
+
+ vertexConfigs.put(vertexName, config);
}
private boolean isChainable(String vertexName, String outName) {
- return outEdgeList.get(vertexName).size() == 1
- && inEdgeList.get(outName).size() == 1
+ return inEdgeList.get(outName).size() == 1
+ && invokableObjects.get(outName) != null
&& outputSelectors.get(vertexName) == null
&& invokableObjects.get(outName).getChainingStrategy() == ChainingStrategy.ALWAYS
&& (invokableObjects.get(vertexName).getChainingStrategy() == ChainingStrategy.HEAD || invokableObjects
.get(vertexName).getChainingStrategy() == ChainingStrategy.ALWAYS)
- && outPartitioning.get(vertexName).get(0).getStrategy() == PartitioningStrategy.FORWARD
+ && outPartitioning.get(vertexName)
+ .get(outEdgeList.get(vertexName).indexOf(outName)).getStrategy() == PartitioningStrategy.FORWARD
&& vertexParallelism.get(vertexName) == vertexParallelism.get(outName) && chaining;
}
- private void chainTasks(String first, String second) {
+ private <T> void connect(String headOfChain, Tuple2<String, String> edge) {
- List<String> chained = chainedVertices.get(first);
- if (chained == null) {
- chained = new ArrayList<String>();
- }
- chained.add(second);
- chainedVertices.put(first, chained);
- lastInChains.put(first, second);
-
- }
+ String upStreamVertexName = edge.f0;
+ String downStreamVertexName = edge.f1;
- /**
- * Connects two vertices with the given names, partitioning and channel type
- *
- * @param upStreamVertexName
- * Name of the upstream vertex, that will emit the values
- * @param downStreamVertexName
- * Name of the downstream vertex, that will receive the values
- * @param partitionerObject
- * The partitioner
- */
- private <T> void connect(String upStreamVertexName, String downStreamVertexName,
- StreamPartitioner<T> partitionerObject) {
+ int outputIndex = outEdgeList.get(upStreamVertexName).indexOf(downStreamVertexName);
- AbstractJobVertex upStreamVertex = streamVertices.get(upStreamVertexName);
+ AbstractJobVertex headVertex = streamVertices.get(headOfChain);
AbstractJobVertex downStreamVertex = streamVertices.get(downStreamVertexName);
- StreamConfig config = new StreamConfig(upStreamVertex.getConfiguration());
+ StreamConfig downStreamConfig = new StreamConfig(downStreamVertex.getConfiguration());
+ StreamConfig upStreamConfig = new StreamConfig(headVertex.getConfiguration());
+
+ List<Integer> outEdgeIndexList = outEdgeIndex.get(upStreamVertexName);
+ int numOfInputs = downStreamConfig.getNumberOfInputs();
+
+ downStreamConfig.setInputIndex(numOfInputs++, outEdgeIndexList.get(outputIndex));
+ downStreamConfig.setNumberOfInputs(numOfInputs);
+
+ StreamPartitioner<?> partitionerObject = outPartitioning.get(upStreamVertexName).get(
+ outputIndex);
+
+ upStreamConfig.setPartitioner(downStreamVertexName, partitionerObject);
if (partitionerObject.getStrategy() == PartitioningStrategy.FORWARD) {
- downStreamVertex
- .connectNewDataSetAsInput(upStreamVertex, DistributionPattern.POINTWISE);
+ downStreamVertex.connectNewDataSetAsInput(headVertex, DistributionPattern.POINTWISE);
} else {
- downStreamVertex.connectNewDataSetAsInput(upStreamVertex,
- DistributionPattern.ALL_TO_ALL);
+ downStreamVertex.connectNewDataSetAsInput(headVertex, DistributionPattern.ALL_TO_ALL);
}
if (LOG.isDebugEnabled()) {
LOG.debug("CONNECTED: {} - {} -> {}", partitionerObject.getClass().getSimpleName(),
- upStreamVertexName, downStreamVertexName);
+ headOfChain, downStreamVertexName);
}
- int outputIndex = upStreamVertex.getNumberOfProducedIntermediateDataSets() - 1;
-
- config.setOutputName(outputIndex, outEdgeNames.get(lastInChains.get(upStreamVertexName))
+ upStreamConfig.setOutputNames(downStreamVertexName, outEdgeNames.get(upStreamVertexName)
.get(outputIndex));
- config.setSelectAll(outputIndex, outEdgeSelectAll.get(lastInChains.get(upStreamVertexName))
+ upStreamConfig.setSelectAll(downStreamVertexName, outEdgeSelectAll.get(upStreamVertexName)
.get(outputIndex));
- config.setPartitioner(outputIndex, partitionerObject);
- config.setNumberOfOutputChannels(outputIndex, vertexParallelism.get(downStreamVertexName));
}
/**
@@ -630,27 +636,6 @@ public class JobGraphBuilder {
}
/**
- * Writes number of inputs into each JobVertex's config
- */
- private void setNumberOfJobInputs() {
- for (AbstractJobVertex vertex : streamVertices.values()) {
- (new StreamConfig(vertex.getConfiguration())).setNumberOfInputs(vertex
- .getNumberOfInputs());
- }
- }
-
- /**
- * Writes the number of outputs and output channels into each JobVertex's
- * config
- */
- private void setNumberOfJobOutputs() {
- for (AbstractJobVertex vertex : streamVertices.values()) {
- (new StreamConfig(vertex.getConfiguration())).setNumberOfOutputs(vertex
- .getNumberOfProducedIntermediateDataSets());
- }
- }
-
- /**
* Gets the assembled {@link JobGraph} and adds a default name for it.
*/
public JobGraph getJobGraph() {
@@ -677,33 +662,10 @@ public class JobGraphBuilder {
private void buildJobGraph() {
for (String sourceName : sources) {
- createVertex(sourceName);
- }
-
- for (String upStreamVertexName : builtVertices) {
- int i = 0;
-
- List<Integer> outEdgeTypeList = outEdgeIndex.get(lastInChains.get(upStreamVertexName));
-
- for (String downStreamVertexName : outEdgeList
- .get(lastInChains.get(upStreamVertexName))) {
- StreamConfig downStreamVertexConfig = new StreamConfig(streamVertices.get(
- downStreamVertexName).getConfiguration());
-
- int inputNumber = downStreamVertexConfig.getNumberOfInputs();
-
- downStreamVertexConfig.setInputIndex(inputNumber++, outEdgeTypeList.get(i));
- downStreamVertexConfig.setNumberOfInputs(inputNumber);
-
- connect(upStreamVertexName, downStreamVertexName,
- outPartitioning.get(lastInChains.get(upStreamVertexName)).get(i));
- i++;
- }
+ createChain(sourceName, sourceName);
}
setSlotSharing();
- setNumberOfJobInputs();
- setNumberOfJobOutputs();
}
public void setChaining(boolean chaining) {
http://git-wip-us.apache.org/repos/asf/flink/blob/7dbb55ec/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
index ada3aae..6fffaa6 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
@@ -17,16 +17,16 @@
package org.apache.flink.streaming.api;
-import java.io.IOException;
import java.io.Serializable;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang3.SerializationException;
import org.apache.commons.lang3.SerializationUtils;
+import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.collector.OutputSelector;
-import org.apache.flink.streaming.api.invokable.ChainableInvokable;
import org.apache.flink.streaming.api.invokable.StreamInvokable;
import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
import org.apache.flink.streaming.api.streamvertex.StreamVertexException;
@@ -35,17 +35,20 @@ import org.apache.flink.streaming.partitioner.StreamPartitioner;
import org.apache.flink.streaming.state.OperatorState;
import org.apache.flink.util.InstantiationUtil;
-public class StreamConfig {
+public class StreamConfig implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
private static final String INPUT_TYPE = "inputType_";
private static final String NUMBER_OF_OUTPUTS = "numberOfOutputs";
private static final String NUMBER_OF_INPUTS = "numberOfInputs";
- private static final String NUMBER_OF_CHAINED_TASKS = "numOfChained";
- private static final String CHAINED_IN_SERIALIZER = "chainedSerializer_";
- private static final String CHAINED_INVOKABLE = "chainedInvokable_";
+ private static final String CHAINED_OUTPUTS = "chainedOutputs";
+ private static final String CHAINED_TASK_CONFIG = "chainedTaskConfig_";
+ private static final String IS_CHAINED_VERTEX = "isChainedSubtask";
private static final String OUTPUT_NAME = "outputName_";
private static final String OUTPUT_SELECT_ALL = "outputSelectAll_";
private static final String PARTITIONER_OBJECT = "partitionerObject_";
- private static final String NUMBER_OF_OUTPUT_CHANNELS = "numOfOutputs_";
+ private static final String VERTEX_NAME = "vertexName";
private static final String ITERATION_ID = "iteration-id";
private static final String OUTPUT_SELECTOR = "outputSelector";
private static final String DIRECTED_EMIT = "directedEmit";
@@ -58,6 +61,8 @@ public class StreamConfig {
private static final String TYPE_SERIALIZER_OUT_1 = "typeSerializer_out_1";
private static final String TYPE_SERIALIZER_OUT_2 = "typeSerializer_out_2";
private static final String ITERATON_WAIT = "iterationWait";
+ private static final String OUTPUTS = "outVertexNames";
+ private static final String RW_ORDER = "rwOrder";
// DEFAULT VALUES
@@ -75,6 +80,14 @@ public class StreamConfig {
return config;
}
+ public void setVertexName(String vertexName) {
+ config.setString(VERTEX_NAME, vertexName);
+ }
+
+ public String getTaskName() {
+ return config.getString(VERTEX_NAME, "Missing");
+ }
+
public void setTypeSerializerIn1(StreamRecordSerializer<?> serializer) {
setTypeSerializer(TYPE_SERIALIZER_IN_1, serializer);
}
@@ -206,25 +219,21 @@ public class StreamConfig {
return config.getLong(ITERATON_WAIT, 0);
}
- public void setNumberOfOutputChannels(int outputIndex, Integer numberOfOutputChannels) {
- config.setInteger(NUMBER_OF_OUTPUT_CHANNELS + outputIndex, numberOfOutputChannels);
- }
+ public <T> void setPartitioner(String output, StreamPartitioner<T> partitionerObject) {
- public int getNumberOfOutputChannels(int outputIndex) {
- return config.getInteger(NUMBER_OF_OUTPUT_CHANNELS + outputIndex, 0);
- }
-
- public <T> void setPartitioner(int outputIndex, StreamPartitioner<T> partitionerObject) {
-
- config.setBytes(PARTITIONER_OBJECT + outputIndex,
+ config.setBytes(PARTITIONER_OBJECT + output,
SerializationUtils.serialize(partitionerObject));
}
- public <T> StreamPartitioner<T> getPartitioner(ClassLoader cl, int outputIndex)
- throws ClassNotFoundException, IOException {
- @SuppressWarnings("unchecked")
- StreamPartitioner<T> partitioner = (StreamPartitioner<T>) InstantiationUtil
- .readObjectFromConfig(this.config, PARTITIONER_OBJECT + outputIndex, cl);
+ @SuppressWarnings("unchecked")
+ public <T> StreamPartitioner<T> getPartitioner(ClassLoader cl, String output) {
+ StreamPartitioner<T> partitioner = null;
+ try {
+ partitioner = (StreamPartitioner<T>) InstantiationUtil.readObjectFromConfig(
+ this.config, PARTITIONER_OBJECT + output, cl);
+ } catch (Exception e) {
+ throw new RuntimeException("Partitioner could not be instantiated.");
+ }
if (partitioner != null) {
return partitioner;
} else {
@@ -232,27 +241,27 @@ public class StreamConfig {
}
}
- public void setSelectAll(int outputIndex, Boolean selectAll) {
+ public void setSelectAll(String output, Boolean selectAll) {
if (selectAll != null) {
- config.setBoolean(OUTPUT_SELECT_ALL + outputIndex, selectAll);
+ config.setBoolean(OUTPUT_SELECT_ALL + output, selectAll);
}
}
- public boolean isSelectAll(int outputIndex) {
- return config.getBoolean(OUTPUT_SELECT_ALL + outputIndex, false);
+ public boolean isSelectAll(String output) {
+ return config.getBoolean(OUTPUT_SELECT_ALL + output, true);
}
- public void setOutputName(int outputIndex, List<String> outputName) {
+ public void setOutputNames(String output, List<String> outputName) {
if (outputName != null) {
- config.setBytes(OUTPUT_NAME + outputIndex,
+ config.setBytes(OUTPUT_NAME + output,
SerializationUtils.serialize((Serializable) outputName));
}
}
@SuppressWarnings("unchecked")
- public List<String> getOutputNames(int outputIndex) {
- return (List<String>) SerializationUtils.deserialize(config.getBytes(OUTPUT_NAME
- + outputIndex, null));
+ public List<String> getOutputNames(String output) {
+ return (List<String>) SerializationUtils.deserialize(config.getBytes(OUTPUT_NAME + output,
+ null));
}
public void setNumberOfInputs(int numberOfInputs) {
@@ -271,6 +280,38 @@ public class StreamConfig {
return config.getInteger(NUMBER_OF_OUTPUTS, 0);
}
+ public void setOutputs(List<String> outputVertexNames) {
+ config.setBytes(OUTPUTS, SerializationUtils.serialize((Serializable) outputVertexNames));
+ }
+
+ @SuppressWarnings("unchecked")
+ public List<String> getOutputs(ClassLoader cl) {
+ try {
+ return (List<String>) InstantiationUtil.readObjectFromConfig(this.config, OUTPUTS, cl);
+ } catch (Exception e) {
+ throw new RuntimeException("Could not instantiate outputs.");
+ }
+ }
+
+ public void setRecordWriterOrder(List<Tuple2<String, String>> outEdgeList) {
+
+ List<String> outVertices = new ArrayList<String>();
+ for (Tuple2<String, String> edge : outEdgeList) {
+ outVertices.add(edge.f1);
+ }
+
+ config.setBytes(RW_ORDER, SerializationUtils.serialize((Serializable) outVertices));
+ }
+
+ @SuppressWarnings("unchecked")
+ public List<String> getRecordWriterOrder(ClassLoader cl) {
+ try {
+ return (List<String>) InstantiationUtil.readObjectFromConfig(this.config, RW_ORDER, cl);
+ } catch (Exception e) {
+ throw new RuntimeException("Could not instantiate outputs.");
+ }
+ }
+
public void setInputIndex(int inputNumber, Integer inputTypeNumber) {
config.setInteger(INPUT_TYPE + inputNumber++, inputTypeNumber);
}
@@ -293,40 +334,77 @@ public class StreamConfig {
}
}
- public int getNumberofChainedTasks() {
- return config.getInteger(NUMBER_OF_CHAINED_TASKS, 0);
+ public void setChainedOutputs(List<String> chainedOutputs) {
+ config.setBytes(CHAINED_OUTPUTS,
+ SerializationUtils.serialize((Serializable) chainedOutputs));
}
- public void setNumberofChainedTasks(int n) {
- config.setInteger(NUMBER_OF_CHAINED_TASKS, n);
- }
-
- public ChainableInvokable<?, ?> getChainedInvokable(int chainedTaskIndex, ClassLoader cl) {
+ @SuppressWarnings("unchecked")
+ public List<String> getChainedOutputs(ClassLoader cl) {
try {
- return (ChainableInvokable<?, ?>) InstantiationUtil.readObjectFromConfig(this.config,
- CHAINED_INVOKABLE + chainedTaskIndex, cl);
+ return (List<String>) InstantiationUtil.readObjectFromConfig(this.config,
+ CHAINED_OUTPUTS, cl);
} catch (Exception e) {
- throw new RuntimeException("Could not instantiate invokable.");
+ throw new RuntimeException("Could not instantiate chained outputs.");
}
}
- public StreamRecordSerializer<?> getChainedInSerializer(int chainedTaskIndex, ClassLoader cl) {
+ public void setTransitiveChainedTaskConfigs(Map<String, StreamConfig> chainedTaskConfigs) {
+ config.setBytes(CHAINED_TASK_CONFIG,
+ SerializationUtils.serialize((Serializable) chainedTaskConfigs));
+ }
+
+ @SuppressWarnings("unchecked")
+ public Map<String, StreamConfig> getTransitiveChainedTaskConfigs(ClassLoader cl) {
try {
- return (StreamRecordSerializer<?>) InstantiationUtil.readObjectFromConfig(this.config,
- CHAINED_IN_SERIALIZER + chainedTaskIndex, cl);
+
+ return (Map<String, StreamConfig>) InstantiationUtil.readObjectFromConfig(this.config,
+ CHAINED_TASK_CONFIG, cl);
} catch (Exception e) {
- throw new RuntimeException("Could not instantiate serializer.");
+ throw new RuntimeException("Could not instantiate configuration.");
}
}
- public void setChainedSerializer(StreamRecordSerializer<?> typeWrapper, int chainedTaskIndex) {
- config.setBytes(CHAINED_IN_SERIALIZER + chainedTaskIndex,
- SerializationUtils.serialize(typeWrapper));
+ public void setChainStart() {
+ config.setBoolean(IS_CHAINED_VERTEX, true);
}
- public void setChainedInvokable(ChainableInvokable<?, ?> invokable, int chainedTaskIndex) {
- config.setBytes(CHAINED_INVOKABLE + chainedTaskIndex,
- SerializationUtils.serialize(invokable));
+ public boolean isChainStart() {
+ return config.getBoolean(IS_CHAINED_VERTEX, false);
}
+ @Override
+ public String toString() {
+
+ ClassLoader cl = getClass().getClassLoader();
+
+ StringBuilder builder = new StringBuilder();
+ builder.append("\n=======================");
+ builder.append("Stream Config");
+ builder.append("=======================");
+ builder.append("\nTask name: " + getTaskName());
+ builder.append("\nNumber of non-chained inputs: " + getNumberOfInputs());
+ builder.append("\nNumber of non-chained outputs: " + getNumberOfOutputs());
+ builder.append("\nOutput names: " + getOutputs(cl));
+ builder.append("\nPartitioning:");
+ for (String outputname : getOutputs(cl)) {
+ builder.append("\n\t" + outputname + ": "
+ + getPartitioner(cl, outputname).getClass().getSimpleName());
+ }
+
+ builder.append("\nChained subtasks: " + getChainedOutputs(cl));
+
+ try {
+ builder.append("\nInvokable: " + getUserInvokable(cl).getClass().getSimpleName());
+ } catch (Exception e) {
+ builder.append("\nInvokable: Missing");
+ }
+ builder.append("\nBuffer timeout: " + getBufferTimeout());
+ if (isChainStart() && getChainedOutputs(cl).size() > 0) {
+ builder.append("\n\n\n---------------------\nChained task configs\n---------------------\n");
+ builder.append(getTransitiveChainedTaskConfigs(cl)).toString();
+ }
+
+ return builder.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/7dbb55ec/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
index b7e57e0..a95973b 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
@@ -30,7 +30,7 @@ public class CollectorWrapper<OUT> implements Collector<OUT> {
this.outputs = new LinkedList<Collector<OUT>>();
}
- public void addChainedOutput(Collector<OUT> output) {
+ public void addCollector(Collector<OUT> output) {
outputs.add(output);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/7dbb55ec/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java
index 4c21564..6fd1b98 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java
@@ -17,11 +17,13 @@
package org.apache.flink.streaming.api.collector;
+import java.io.IOException;
import java.util.List;
import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
import org.apache.flink.runtime.plugable.SerializationDelegate;
import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.apache.flink.streaming.io.StreamRecordWriter;
import org.apache.flink.util.Collector;
public class StreamOutput<OUT> implements Collector<SerializationDelegate<StreamRecord<OUT>>> {
@@ -52,6 +54,15 @@ public class StreamOutput<OUT> implements Collector<SerializationDelegate<Stream
@Override
public void close() {
+ if (output instanceof StreamRecordWriter) {
+ ((StreamRecordWriter<SerializationDelegate<StreamRecord<OUT>>>) output).close();
+ } else {
+ try {
+ output.flush();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
}
public boolean isSelectAll() {
@@ -62,4 +73,8 @@ public class StreamOutput<OUT> implements Collector<SerializationDelegate<Stream
return selectedNames;
}
+ public RecordWriter<SerializationDelegate<StreamRecord<OUT>>> getRecordWriter() {
+ return output;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/flink/blob/7dbb55ec/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutputWrapper.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutputWrapper.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutputWrapper.java
index fa374b1..c3e4c9d 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutputWrapper.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutputWrapper.java
@@ -20,7 +20,6 @@ package org.apache.flink.streaming.api.collector;
import java.util.LinkedList;
import java.util.List;
-import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
import org.apache.flink.runtime.plugable.SerializationDelegate;
import org.apache.flink.streaming.api.streamrecord.StreamRecord;
import org.apache.flink.util.Collector;
@@ -80,11 +79,6 @@ public class StreamOutputWrapper<OUT> implements Collector<OUT> {
outputs.add(output);
}
- protected void addOneOutput(RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output,
- List<String> outputNames, boolean isSelectAllOutput) {
-
- }
-
/**
* Collects and emits a tuple/object to the outputs by reusing a
* StreamRecord object.
http://git-wip-us.apache.org/repos/asf/flink/blob/7dbb55ec/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
index 60a7b14..99f826d 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
@@ -19,13 +19,15 @@ package org.apache.flink.streaming.api.streamvertex;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.LinkedList;
+import java.util.Collection;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
import org.apache.flink.runtime.plugable.SerializationDelegate;
import org.apache.flink.streaming.api.StreamConfig;
+import org.apache.flink.streaming.api.collector.CollectorWrapper;
import org.apache.flink.streaming.api.collector.DirectedOutputWrapper;
import org.apache.flink.streaming.api.collector.OutputSelector;
import org.apache.flink.streaming.api.collector.StreamOutput;
@@ -45,83 +47,142 @@ public class OutputHandler<OUT> {
private StreamVertex<?, OUT> vertex;
private StreamConfig configuration;
-
- private List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> outputs;
+ private ClassLoader cl;
private Collector<OUT> outerCollector;
- TypeInformation<OUT> outTypeInfo = null;
- StreamRecordSerializer<OUT> outSerializer = null;
- SerializationDelegate<StreamRecord<OUT>> outSerializationDelegate = null;
-
public List<ChainableInvokable<?, ?>> chainedInvokables;
- private int numberOfChainedTasks;
+ private Map<String, StreamOutput<?>> outputMap;
+ private Map<String, StreamConfig> chainedConfigs;
+ private List<String> recordWriterOrder;
public OutputHandler(StreamVertex<?, OUT> vertex) {
+
+ // Initialize some fields
this.vertex = vertex;
- this.outputs = new LinkedList<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>>();
this.configuration = new StreamConfig(vertex.getTaskConfiguration());
this.chainedInvokables = new ArrayList<ChainableInvokable<?, ?>>();
- this.numberOfChainedTasks = configuration.getNumberofChainedTasks();
+ this.outputMap = new HashMap<String, StreamOutput<?>>();
+ this.cl = vertex.getUserCodeClassLoader();
+
+ // We read the chained configs, and the order of record writer
+ // registrations by outputname
+ this.chainedConfigs = configuration.getTransitiveChainedTaskConfigs(cl);
+ this.recordWriterOrder = configuration.getRecordWriterOrder(cl);
+
+ // For the network outputs of the chain head we create the stream
+ // outputs
+ for (String outName : configuration.getOutputs(cl)) {
+ StreamOutput<?> streamOutput = createStreamOutput(outName, configuration);
+ outputMap.put(outName, streamOutput);
+ }
- this.outerCollector = createChainedCollector(0);
+ // If we have chained tasks we iterate through them and create the
+ // stream outputs for the network outputs
+ if (chainedConfigs != null) {
+ for (StreamConfig config : chainedConfigs.values()) {
+ for (String outName : config.getOutputs(cl)) {
+ StreamOutput<?> streamOutput = createStreamOutput(outName, config);
+ outputMap.put(outName, streamOutput);
+ }
+ }
+ }
+
+ // We create the outer collector that will be passed to the first task
+ // in the chain
+ this.outerCollector = createChainedCollector(configuration);
}
- public List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> getOutputs() {
- return outputs;
+ public Collection<StreamOutput<?>> getOutputs() {
+ return outputMap.values();
}
- // We create the outer collector by nesting the chainable invokables into
- // each other
+ /**
+ * This method builds up a nested collector which encapsulates all the
+ * chained operators and their network output. The result of this recursive
+ * call will be passed as collector to the first invokable in the chain.
+ *
+ * @param chainedTaskConfig
+ * The configuration of the starting operator of the chain, we
+ * use this paramater to recursively build the whole chain
+ * @return Returns the collector for the chain starting from the given
+ * config
+ */
@SuppressWarnings({ "unchecked", "rawtypes" })
- private Collector<OUT> createChainedCollector(int chainedTaskIndex) {
+ private Collector<OUT> createChainedCollector(StreamConfig chainedTaskConfig) {
- if (numberOfChainedTasks == chainedTaskIndex) {
- // At the end of the chain we create the collector that sends data
- // to the recordwriters
- return createNetworkCollector();
- } else {
+ // We create a wrapper that will encapsulate the chained operators and
+ // network outputs
+ CollectorWrapper<OUT> wrapper = new CollectorWrapper<OUT>();
- ChainableInvokable chainableInvokable = configuration.getChainedInvokable(
- chainedTaskIndex, vertex.getUserCodeClassLoader());
+ // If the task has network outputs we create a collector for those and
+ // pass
+ // it to the wrapper
+ if (chainedTaskConfig.getNumberOfOutputs() > 0) {
+ wrapper.addCollector((Collector<OUT>) createNetworkCollector(chainedTaskConfig));
+ }
- // The nesting is done by calling this method recursively when
- // passing the collector to the invokable
- chainableInvokable.setup(
- createChainedCollector(chainedTaskIndex + 1),
- configuration.getChainedInSerializer(chainedTaskIndex,
- vertex.getUserCodeClassLoader()));
+ // If the task has chained outputs we create a chained collector for
+ // each of them and pass it to the wrapper
+ for (String output : chainedTaskConfig.getChainedOutputs(cl)) {
+ wrapper.addCollector(createChainedCollector(chainedConfigs.get(output)));
+ }
- // We hold a list of the chained invokables for initializaton
- // afterwards
- chainedInvokables.add(chainableInvokable);
+ if (chainedTaskConfig.isChainStart()) {
+ // The current task is the first chained task at this vertex so we
+ // return the wrapper
+ return wrapper;
+ } else {
+ // The current task is a part of the chain so we get the chainable
+ // invokable which will be returned and set it up using the wrapper
+ ChainableInvokable chainableInvokable = chainedTaskConfig.getUserInvokable(vertex
+ .getUserCodeClassLoader());
+ chainableInvokable.setup(wrapper,
+ chainedTaskConfig.getTypeSerializerIn1(vertex.getUserCodeClassLoader()));
+ chainedInvokables.add(chainableInvokable);
return chainableInvokable;
}
}
- private Collector<OUT> createNetworkCollector() {
+ /**
+ * We create the collector for the network outputs of the task represented
+ * by the config using the StreamOutputs that we have set up in the
+ * constructor.
+ *
+ * @param config
+ * The config of the task
+ * @return We return a collector that represents all the network outputs of
+ * this task
+ */
+ @SuppressWarnings("unchecked")
+ private <T> Collector<T> createNetworkCollector(StreamConfig config) {
+
+ StreamRecordSerializer<T> outSerializer = config
+ .getTypeSerializerOut1(vertex.userClassLoader);
+ SerializationDelegate<StreamRecord<T>> outSerializationDelegate = null;
- createOutSerializer();
+ if (outSerializer != null) {
+ outSerializationDelegate = new SerializationDelegate<StreamRecord<T>>(outSerializer);
+ outSerializationDelegate.setInstance(outSerializer.createInstance());
+ }
- StreamOutputWrapper<OUT> collector;
+ StreamOutputWrapper<T> collector;
if (vertex.configuration.isDirectedEmit()) {
- OutputSelector<OUT> outputSelector = vertex.configuration
+ OutputSelector<T> outputSelector = vertex.configuration
.getOutputSelector(vertex.userClassLoader);
- collector = new DirectedOutputWrapper<OUT>(vertex.getInstanceID(),
+ collector = new DirectedOutputWrapper<T>(vertex.getInstanceID(),
outSerializationDelegate, outputSelector);
} else {
- collector = new StreamOutputWrapper<OUT>(vertex.getInstanceID(),
- outSerializationDelegate);
+ collector = new StreamOutputWrapper<T>(vertex.getInstanceID(), outSerializationDelegate);
}
- int numberOfOutputs = configuration.getNumberOfOutputs();
- for (int i = 0; i < numberOfOutputs; i++) {
- collector = (StreamOutputWrapper<OUT>) addStreamOutput(i, collector);
+ for (String output : config.getOutputs(cl)) {
+ collector.addOutput((StreamOutput<T>) outputMap.get(output));
}
return collector;
@@ -131,32 +192,35 @@ public class OutputHandler<OUT> {
return outerCollector;
}
- void createOutSerializer() {
- outSerializer = configuration.getTypeSerializerOut1(vertex.userClassLoader);
- if (outSerializer != null) {
- outSerializationDelegate = new SerializationDelegate<StreamRecord<OUT>>(outSerializer);
- outSerializationDelegate.setInstance(outSerializer.createInstance());
- }
- }
+ /**
+ * We create the StreamOutput for the specific output given by the name, and
+ * the configuration of its source task
+ *
+ * @param name
+ * Name of the output to which the streamoutput will be set up
+ * @param configuration
+ * The config of upStream task
+ * @return
+ */
+ private <T> StreamOutput<T> createStreamOutput(String name, StreamConfig configuration) {
- Collector<OUT> addStreamOutput(int outputNumber, StreamOutputWrapper<OUT> networkCollector) {
+ int outputNumber = recordWriterOrder.indexOf(name);
- StreamPartitioner<OUT> outputPartitioner;
+ StreamPartitioner<T> outputPartitioner;
try {
- outputPartitioner = configuration.getPartitioner(vertex.userClassLoader,
- outputNumber);
+ outputPartitioner = configuration.getPartitioner(vertex.userClassLoader, name);
} catch (Exception e) {
throw new StreamVertexException("Cannot deserialize partitioner for "
- + vertex.getName() + " with " + outputNumber + " outputs", e);
+ + vertex.getName() + " with " + name + " outputs", e);
}
- RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output;
+ RecordWriter<SerializationDelegate<StreamRecord<T>>> output;
long bufferTimeout = configuration.getBufferTimeout();
if (bufferTimeout >= 0) {
- output = new StreamRecordWriter<SerializationDelegate<StreamRecord<OUT>>>(vertex
+ output = new StreamRecordWriter<SerializationDelegate<StreamRecord<T>>>(vertex
.getEnvironment().getWriter(outputNumber), outputPartitioner, bufferTimeout);
if (LOG.isTraceEnabled()) {
@@ -164,7 +228,7 @@ public class OutputHandler<OUT> {
bufferTimeout, vertex.getClass().getSimpleName());
}
} else {
- output = new RecordWriter<SerializationDelegate<StreamRecord<OUT>>>(vertex
+ output = new RecordWriter<SerializationDelegate<StreamRecord<T>>>(vertex
.getEnvironment().getWriter(outputNumber), outputPartitioner);
if (LOG.isTraceEnabled()) {
@@ -172,34 +236,28 @@ public class OutputHandler<OUT> {
}
}
- outputs.add(output);
-
- networkCollector.addOutput(new StreamOutput<OUT>(output, configuration
- .isSelectAll(outputNumber) ? null : configuration.getOutputNames(outputNumber)));
+ StreamOutput<T> streamOutput = new StreamOutput<T>(output,
+ configuration.isSelectAll(name) ? null : configuration.getOutputNames(name));
if (LOG.isTraceEnabled()) {
LOG.trace("Partitioner set: {} with {} outputs for {}", outputPartitioner.getClass()
.getSimpleName(), outputNumber, vertex.getClass().getSimpleName());
}
- return networkCollector;
+ return streamOutput;
}
public void flushOutputs() throws IOException, InterruptedException {
- for (RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output : outputs) {
- if (output instanceof StreamRecordWriter) {
- ((StreamRecordWriter<SerializationDelegate<StreamRecord<OUT>>>) output).close();
- } else {
- output.flush();
- }
+ for (StreamOutput<?> streamOutput : getOutputs()) {
+ streamOutput.close();
}
}
public void invokeUserFunction(String componentTypeName, StreamInvokable<?, OUT> userInvokable)
throws IOException, InterruptedException {
if (LOG.isDebugEnabled()) {
- LOG.debug("{} {} invoked with instance id {}", componentTypeName,
- vertex.getName(), vertex.getInstanceID());
+ LOG.debug("{} {} invoked with instance id {}", componentTypeName, vertex.getName(),
+ vertex.getInstanceID());
}
try {
@@ -210,8 +268,8 @@ public class OutputHandler<OUT> {
}
if (LOG.isDebugEnabled()) {
- LOG.debug("{} {} invoke finished instance id {}", componentTypeName,
- vertex.getName(), vertex.getInstanceID());
+ LOG.debug("{} {} invoke finished instance id {}", componentTypeName, vertex.getName(),
+ vertex.getInstanceID());
}
flushOutputs();
http://git-wip-us.apache.org/repos/asf/flink/blob/7dbb55ec/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationHead.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationHead.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationHead.java
index 43b455e..cba23b8 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationHead.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationHead.java
@@ -17,19 +17,22 @@
package org.apache.flink.streaming.api.streamvertex;
+import java.util.LinkedList;
+import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.TimeUnit;
import org.apache.flink.api.java.tuple.Tuple;
-import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
import org.apache.flink.runtime.plugable.SerializationDelegate;
+import org.apache.flink.streaming.api.collector.StreamOutput;
import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
import org.apache.flink.streaming.io.BlockingQueueBroker;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class StreamIterationHead<OUT extends Tuple> extends StreamVertex<OUT,OUT> {
+public class StreamIterationHead<OUT extends Tuple> extends StreamVertex<OUT, OUT> {
private static final Logger LOG = LoggerFactory.getLogger(StreamIterationHead.class);
@@ -72,6 +75,15 @@ public class StreamIterationHead<OUT extends Tuple> extends StreamVertex<OUT,OUT
}
StreamRecord<OUT> nextRecord;
+ StreamRecordSerializer<OUT> serializer = configuration
+ .getTypeSerializerOut1(userClassLoader);
+ SerializationDelegate<StreamRecord<OUT>> serializationDelegate = new SerializationDelegate<StreamRecord<OUT>>(
+ serializer);
+
+ List<StreamOutput<OUT>> outputs = new LinkedList<StreamOutput<OUT>>();
+ for (StreamOutput<?> output : outputHandler.getOutputs()) {
+ outputs.add((StreamOutput<OUT>) output);
+ }
while (true) {
if (shouldWait) {
@@ -82,10 +94,9 @@ public class StreamIterationHead<OUT extends Tuple> extends StreamVertex<OUT,OUT
if (nextRecord == null) {
break;
}
- for (RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output : outputHandler
- .getOutputs()) {
- outputHandler.outSerializationDelegate.setInstance(nextRecord);
- output.emit(outputHandler.outSerializationDelegate);
+ for (StreamOutput<OUT> output : outputs) {
+ serializationDelegate.setInstance(nextRecord);
+ output.collect(serializationDelegate);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/7dbb55ec/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
index 53b75a0..698b193 100644
--- a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
+++ b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
@@ -46,6 +46,7 @@ import org.apache.flink.streaming.api.function.aggregation.AggregationFunction
import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.AggregationType
import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
import org.apache.flink.api.streaming.scala.ScalaStreamingAggregator
+import org.apache.flink.streaming.api.invokable.StreamInvokable.ChainingStrategy
class DataStream[T](javaStream: JavaStream[T]) {
@@ -78,6 +79,15 @@ class DataStream[T](javaStream: JavaStream[T]) {
" " +
"parallelism.")
}
+
+ def setChainingStrategy(strategy: ChainingStrategy): DataStream[T] = {
+ javaStream match {
+ case ds: SingleOutputStreamOperator[_, _] => ds.setChainingStrategy(strategy)
+ case _ =>
+ throw new UnsupportedOperationException("Only supported for operators.")
+ }
+ this
+ }
/**
* Creates a new DataStream by merging DataStream outputs of
[7/8] flink git commit: [FLINK-1345] [streaming] Added support for
chaining operators with directed outputs
Posted by mb...@apache.org.
[FLINK-1345] [streaming] Added support for chaining operators with directed outputs
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/92947f0d
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/92947f0d
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/92947f0d
Branch: refs/heads/master
Commit: 92947f0d8308afa74b666e11d5ec3af08f0198c1
Parents: e3b608c
Author: Gyula Fora <gy...@apache.org>
Authored: Wed Jan 21 13:41:44 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Wed Jan 21 16:06:34 2015 +0100
----------------------------------------------------------------------
.../flink/streaming/api/StreamConfig.java | 23 ++-
.../apache/flink/streaming/api/StreamGraph.java | 22 ++-
.../api/StreamingJobGraphGenerator.java | 18 +--
.../api/collector/CollectorWrapper.java | 7 +-
.../api/collector/DirectedCollectorWrapper.java | 128 +++++++++++++++++
.../api/collector/DirectedOutputWrapper.java | 144 -------------------
.../streaming/api/collector/StreamOutput.java | 56 ++++----
.../api/collector/StreamOutputWrapper.java | 116 ---------------
.../streaming/api/datastream/DataStream.java | 5 +-
.../api/datastream/SplitDataStream.java | 7 +-
.../environment/StreamExecutionEnvironment.java | 2 +-
.../api/streamvertex/OutputHandler.java | 95 +++++-------
.../api/streamvertex/StreamIterationHead.java | 12 +-
.../api/streamvertex/StreamIterationTail.java | 3 +-
.../api/collector/StreamCollectorTest.java | 96 ++++++-------
15 files changed, 283 insertions(+), 451 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/92947f0d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
index 213a892..94349d7 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
@@ -18,6 +18,7 @@
package org.apache.flink.streaming.api;
import java.io.Serializable;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -46,7 +47,6 @@ public class StreamConfig implements Serializable {
private static final String CHAINED_TASK_CONFIG = "chainedTaskConfig_";
private static final String IS_CHAINED_VERTEX = "isChainedSubtask";
private static final String OUTPUT_NAME = "outputName_";
- private static final String OUTPUT_SELECT_ALL = "outputSelectAll_";
private static final String PARTITIONER_OBJECT = "partitionerObject_";
private static final String VERTEX_NAME = "vertexName";
private static final String ITERATION_ID = "iteration-id";
@@ -241,25 +241,18 @@ public class StreamConfig implements Serializable {
}
}
- public void setSelectAll(String output, Boolean selectAll) {
- if (selectAll != null) {
- config.setBoolean(OUTPUT_SELECT_ALL + output, selectAll);
- }
- }
-
- public boolean isSelectAll(String output) {
- return config.getBoolean(OUTPUT_SELECT_ALL + output, true);
- }
-
- public void setOutputNames(String output, List<String> outputName) {
- if (outputName != null) {
+ public void setSelectedNames(String output, List<String> selected) {
+ if (selected != null) {
+ config.setBytes(OUTPUT_NAME + output,
+ SerializationUtils.serialize((Serializable) selected));
+ } else {
config.setBytes(OUTPUT_NAME + output,
- SerializationUtils.serialize((Serializable) outputName));
+ SerializationUtils.serialize((Serializable) new ArrayList<String>()));
}
}
@SuppressWarnings("unchecked")
- public List<String> getOutputNames(String output) {
+ public List<String> getSelectedNames(String output) {
return (List<String>) SerializationUtils.deserialize(config.getBytes(OUTPUT_NAME + output,
null));
}
http://git-wip-us.apache.org/repos/asf/flink/blob/92947f0d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamGraph.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamGraph.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamGraph.java
index 3dd1bdb..c9ecd55 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamGraph.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamGraph.java
@@ -58,7 +58,6 @@ public class StreamGraph {
private Map<String, List<String>> outEdgeLists;
private Map<String, List<Integer>> outEdgeTypes;
private Map<String, List<List<String>>> selectedNames;
- private Map<String, List<Boolean>> outEdgeSelectAlls;
private Map<String, List<String>> inEdgeLists;
private Map<String, List<StreamPartitioner<?>>> outputPartitioners;
private Map<String, String> operatorNames;
@@ -94,7 +93,6 @@ public class StreamGraph {
outEdgeLists = new HashMap<String, List<String>>();
outEdgeTypes = new HashMap<String, List<Integer>>();
selectedNames = new HashMap<String, List<List<String>>>();
- outEdgeSelectAlls = new HashMap<String, List<Boolean>>();
inEdgeLists = new HashMap<String, List<String>>();
outputPartitioners = new HashMap<String, List<StreamPartitioner<?>>>();
operatorNames = new HashMap<String, String>();
@@ -186,7 +184,7 @@ public class StreamGraph {
setEdge(vertexName, iterationHead,
outputPartitioners.get(inEdgeLists.get(iterationHead).get(0)).get(0), 0,
- new ArrayList<String>(), false);
+ new ArrayList<String>());
iterationTimeouts.put(iterationIDtoHeadName.get(iterationID), waitTime);
@@ -274,7 +272,6 @@ public class StreamGraph {
outEdgeLists.put(vertexName, new ArrayList<String>());
outEdgeTypes.put(vertexName, new ArrayList<Integer>());
selectedNames.put(vertexName, new ArrayList<List<String>>());
- outEdgeSelectAlls.put(vertexName, new ArrayList<Boolean>());
inEdgeLists.put(vertexName, new ArrayList<String>());
outputPartitioners.put(vertexName, new ArrayList<StreamPartitioner<?>>());
iterationTailCount.put(vertexName, 0);
@@ -296,14 +293,12 @@ public class StreamGraph {
* User defined names of the out edge
*/
public void setEdge(String upStreamVertexName, String downStreamVertexName,
- StreamPartitioner<?> partitionerObject, int typeNumber, List<String> outputNames,
- boolean selectAll) {
+ StreamPartitioner<?> partitionerObject, int typeNumber, List<String> outputNames) {
outEdgeLists.get(upStreamVertexName).add(downStreamVertexName);
outEdgeTypes.get(upStreamVertexName).add(typeNumber);
inEdgeLists.get(downStreamVertexName).add(upStreamVertexName);
outputPartitioners.get(upStreamVertexName).add(partitionerObject);
selectedNames.get(upStreamVertexName).add(outputNames);
- outEdgeSelectAlls.get(upStreamVertexName).add(selectAll);
}
private void addTypeSerializers(String vertexName, StreamRecordSerializer<?> in1,
@@ -494,16 +489,15 @@ public class StreamGraph {
return outEdgeTypes.get(vertexName);
}
- public StreamPartitioner<?> getOutPartitioner(String vertexName, int outputIndex) {
- return outputPartitioners.get(vertexName).get(outputIndex);
+ public StreamPartitioner<?> getOutPartitioner(String upStreamVertex, String downStreamVertex) {
+ return outputPartitioners.get(upStreamVertex).get(
+ outEdgeLists.get(upStreamVertex).indexOf(downStreamVertex));
}
- public List<String> getSelectedNames(String vertexName, int outputIndex) {
- return selectedNames.get(vertexName).get(outputIndex);
- }
+ public List<String> getSelectedNames(String upStreamVertex, String downStreamVertex) {
- public Boolean isSelectAll(String vertexName, int outputIndex) {
- return outEdgeSelectAlls.get(vertexName).get(outputIndex);
+ return selectedNames.get(upStreamVertex).get(
+ outEdgeLists.get(upStreamVertex).indexOf(downStreamVertex));
}
public Collection<Integer> getIterationIDs() {
http://git-wip-us.apache.org/repos/asf/flink/blob/92947f0d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamingJobGraphGenerator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamingJobGraphGenerator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamingJobGraphGenerator.java
index 6c0cc20..3b2d135 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamingJobGraphGenerator.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamingJobGraphGenerator.java
@@ -206,6 +206,13 @@ public class StreamingJobGraphGenerator {
config.setIterationWaitTime(streamGraph.getIterationTimeout(vertexName));
}
+ List<String> allOutputs = new ArrayList<String>(chainableOutputs);
+ allOutputs.addAll(nonChainableOutputs);
+
+ for (String output : allOutputs) {
+ config.setSelectedNames(output, streamGraph.getSelectedNames(vertexName, output));
+ }
+
vertexConfigs.put(vertexName, config);
}
@@ -229,7 +236,7 @@ public class StreamingJobGraphGenerator {
downStreamConfig.setNumberOfInputs(numOfInputs);
StreamPartitioner<?> partitioner = streamGraph.getOutPartitioner(upStreamVertexName,
- outputIndex);
+ downStreamVertexName);
upStreamConfig.setPartitioner(downStreamVertexName, partitioner);
@@ -243,11 +250,6 @@ public class StreamingJobGraphGenerator {
LOG.debug("CONNECTED: {} - {} -> {}", partitioner.getClass().getSimpleName(),
headOfChain, downStreamVertexName);
}
-
- upStreamConfig.setOutputNames(downStreamVertexName,
- streamGraph.getSelectedNames(upStreamVertexName, outputIndex));
- upStreamConfig.setSelectAll(downStreamVertexName,
- streamGraph.isSelectAll(upStreamVertexName, outputIndex));
}
private boolean isChainable(String vertexName, String outName) {
@@ -257,12 +259,10 @@ public class StreamingJobGraphGenerator {
return streamGraph.getInEdges(outName).size() == 1
&& outInvokable != null
- && streamGraph.getOutputSelector(vertexName) == null
&& outInvokable.getChainingStrategy() == ChainingStrategy.ALWAYS
&& (headInvokable.getChainingStrategy() == ChainingStrategy.HEAD || headInvokable
.getChainingStrategy() == ChainingStrategy.ALWAYS)
- && streamGraph.getOutPartitioner(vertexName,
- streamGraph.getOutEdges(vertexName).indexOf(outName)).getStrategy() == PartitioningStrategy.FORWARD
+ && streamGraph.getOutPartitioner(vertexName, outName).getStrategy() == PartitioningStrategy.FORWARD
&& streamGraph.getParallelism(vertexName) == streamGraph.getParallelism(outName)
&& streamGraph.chaining;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/92947f0d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
index a95973b..1281bf0 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
@@ -24,14 +24,15 @@ import org.apache.flink.util.Collector;
public class CollectorWrapper<OUT> implements Collector<OUT> {
- List<Collector<OUT>> outputs;
+ private List<Collector<OUT>> outputs;
public CollectorWrapper() {
this.outputs = new LinkedList<Collector<OUT>>();
}
- public void addCollector(Collector<OUT> output) {
- outputs.add(output);
+ @SuppressWarnings("unchecked")
+ public void addCollector(Collector<?> output) {
+ outputs.add((Collector<OUT>) output);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/92947f0d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedCollectorWrapper.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedCollectorWrapper.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedCollectorWrapper.java
new file mode 100755
index 0000000..66fb667
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedCollectorWrapper.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.collector;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.flink.util.Collector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A StreamCollector that uses user defined output names and a user defined
+ * output selector to make directed emits.
+ *
+ * @param <OUT>
+ * Type of the Tuple collected.
+ */
+public class DirectedCollectorWrapper<OUT> extends CollectorWrapper<OUT> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(DirectedCollectorWrapper.class);
+
+ OutputSelector<OUT> outputSelector;
+
+ protected Map<String, List<Collector<OUT>>> outputMap;
+
+ private List<Collector<OUT>> selectAllOutputs;
+ private Set<Collector<OUT>> emitted;
+
+ /**
+ * Creates a new DirectedStreamCollector
+ *
+ * @param outputSelector
+ * User defined {@link OutputSelector}
+ */
+ public DirectedCollectorWrapper(OutputSelector<OUT> outputSelector) {
+ this.outputSelector = outputSelector;
+ this.emitted = new HashSet<Collector<OUT>>();
+ this.selectAllOutputs = new LinkedList<Collector<OUT>>();
+ this.outputMap = new HashMap<String, List<Collector<OUT>>>();
+
+ }
+
+ @Override
+ public void addCollector(Collector<?> output) {
+ addCollector(output, new ArrayList<String>());
+ }
+
+ @SuppressWarnings("unchecked")
+ public void addCollector(Collector<?> output, List<String> selectedNames) {
+
+ if (selectedNames.isEmpty()) {
+ selectAllOutputs.add((Collector<OUT>) output);
+ } else {
+ for (String selectedName : selectedNames) {
+
+ if (!outputMap.containsKey(selectedName)) {
+ outputMap.put(selectedName, new LinkedList<Collector<OUT>>());
+ outputMap.get(selectedName).add((Collector<OUT>) output);
+ } else {
+ if (!outputMap.get(selectedName).contains(output)) {
+ outputMap.get(selectedName).add((Collector<OUT>) output);
+ }
+ }
+
+ }
+ }
+ }
+
+ @Override
+ public void collect(OUT record) {
+ emitted.clear();
+
+ Iterable<String> outputNames = outputSelector.select(record);
+
+ for (Collector<OUT> output : selectAllOutputs) {
+ output.collect(record);
+ emitted.add(output);
+ }
+
+ for (String outputName : outputNames) {
+ List<Collector<OUT>> outputList = outputMap.get(outputName);
+ if (outputList == null) {
+ if (LOG.isErrorEnabled()) {
+ String format = String.format(
+ "Cannot emit because no output is selected with the name: %s",
+ outputName);
+ LOG.error(format);
+
+ }
+ } else {
+ for (Collector<OUT> output : outputList) {
+ if (!emitted.contains(output)) {
+ output.collect(record);
+ emitted.add(output);
+ }
+ }
+
+ }
+
+ }
+ }
+
+ @Override
+ public void close() {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/92947f0d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedOutputWrapper.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedOutputWrapper.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedOutputWrapper.java
deleted file mode 100755
index 6ff03c4..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/DirectedOutputWrapper.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.collector;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.flink.runtime.plugable.SerializationDelegate;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.apache.flink.util.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * A StreamCollector that uses user defined output names and a user defined
- * output selector to make directed emits.
- *
- * @param <OUT>
- * Type of the Tuple collected.
- */
-public class DirectedOutputWrapper<OUT> extends StreamOutputWrapper<OUT> {
-
- private static final Logger LOG = LoggerFactory.getLogger(DirectedOutputWrapper.class);
-
- OutputSelector<OUT> outputSelector;
-
- protected Map<String, List<StreamOutput<OUT>>> outputMap;
-
- private List<StreamOutput<OUT>> selectAllOutputs;
- private Set<StreamOutput<OUT>> emitted;
-
- /**
- * Creates a new DirectedStreamCollector
- *
- * @param channelID
- * Channel ID of the Task
- * @param serializationDelegate
- * Serialization delegate used for serialization
- * @param outputSelector
- * User defined {@link OutputSelector}
- */
- public DirectedOutputWrapper(int channelID,
- SerializationDelegate<StreamRecord<OUT>> serializationDelegate,
- OutputSelector<OUT> outputSelector) {
- super(channelID, serializationDelegate);
- this.outputSelector = outputSelector;
- this.emitted = new HashSet<StreamOutput<OUT>>();
- this.selectAllOutputs = new LinkedList<StreamOutput<OUT>>();
- this.outputMap = new HashMap<String, List<StreamOutput<OUT>>>();
-
- }
-
- @Override
- public void addOutput(StreamOutput<OUT> output) {
-
- super.addOutput(output);
-
- if (output.isSelectAll()) {
- selectAllOutputs.add(output);
- } else {
- for (String selectedName : output.getSelectedNames()) {
- if (selectedName != null) {
- if (!outputMap.containsKey(selectedName)) {
- outputMap.put(selectedName, new LinkedList<StreamOutput<OUT>>());
- outputMap.get(selectedName).add(output);
- } else {
- if (!outputMap.get(selectedName).contains(output)) {
- outputMap.get(selectedName).add(output);
- }
- }
-
- }
- }
- }
- }
-
- @Override
- protected void emit() {
- Iterable<String> outputNames = outputSelector.select(serializationDelegate.getInstance()
- .getObject());
- emitted.clear();
-
- for (StreamOutput<OUT> output : selectAllOutputs) {
- try {
- output.collect(serializationDelegate);
- } catch (Exception e) {
- if (LOG.isErrorEnabled()) {
- LOG.error("Emit to {} failed due to: {}", output,
- StringUtils.stringifyException(e));
- }
- }
- }
-
- for (String outputName : outputNames) {
- List<StreamOutput<OUT>> outputList = outputMap.get(outputName);
- try {
- if (outputList == null) {
- if (LOG.isErrorEnabled()) {
- String format = String.format(
- "Cannot emit because no output is selected with the name: %s",
- outputName);
- LOG.error(format);
-
- }
- } else {
-
- for (StreamOutput<OUT> output : outputList) {
- if (!emitted.contains(output)) {
- output.collect(serializationDelegate);
- emitted.add(output);
- }
- }
-
- }
-
- } catch (Exception e) {
- if (LOG.isErrorEnabled()) {
- LOG.error("Emit to {} failed due to: {}", outputName,
- StringUtils.stringifyException(e));
- }
- }
-
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/92947f0d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java
index 6fd1b98..4551c5a 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutput.java
@@ -18,37 +18,55 @@
package org.apache.flink.streaming.api.collector;
import java.io.IOException;
-import java.util.List;
import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
import org.apache.flink.runtime.plugable.SerializationDelegate;
import org.apache.flink.streaming.api.streamrecord.StreamRecord;
import org.apache.flink.streaming.io.StreamRecordWriter;
import org.apache.flink.util.Collector;
+import org.apache.flink.util.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-public class StreamOutput<OUT> implements Collector<SerializationDelegate<StreamRecord<OUT>>> {
+public class StreamOutput<OUT> implements Collector<OUT> {
- private RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output;
+ private static final Logger LOG = LoggerFactory.getLogger(StreamOutput.class);
- private List<String> selectedNames;
- private boolean selectAll = true;
+ private RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output;
+ private SerializationDelegate<StreamRecord<OUT>> serializationDelegate;
+ private StreamRecord<OUT> streamRecord;
+ private int channelID;
public StreamOutput(RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output,
- List<String> selectedNames) {
+ int channelID, SerializationDelegate<StreamRecord<OUT>> serializationDelegate) {
- this.output = output;
+ this.serializationDelegate = serializationDelegate;
- if (selectedNames != null) {
- this.selectedNames = selectedNames;
- selectAll = false;
+ if (serializationDelegate != null) {
+ this.streamRecord = serializationDelegate.getInstance();
+ } else {
+ throw new RuntimeException("Serializer cannot be null");
}
+ this.channelID = channelID;
+ this.output = output;
}
- public void collect(SerializationDelegate<StreamRecord<OUT>> record) {
+ public RecordWriter<SerializationDelegate<StreamRecord<OUT>>> getRecordWriter() {
+ return output;
+ }
+
+ @Override
+ public void collect(OUT record) {
+ streamRecord.setObject(record);
+ streamRecord.newId(channelID);
+ serializationDelegate.setInstance(streamRecord);
+
try {
- output.emit(record);
+ output.emit(serializationDelegate);
} catch (Exception e) {
- throw new RuntimeException("Could not emit record: " + record.getInstance());
+ if (LOG.isErrorEnabled()) {
+ LOG.error("Emit failed due to: {}", StringUtils.stringifyException(e));
+ }
}
}
@@ -65,16 +83,4 @@ public class StreamOutput<OUT> implements Collector<SerializationDelegate<Stream
}
}
- public boolean isSelectAll() {
- return selectAll;
- }
-
- public List<String> getSelectedNames() {
- return selectedNames;
- }
-
- public RecordWriter<SerializationDelegate<StreamRecord<OUT>>> getRecordWriter() {
- return output;
- }
-
}
http://git-wip-us.apache.org/repos/asf/flink/blob/92947f0d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutputWrapper.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutputWrapper.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutputWrapper.java
deleted file mode 100755
index c3e4c9d..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/StreamOutputWrapper.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.collector;
-
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.flink.runtime.plugable.SerializationDelegate;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.apache.flink.util.Collector;
-import org.apache.flink.util.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Collector for tuples in Apache Flink stream processing. The collected values
- * will be wrapped with ID in a {@link StreamRecord} and then emitted to the
- * outputs.
- *
- * @param <OUT>
- * Type of the Tuples/Objects collected.
- */
-public class StreamOutputWrapper<OUT> implements Collector<OUT> {
-
- private static final Logger LOG = LoggerFactory.getLogger(StreamOutputWrapper.class);
-
- protected StreamRecord<OUT> streamRecord;
- protected int channelID;
- protected List<StreamOutput<OUT>> outputs;
- protected SerializationDelegate<StreamRecord<OUT>> serializationDelegate;
-
- /**
- * Creates a new StreamCollector
- *
- * @param channelID
- * Channel ID of the Task
- * @param serializationDelegate
- * Serialization delegate used for serialization
- */
- public StreamOutputWrapper(int channelID,
- SerializationDelegate<StreamRecord<OUT>> serializationDelegate) {
- this.serializationDelegate = serializationDelegate;
-
- if (serializationDelegate != null) {
- this.streamRecord = serializationDelegate.getInstance();
- } else {
- this.streamRecord = new StreamRecord<OUT>();
- }
- this.channelID = channelID;
- this.outputs = new LinkedList<StreamOutput<OUT>>();
- }
-
- /**
- * Adds an output with the given user defined name
- *
- * @param output
- * The RecordWriter object representing the output.
- * @param outputNames
- * User defined names of the output.
- * @param isSelectAllOutput
- * Marks whether all the outputs are selected.
- */
- public void addOutput(StreamOutput<OUT> output) {
- outputs.add(output);
- }
-
- /**
- * Collects and emits a tuple/object to the outputs by reusing a
- * StreamRecord object.
- *
- * @param outputObject
- * Object to be collected and emitted.
- */
- @Override
- public void collect(OUT outputObject) {
- streamRecord.setObject(outputObject);
- streamRecord.newId(channelID);
- serializationDelegate.setInstance(streamRecord);
-
- emit();
- }
-
- /**
- * Emits the current streamrecord to the outputs.
- */
- protected void emit() {
- for (StreamOutput<OUT> output : outputs) {
- try {
- output.collect(serializationDelegate);
- } catch (Exception e) {
- if (LOG.isErrorEnabled()) {
- LOG.error("Emit failed due to: {}", StringUtils.stringifyException(e));
- }
- }
- }
- }
-
- @Override
- public void close() {
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/92947f0d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index 8e7d823..0f633ec 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -96,7 +96,6 @@ public class DataStream<OUT> {
protected final String id;
protected int degreeOfParallelism;
protected List<String> userDefinedNames;
- protected boolean selectAll;
protected StreamPartitioner<OUT> partitioner;
@SuppressWarnings("rawtypes")
protected TypeInformation typeInfo;
@@ -127,7 +126,6 @@ public class DataStream<OUT> {
this.degreeOfParallelism = environment.getDegreeOfParallelism();
this.streamGraph = environment.getStreamGraph();
this.userDefinedNames = new ArrayList<String>();
- this.selectAll = true;
this.partitioner = new DistributePartitioner<OUT>(true);
this.typeInfo = typeInfo;
this.mergedStreams = new ArrayList<DataStream<OUT>>();
@@ -145,7 +143,6 @@ public class DataStream<OUT> {
this.id = dataStream.id;
this.degreeOfParallelism = dataStream.degreeOfParallelism;
this.userDefinedNames = new ArrayList<String>(dataStream.userDefinedNames);
- this.selectAll = dataStream.selectAll;
this.partitioner = dataStream.partitioner;
this.streamGraph = dataStream.streamGraph;
this.typeInfo = dataStream.typeInfo;
@@ -1158,7 +1155,7 @@ public class DataStream<OUT> {
protected <X> void connectGraph(DataStream<X> inputStream, String outputID, int typeNumber) {
for (DataStream<X> stream : inputStream.mergedStreams) {
streamGraph.setEdge(stream.getId(), outputID, stream.partitioner, typeNumber,
- inputStream.userDefinedNames, inputStream.selectAll);
+ inputStream.userDefinedNames);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/92947f0d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SplitDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SplitDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SplitDataStream.java
index 1bf4f9c..2b5b7c5 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SplitDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SplitDataStream.java
@@ -49,8 +49,13 @@ public class SplitDataStream<OUT> extends DataStream<OUT> {
}
private DataStream<OUT> selectOutput(String[] outputNames) {
+ for (String outName : outputNames) {
+ if (outName == null) {
+ throw new RuntimeException("Selected names must not be null");
+ }
+ }
+
DataStream<OUT> returnStream = copy();
- returnStream.selectAll = false;
returnStream.userDefinedNames = Arrays.asList(outputNames);
return returnStream;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/92947f0d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
index 51dc0ae..8a9be93 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
@@ -313,7 +313,7 @@ public abstract class StreamExecutionEnvironment {
*/
public DataStreamSource<String> socketTextStream(String hostname, int port, char delimiter) {
return addSource(new SocketTextStreamFunction(hostname, port, delimiter), null,
- "socketStrean");
+ "socketStream");
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/92947f0d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
index 135f742..0d60939 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
@@ -29,10 +29,8 @@ import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
import org.apache.flink.runtime.plugable.SerializationDelegate;
import org.apache.flink.streaming.api.StreamConfig;
import org.apache.flink.streaming.api.collector.CollectorWrapper;
-import org.apache.flink.streaming.api.collector.DirectedOutputWrapper;
-import org.apache.flink.streaming.api.collector.OutputSelector;
+import org.apache.flink.streaming.api.collector.DirectedCollectorWrapper;
import org.apache.flink.streaming.api.collector.StreamOutput;
-import org.apache.flink.streaming.api.collector.StreamOutputWrapper;
import org.apache.flink.streaming.api.invokable.ChainableInvokable;
import org.apache.flink.streaming.api.invokable.StreamInvokable;
import org.apache.flink.streaming.api.streamrecord.StreamRecord;
@@ -70,7 +68,7 @@ public class OutputHandler<OUT> {
// registrations by outputname
this.chainedConfigs = configuration.getTransitiveChainedTaskConfigs(cl);
this.chainedConfigs.put(configuration.getTaskName(), configuration);
-
+
this.outEdgesInOrder = configuration.getOutEdgesInOrder(cl);
// We iterate through all the out edges from this job vertex and create
@@ -105,21 +103,35 @@ public class OutputHandler<OUT> {
@SuppressWarnings({ "unchecked", "rawtypes" })
private Collector<OUT> createChainedCollector(StreamConfig chainedTaskConfig) {
+ boolean isDirectEmit = chainedTaskConfig.isDirectedEmit();
+
// We create a wrapper that will encapsulate the chained operators and
// network outputs
- CollectorWrapper<OUT> wrapper = new CollectorWrapper<OUT>();
+ CollectorWrapper<OUT> wrapper = isDirectEmit ? new DirectedCollectorWrapper(
+ chainedTaskConfig.getOutputSelector(cl)) : new CollectorWrapper<OUT>();
+
+ // Create collectors for the network outputs
+ for (String output : chainedTaskConfig.getOutputs(cl)) {
- // If the task has network outputs we create a collector for those and
- // pass
- // it to the wrapper
- if (chainedTaskConfig.getNumberOfOutputs() > 0) {
- wrapper.addCollector((Collector<OUT>) createNetworkCollector(chainedTaskConfig));
+ Collector<?> outCollector = outputMap.get(output);
+
+ if (isDirectEmit) {
+ ((DirectedCollectorWrapper<OUT>) wrapper).addCollector(outCollector,
+ chainedTaskConfig.getSelectedNames(output));
+ } else {
+ wrapper.addCollector(outCollector);
+ }
}
- // If the task has chained outputs we create a chained collector for
- // each of them and pass it to the wrapper
+ // Create collectors for the chained outputs
for (String output : chainedTaskConfig.getChainedOutputs(cl)) {
- wrapper.addCollector(createChainedCollector(chainedConfigs.get(output)));
+ Collector<?> outCollector = createChainedCollector(chainedConfigs.get(output));
+ if (isDirectEmit) {
+ ((DirectedCollectorWrapper<OUT>) wrapper).addCollector(outCollector,
+ chainedTaskConfig.getSelectedNames(output));
+ } else {
+ wrapper.addCollector(outCollector);
+ }
}
if (chainedTaskConfig.isChainStart()) {
@@ -140,47 +152,6 @@ public class OutputHandler<OUT> {
}
- /**
- * We create the collector for the network outputs of the task represented
- * by the config using the StreamOutputs that we have set up in the
- * constructor.
- *
- * @param config
- * The config of the task
- * @return We return a collector that represents all the network outputs of
- * this task
- */
- @SuppressWarnings("unchecked")
- private <T> Collector<T> createNetworkCollector(StreamConfig config) {
-
- StreamRecordSerializer<T> outSerializer = config
- .getTypeSerializerOut1(vertex.userClassLoader);
- SerializationDelegate<StreamRecord<T>> outSerializationDelegate = null;
-
- if (outSerializer != null) {
- outSerializationDelegate = new SerializationDelegate<StreamRecord<T>>(outSerializer);
- outSerializationDelegate.setInstance(outSerializer.createInstance());
- }
-
- StreamOutputWrapper<T> collector;
-
- if (vertex.configuration.isDirectedEmit()) {
- OutputSelector<T> outputSelector = vertex.configuration
- .getOutputSelector(vertex.userClassLoader);
-
- collector = new DirectedOutputWrapper<T>(vertex.getInstanceID(),
- outSerializationDelegate, outputSelector);
- } else {
- collector = new StreamOutputWrapper<T>(vertex.getInstanceID(), outSerializationDelegate);
- }
-
- for (String output : config.getOutputs(cl)) {
- collector.addOutput((StreamOutput<T>) outputMap.get(output));
- }
-
- return collector;
- }
-
public Collector<OUT> getCollector() {
return outerCollector;
}
@@ -197,7 +168,16 @@ public class OutputHandler<OUT> {
*/
private <T> StreamOutput<T> createStreamOutput(String outputVertex, StreamConfig configuration,
int outputIndex) {
-
+
+ StreamRecordSerializer<T> outSerializer = configuration
+ .getTypeSerializerOut1(vertex.userClassLoader);
+ SerializationDelegate<StreamRecord<T>> outSerializationDelegate = null;
+
+ if (outSerializer != null) {
+ outSerializationDelegate = new SerializationDelegate<StreamRecord<T>>(outSerializer);
+ outSerializationDelegate.setInstance(outSerializer.createInstance());
+ }
+
StreamPartitioner<T> outputPartitioner = configuration.getPartitioner(cl, outputVertex);
RecordWriter<SerializationDelegate<StreamRecord<T>>> output;
@@ -221,9 +201,8 @@ public class OutputHandler<OUT> {
}
}
- StreamOutput<T> streamOutput = new StreamOutput<T>(output,
- configuration.isSelectAll(outputVertex) ? null
- : configuration.getOutputNames(outputVertex));
+ StreamOutput<T> streamOutput = new StreamOutput<T>(output, vertex.instanceID,
+ outSerializationDelegate);
if (LOG.isTraceEnabled()) {
LOG.trace("Partitioner set: {} with {} outputs for {}", outputPartitioner.getClass()
http://git-wip-us.apache.org/repos/asf/flink/blob/92947f0d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationHead.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationHead.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationHead.java
index cba23b8..bc89241 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationHead.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationHead.java
@@ -23,16 +23,13 @@ import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.TimeUnit;
-import org.apache.flink.api.java.tuple.Tuple;
-import org.apache.flink.runtime.plugable.SerializationDelegate;
import org.apache.flink.streaming.api.collector.StreamOutput;
import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
import org.apache.flink.streaming.io.BlockingQueueBroker;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class StreamIterationHead<OUT extends Tuple> extends StreamVertex<OUT, OUT> {
+public class StreamIterationHead<OUT> extends StreamVertex<OUT, OUT> {
private static final Logger LOG = LoggerFactory.getLogger(StreamIterationHead.class);
@@ -75,10 +72,6 @@ public class StreamIterationHead<OUT extends Tuple> extends StreamVertex<OUT, OU
}
StreamRecord<OUT> nextRecord;
- StreamRecordSerializer<OUT> serializer = configuration
- .getTypeSerializerOut1(userClassLoader);
- SerializationDelegate<StreamRecord<OUT>> serializationDelegate = new SerializationDelegate<StreamRecord<OUT>>(
- serializer);
List<StreamOutput<OUT>> outputs = new LinkedList<StreamOutput<OUT>>();
for (StreamOutput<?> output : outputHandler.getOutputs()) {
@@ -95,8 +88,7 @@ public class StreamIterationHead<OUT extends Tuple> extends StreamVertex<OUT, OU
break;
}
for (StreamOutput<OUT> output : outputs) {
- serializationDelegate.setInstance(nextRecord);
- output.collect(serializationDelegate);
+ output.collect(nextRecord.getObject());
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/92947f0d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationTail.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationTail.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationTail.java
index 1883c06..b3ecdf8 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationTail.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamIterationTail.java
@@ -20,14 +20,13 @@ package org.apache.flink.streaming.api.streamvertex;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.TimeUnit;
-import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.streaming.api.streamrecord.StreamRecord;
import org.apache.flink.streaming.io.BlockingQueueBroker;
import org.apache.flink.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class StreamIterationTail<IN extends Tuple> extends StreamVertex<IN, IN> {
+public class StreamIterationTail<IN> extends StreamVertex<IN, IN> {
private static final Logger LOG = LoggerFactory.getLogger(StreamIterationTail.class);
http://git-wip-us.apache.org/repos/asf/flink/blob/92947f0d/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/StreamCollectorTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/StreamCollectorTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/StreamCollectorTest.java
index 0bc3d7d..49b3bf8 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/StreamCollectorTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/collector/StreamCollectorTest.java
@@ -1,54 +1,52 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
* limitations under the License.
- */
-
-package org.apache.flink.streaming.api.collector;
-
-import static org.junit.Assert.assertArrayEquals;
+ */
-import java.util.ArrayList;
+package org.apache.flink.streaming.api.collector;
-import org.apache.flink.api.java.tuple.Tuple1;
-import org.apache.flink.runtime.plugable.SerializationDelegate;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import static org.junit.Assert.assertArrayEquals;
+
+import org.apache.flink.api.java.tuple.Tuple1;
+import org.apache.flink.runtime.plugable.SerializationDelegate;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
import org.apache.flink.streaming.api.streamvertex.MockRecordWriter;
-import org.apache.flink.streaming.util.MockRecordWriterFactory;
-import org.junit.Test;
-
-public class StreamCollectorTest {
-
- @Test
- public void testCollect() {
- MockRecordWriter recWriter = MockRecordWriterFactory.create();
- SerializationDelegate<StreamRecord<Tuple1<Integer>>> sd = new SerializationDelegate<StreamRecord<Tuple1<Integer>>>(
- null);
- sd.setInstance(new StreamRecord<Tuple1<Integer>>().setObject(new Tuple1<Integer>()));
-
- StreamOutputWrapper<Tuple1<Integer>> collector = new StreamOutputWrapper<Tuple1<Integer>>(2, sd);
- collector.addOutput(new StreamOutput<Tuple1<Integer>>(recWriter, new ArrayList<String>()));
- collector.collect(new Tuple1<Integer>(3));
- collector.collect(new Tuple1<Integer>(4));
- collector.collect(new Tuple1<Integer>(5));
- collector.collect(new Tuple1<Integer>(6));
-
- assertArrayEquals(new Integer[] { 3, 4, 5, 6 }, recWriter.emittedRecords.toArray());
- }
-
- @Test
- public void testClose() {
- }
-
-}
+import org.apache.flink.streaming.util.MockRecordWriterFactory;
+import org.apache.flink.util.Collector;
+import org.junit.Test;
+
+public class StreamCollectorTest {
+
+ @Test
+ public void testCollect() {
+ MockRecordWriter recWriter = MockRecordWriterFactory.create();
+ SerializationDelegate<StreamRecord<Tuple1<Integer>>> sd = new SerializationDelegate<StreamRecord<Tuple1<Integer>>>(
+ null);
+ sd.setInstance(new StreamRecord<Tuple1<Integer>>().setObject(new Tuple1<Integer>()));
+
+ Collector<Tuple1<Integer>> collector = new StreamOutput<Tuple1<Integer>>(recWriter, 2, sd);
+ collector.collect(new Tuple1<Integer>(3));
+ collector.collect(new Tuple1<Integer>(4));
+ collector.collect(new Tuple1<Integer>(5));
+ collector.collect(new Tuple1<Integer>(6));
+
+ assertArrayEquals(new Integer[] { 3, 4, 5, 6 }, recWriter.emittedRecords.toArray());
+ }
+
+ @Test
+ public void testClose() {
+ }
+
+}
[8/8] flink git commit: [FLINK-1345] [streaming] Chaining refactor +
ChainingStrategy exposed through the API for operators
Posted by mb...@apache.org.
[FLINK-1345] [streaming] Chaining refactor + ChainingStrategy exposed through the API for operators
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/3e30c6f7
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/3e30c6f7
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/3e30c6f7
Branch: refs/heads/master
Commit: 3e30c6f73a12e2f50449a4d0dce452031f9a7317
Parents: 26535c4
Author: Gyula Fora <gy...@apache.org>
Authored: Fri Jan 16 15:11:38 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Wed Jan 21 16:06:34 2015 +0100
----------------------------------------------------------------------
.../flink/streaming/api/JobGraphBuilder.java | 103 ++++++++---------
.../flink/streaming/api/StreamConfig.java | 21 +---
.../api/datastream/ConnectedDataStream.java | 2 +-
.../streaming/api/datastream/DataStream.java | 13 ++-
.../api/datastream/DataStreamSink.java | 5 +-
.../api/datastream/DataStreamSource.java | 5 +-
.../api/datastream/IterativeDataStream.java | 3 +-
.../datastream/SingleOutputStreamOperator.java | 12 +-
.../environment/StreamExecutionEnvironment.java | 9 +-
.../api/invokable/ChainableInvokable.java | 1 +
.../api/invokable/StreamInvokable.java | 20 +++-
.../operator/GroupedReduceInvokable.java | 6 +-
.../operator/GroupedWindowInvokable.java | 1 +
.../api/invokable/operator/WindowInvokable.java | 3 +-
.../api/streamvertex/CoStreamVertex.java | 2 +-
.../api/streamvertex/OutputHandler.java | 110 +++++++++----------
16 files changed, 159 insertions(+), 157 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/3e30c6f7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
index 35e145e..0020d48 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
@@ -36,6 +36,7 @@ import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
import org.apache.flink.streaming.api.collector.OutputSelector;
import org.apache.flink.streaming.api.invokable.ChainableInvokable;
import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.invokable.StreamInvokable.ChainingStrategy;
import org.apache.flink.streaming.api.invokable.operator.co.CoInvokable;
import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
import org.apache.flink.streaming.api.streamvertex.CoStreamVertex;
@@ -64,7 +65,7 @@ public class JobGraphBuilder {
private Map<String, Integer> vertexParallelism;
private Map<String, Long> bufferTimeout;
private Map<String, List<String>> outEdgeList;
- private Map<String, List<Integer>> outEdgeType;
+ private Map<String, List<Integer>> outEdgeIndex;
private Map<String, List<List<String>>> outEdgeNames;
private Map<String, List<Boolean>> outEdgeSelectAll;
private Map<String, List<String>> inEdgeList;
@@ -85,8 +86,7 @@ public class JobGraphBuilder {
private Map<String, Map<String, OperatorState<?>>> operatorStates;
private Map<String, InputFormat<String, ?>> inputFormatList;
private Map<String, List<String>> chainedVertices;
- private Map<String, List<ChainableInvokable<?, ?>>> chainedInvokable;
- private Map<String, List<StreamRecordSerializer<?>>> chainedSerializer;
+ private Map<String, String> lastInChains;
private Set<String> sources;
private Set<String> builtVertices;
@@ -97,11 +97,19 @@ public class JobGraphBuilder {
*/
public JobGraphBuilder() {
+ initGraph();
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("JobGraph created");
+ }
+ }
+
+ public void initGraph() {
streamVertices = new HashMap<String, AbstractJobVertex>();
vertexParallelism = new HashMap<String, Integer>();
bufferTimeout = new HashMap<String, Long>();
outEdgeList = new HashMap<String, List<String>>();
- outEdgeType = new HashMap<String, List<Integer>>();
+ outEdgeIndex = new HashMap<String, List<Integer>>();
outEdgeNames = new HashMap<String, List<List<String>>>();
outEdgeSelectAll = new HashMap<String, List<Boolean>>();
inEdgeList = new HashMap<String, List<String>>();
@@ -121,16 +129,11 @@ public class JobGraphBuilder {
iterationWaitTime = new HashMap<String, Long>();
operatorStates = new HashMap<String, Map<String, OperatorState<?>>>();
inputFormatList = new HashMap<String, InputFormat<String, ?>>();
- chainedInvokable = new HashMap<String, List<ChainableInvokable<?, ?>>>();
- chainedSerializer = new HashMap<String, List<StreamRecordSerializer<?>>>();
chainedVertices = new HashMap<String, List<String>>();
+ lastInChains = new HashMap<String, String>();
sources = new HashSet<String>();
builtVertices = new HashSet<String>();
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("JobGraph created");
- }
}
/**
@@ -198,7 +201,7 @@ public class JobGraphBuilder {
iterationIds.put(vertexName, iterationID);
iterationIDtoHeadName.put(iterationID, vertexName);
- setBytesFrom(iterationHead, vertexName);
+ setSerializersFrom(iterationHead, vertexName);
setEdge(vertexName, iterationHead, outPartitioning
.get(inEdgeList.get(iterationHead).get(0)).get(0), 0, new ArrayList<String>(),
@@ -241,7 +244,7 @@ public class JobGraphBuilder {
iterationIds.put(vertexName, iterationID);
iterationIDtoTailName.put(iterationID, vertexName);
- setBytesFrom(iterationTail, vertexName);
+ setSerializersFrom(iterationTail, vertexName);
iterationWaitTime.put(iterationIDtoTailName.get(iterationID), waitTime);
if (LOG.isDebugEnabled()) {
@@ -288,12 +291,13 @@ public class JobGraphBuilder {
invokableObjects.put(vertexName, invokableObject);
operatorNames.put(vertexName, operatorName);
outEdgeList.put(vertexName, new ArrayList<String>());
- outEdgeType.put(vertexName, new ArrayList<Integer>());
+ outEdgeIndex.put(vertexName, new ArrayList<Integer>());
outEdgeNames.put(vertexName, new ArrayList<List<String>>());
outEdgeSelectAll.put(vertexName, new ArrayList<Boolean>());
inEdgeList.put(vertexName, new ArrayList<String>());
outPartitioning.put(vertexName, new ArrayList<StreamPartitioner<?>>());
iterationTailCount.put(vertexName, 0);
+ lastInChains.put(vertexName, vertexName);
}
private void addTypeSerializers(String vertexName, StreamRecordSerializer<?> in1,
@@ -327,11 +331,16 @@ public class JobGraphBuilder {
}
}
+ List<String> chainedNames = chainedVertices.get(vertexName);
+ boolean isChained = chainedNames != null;
+ int numChained = isChained ? chainedNames.size() : 0;
+ String lastInChain = lastInChains.get(vertexName);
+
// Get vertex attributes
Class<? extends AbstractInvokable> vertexClass = vertexClasses.get(vertexName);
StreamInvokable<?, ?> invokableObject = invokableObjects.get(vertexName);
int parallelism = vertexParallelism.get(vertexName);
- byte[] outputSelector = outputSelectors.get(vertexName);
+ byte[] outputSelector = outputSelectors.get(lastInChain);
Map<String, OperatorState<?>> state = operatorStates.get(vertexName);
// Create vertex object
@@ -347,16 +356,17 @@ public class JobGraphBuilder {
LOG.debug("Parallelism set: {} for {}", parallelism, vertexName);
}
+ // Set vertex config
+
StreamConfig config = new StreamConfig(vertex.getConfiguration());
- config.setBufferTimeout(bufferTimeout.get(vertexName));
+ config.setBufferTimeout(bufferTimeout.get(lastInChain));
config.setTypeSerializerIn1(typeSerializersIn1.get(vertexName));
config.setTypeSerializerIn2(typeSerializersIn2.get(vertexName));
config.setTypeSerializerOut1(typeSerializersOut1.get(vertexName));
config.setTypeSerializerOut2(typeSerializersOut2.get(vertexName));
- // Set vertex config
config.setUserInvokable(invokableObject);
config.setOutputSelector(outputSelector);
config.setOperatorStates(state);
@@ -371,15 +381,12 @@ public class JobGraphBuilder {
vertex.setInputSplitSource(inputFormatList.get(vertexName));
}
- List<ChainableInvokable<?, ?>> chainedInvokables = chainedInvokable.get(vertexName);
- List<StreamRecordSerializer<?>> chainedSerializers = chainedSerializer.get(vertexName);
-
- int numChained = chainedInvokables == null ? 0 : chainedInvokables.size();
config.setNumberofChainedTasks(numChained);
for (int i = 0; i < numChained; i++) {
- config.setChainedInvokable(chainedInvokables.get(i), i);
- config.setChainedSerializer(chainedSerializers.get(i), i);
+ config.setChainedInvokable(
+ (ChainableInvokable<?, ?>) invokableObjects.get(chainedNames.get(i)), i);
+ config.setChainedSerializer(typeSerializersIn1.get(chainedNames.get(i)), i);
}
streamVertices.put(vertexName, vertex);
@@ -390,12 +397,15 @@ public class JobGraphBuilder {
}
private void chainRecursively(String chainStart, String current, String next) {
+ // We chain the next operator to the start of this chain
chainTasks(chainStart, next);
- // Add multiple chaining here
+ // Now recursively chain the outputs of next (depth first)
for (String output : outEdgeList.get(next)) {
if (isChainable(next, output)) {
+ // Recursive call
chainRecursively(chainStart, next, output);
} else {
+ // If not chainable we continue building the jobgraph from there
createVertex(output);
}
}
@@ -405,25 +415,14 @@ public class JobGraphBuilder {
return outEdgeList.get(vertexName).size() == 1
&& inEdgeList.get(outName).size() == 1
&& outputSelectors.get(vertexName) == null
- && invokableObjects.get(outName).isChainable()
+ && invokableObjects.get(outName).getChainingStrategy() == ChainingStrategy.ALWAYS
+ && (invokableObjects.get(vertexName).getChainingStrategy() == ChainingStrategy.HEAD || invokableObjects
+ .get(vertexName).getChainingStrategy() == ChainingStrategy.ALWAYS)
&& outPartitioning.get(vertexName).get(0).getStrategy() == PartitioningStrategy.FORWARD
&& vertexParallelism.get(vertexName) == vertexParallelism.get(outName) && chaining;
}
private void chainTasks(String first, String second) {
- List<ChainableInvokable<?, ?>> chainedInvokables = chainedInvokable.get(first);
- if (chainedInvokables == null) {
- chainedInvokables = new ArrayList<ChainableInvokable<?, ?>>();
- }
- chainedInvokables.add((ChainableInvokable<?, ?>) invokableObjects.get(second));
- chainedInvokable.put(first, chainedInvokables);
-
- List<StreamRecordSerializer<?>> chainedSerializers = chainedSerializer.get(first);
- if (chainedSerializers == null) {
- chainedSerializers = new ArrayList<StreamRecordSerializer<?>>();
- }
- chainedSerializers.add(typeSerializersIn1.get(second));
- chainedSerializer.put(first, chainedSerializers);
List<String> chained = chainedVertices.get(first);
if (chained == null) {
@@ -431,16 +430,7 @@ public class JobGraphBuilder {
}
chained.add(second);
chainedVertices.put(first, chained);
-
- outEdgeList.put(first, outEdgeList.get(second));
- typeSerializersOut1.put(first, typeSerializersOut1.get(second));
- outPartitioning.put(first, outPartitioning.get(second));
- outEdgeType.put(first, outEdgeType.get(second));
- outEdgeNames.put(first, outEdgeNames.get(second));
- outEdgeSelectAll.put(first, outEdgeSelectAll.get(second));
- outPartitioning.put(first, outPartitioning.get(second));
- bufferTimeout.put(first, bufferTimeout.get(second));
- outputSelectors.put(first, outputSelectors.get(second));
+ lastInChains.put(first, second);
}
@@ -477,8 +467,10 @@ public class JobGraphBuilder {
int outputIndex = upStreamVertex.getNumberOfProducedIntermediateDataSets() - 1;
- config.setOutputName(outputIndex, outEdgeNames.get(upStreamVertexName).get(outputIndex));
- config.setSelectAll(outputIndex, outEdgeSelectAll.get(upStreamVertexName).get(outputIndex));
+ config.setOutputName(outputIndex, outEdgeNames.get(lastInChains.get(upStreamVertexName))
+ .get(outputIndex));
+ config.setSelectAll(outputIndex, outEdgeSelectAll.get(lastInChains.get(upStreamVertexName))
+ .get(outputIndex));
config.setPartitioner(outputIndex, partitionerObject);
config.setNumberOfOutputChannels(outputIndex, vertexParallelism.get(downStreamVertexName));
}
@@ -547,7 +539,7 @@ public class JobGraphBuilder {
StreamPartitioner<?> partitionerObject, int typeNumber, List<String> outputNames,
boolean selectAll) {
outEdgeList.get(upStreamVertexName).add(downStreamVertexName);
- outEdgeType.get(upStreamVertexName).add(typeNumber);
+ outEdgeIndex.get(upStreamVertexName).add(typeNumber);
inEdgeList.get(downStreamVertexName).add(upStreamVertexName);
outPartitioning.get(upStreamVertexName).add(partitionerObject);
outEdgeNames.get(upStreamVertexName).add(outputNames);
@@ -608,7 +600,7 @@ public class JobGraphBuilder {
* @param to
* to
*/
- public void setBytesFrom(String from, String to) {
+ public void setSerializersFrom(String from, String to) {
operatorNames.put(to, operatorNames.get(from));
typeSerializersIn1.put(to, typeSerializersOut1.get(from));
@@ -691,19 +683,20 @@ public class JobGraphBuilder {
for (String upStreamVertexName : builtVertices) {
int i = 0;
- List<Integer> outEdgeTypeList = outEdgeType.get(upStreamVertexName);
+ List<Integer> outEdgeTypeList = outEdgeIndex.get(lastInChains.get(upStreamVertexName));
- for (String downStreamVertexName : outEdgeList.get(upStreamVertexName)) {
+ for (String downStreamVertexName : outEdgeList
+ .get(lastInChains.get(upStreamVertexName))) {
StreamConfig downStreamVertexConfig = new StreamConfig(streamVertices.get(
downStreamVertexName).getConfiguration());
int inputNumber = downStreamVertexConfig.getNumberOfInputs();
- downStreamVertexConfig.setInputType(inputNumber++, outEdgeTypeList.get(i));
+ downStreamVertexConfig.setInputIndex(inputNumber++, outEdgeTypeList.get(i));
downStreamVertexConfig.setNumberOfInputs(inputNumber);
connect(upStreamVertexName, downStreamVertexName,
- outPartitioning.get(upStreamVertexName).get(i));
+ outPartitioning.get(lastInChains.get(upStreamVertexName)).get(i));
i++;
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/3e30c6f7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
index 22bcf92..ada3aae 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
@@ -24,7 +24,6 @@ import java.util.Map;
import org.apache.commons.lang3.SerializationException;
import org.apache.commons.lang3.SerializationUtils;
-import org.apache.flink.api.common.functions.AbstractRichFunction;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.collector.OutputSelector;
import org.apache.flink.streaming.api.invokable.ChainableInvokable;
@@ -170,7 +169,7 @@ public class StreamConfig {
config.setBoolean(DIRECTED_EMIT, directedEmit);
}
- public boolean getDirectedEmit() {
+ public boolean isDirectedEmit() {
return config.getBoolean(DIRECTED_EMIT, false);
}
@@ -239,7 +238,7 @@ public class StreamConfig {
}
}
- public boolean getSelectAll(int outputIndex) {
+ public boolean isSelectAll(int outputIndex) {
return config.getBoolean(OUTPUT_SELECT_ALL + outputIndex, false);
}
@@ -272,26 +271,14 @@ public class StreamConfig {
return config.getInteger(NUMBER_OF_OUTPUTS, 0);
}
- public void setInputType(int inputNumber, Integer inputTypeNumber) {
+ public void setInputIndex(int inputNumber, Integer inputTypeNumber) {
config.setInteger(INPUT_TYPE + inputNumber++, inputTypeNumber);
}
- public int getInputType(int inputNumber) {
+ public int getInputIndex(int inputNumber) {
return config.getInteger(INPUT_TYPE + inputNumber, 0);
}
- public void setFunctionClass(Class<? extends AbstractRichFunction> functionClass) {
- config.setClass("functionClass", functionClass);
- }
-
- public Class<? extends AbstractRichFunction> getFunctionClass(ClassLoader cl) {
- try {
- return config.getClass("functionClass", null, cl);
- } catch (ClassNotFoundException e) {
- throw new RuntimeException("Could not load function class", e);
- }
- }
-
public void setOperatorStates(Map<String, OperatorState<?>> states) {
config.setBytes(OPERATOR_STATES, SerializationUtils.serialize((Serializable) states));
}
http://git-wip-us.apache.org/repos/asf/flink/blob/3e30c6f7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
index a0c8ff8..80ea970 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
@@ -400,7 +400,7 @@ public class ConnectedDataStream<IN1, IN2> {
@SuppressWarnings({ "unchecked", "rawtypes" })
SingleOutputStreamOperator<OUT, ?> returnStream = new SingleOutputStreamOperator(
- environment, functionName, outTypeInfo);
+ environment, functionName, outTypeInfo, functionInvokable);
dataStream1.jobGraphBuilder.addCoTask(returnStream.getId(), functionInvokable,
getInputType1(), getInputType2(), outTypeInfo, functionName,
http://git-wip-us.apache.org/repos/asf/flink/blob/3e30c6f7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index db644e9..f68ab68 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -1083,7 +1083,7 @@ public class DataStream<OUT> {
protected <R> DataStream<OUT> addIterationSource(Integer iterationID, long waitTime) {
DataStream<R> returnStream = new DataStreamSource<R>(environment, "iterationSource", null,
- true);
+ null, true);
jobGraphBuilder.addIterationHead(returnStream.getId(), this.getId(), iterationID,
degreeOfParallelism, waitTime);
@@ -1110,7 +1110,7 @@ public class DataStream<OUT> {
DataStream<OUT> inputStream = this.copy();
@SuppressWarnings({ "unchecked", "rawtypes" })
SingleOutputStreamOperator<R, ?> returnStream = new SingleOutputStreamOperator(environment,
- operatorName, outTypeInfo);
+ operatorName, outTypeInfo, invokable);
jobGraphBuilder.addStreamVertex(returnStream.getId(), invokable, getType(), outTypeInfo,
operatorName, degreeOfParallelism);
@@ -1174,10 +1174,13 @@ public class DataStream<OUT> {
*/
public DataStreamSink<OUT> addSink(SinkFunction<OUT> sinkFunction) {
- DataStreamSink<OUT> returnStream = new DataStreamSink<OUT>(environment, "sink", getType());
+ StreamInvokable<OUT, OUT> sinkInvokable = new SinkInvokable<OUT>(clean(sinkFunction));
- jobGraphBuilder.addStreamVertex(returnStream.getId(), new SinkInvokable<OUT>(
- clean(sinkFunction)), getType(), null, "sink", degreeOfParallelism);
+ DataStreamSink<OUT> returnStream = new DataStreamSink<OUT>(environment, "sink", getType(),
+ sinkInvokable);
+
+ jobGraphBuilder.addStreamVertex(returnStream.getId(), sinkInvokable, getType(), null,
+ "sink", degreeOfParallelism);
this.connectGraph(this.copy(), returnStream.getId(), 0);
http://git-wip-us.apache.org/repos/asf/flink/blob/3e30c6f7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
index 61fc557..f064332 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
@@ -19,6 +19,7 @@ package org.apache.flink.streaming.api.datastream;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
/**
* Represents the end of a DataStream.
@@ -29,8 +30,8 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
public class DataStreamSink<IN> extends SingleOutputStreamOperator<IN, DataStreamSink<IN>> {
protected DataStreamSink(StreamExecutionEnvironment environment, String operatorType,
- TypeInformation<IN> outTypeInfo) {
- super(environment, operatorType, outTypeInfo);
+ TypeInformation<IN> outTypeInfo, StreamInvokable<?,?> invokable) {
+ super(environment, operatorType, outTypeInfo, invokable);
}
protected DataStreamSink(DataStream<IN> dataStream) {
http://git-wip-us.apache.org/repos/asf/flink/blob/3e30c6f7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java
index a649f59..b596cbd 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java
@@ -19,6 +19,7 @@ package org.apache.flink.streaming.api.datastream;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
/**
* The DataStreamSource represents the starting point of a DataStream.
@@ -31,8 +32,8 @@ public class DataStreamSource<OUT> extends SingleOutputStreamOperator<OUT, DataS
boolean isParallel;
public DataStreamSource(StreamExecutionEnvironment environment, String operatorType,
- TypeInformation<OUT> outTypeInfo, boolean isParallel) {
- super(environment, operatorType, outTypeInfo);
+ TypeInformation<OUT> outTypeInfo, StreamInvokable<?, ?> invokable, boolean isParallel) {
+ super(environment, operatorType, outTypeInfo, invokable);
this.isParallel = isParallel;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/3e30c6f7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
index 88feb5d..c4cd1e1 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
@@ -61,7 +61,8 @@ public class IterativeDataStream<IN> extends
*
*/
public DataStream<IN> closeWith(DataStream<IN> iterationTail) {
- DataStream<IN> iterationSink = new DataStreamSink<IN>(environment, "iterationSink", null);
+ DataStream<IN> iterationSink = new DataStreamSink<IN>(environment, "iterationSink", null,
+ null);
jobGraphBuilder.addIterationTail(iterationSink.getId(), iterationTail.getId(), iterationID,
iterationTail.getParallelism(), waitTime);
http://git-wip-us.apache.org/repos/asf/flink/blob/3e30c6f7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
index 7a0abe4..b8ca42c 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
@@ -26,6 +26,8 @@ import org.apache.flink.api.common.functions.RichFunction;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.streaming.api.collector.OutputSelector;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.invokable.StreamInvokable.ChainingStrategy;
import org.apache.flink.streaming.api.streamvertex.StreamingRuntimeContext;
import org.apache.flink.streaming.state.OperatorState;
@@ -42,12 +44,14 @@ public class SingleOutputStreamOperator<OUT, O extends SingleOutputStreamOperato
DataStream<OUT> {
protected boolean isSplit;
+ protected StreamInvokable<?, ?> invokable;
protected SingleOutputStreamOperator(StreamExecutionEnvironment environment,
- String operatorType, TypeInformation<OUT> outTypeInfo) {
+ String operatorType, TypeInformation<OUT> outTypeInfo, StreamInvokable<?, ?> invokable) {
super(environment, operatorType, outTypeInfo);
setBufferTimeout(environment.getBufferTimeout());
this.isSplit = false;
+ this.invokable = invokable;
}
@SuppressWarnings("unchecked")
@@ -55,6 +59,7 @@ public class SingleOutputStreamOperator<OUT, O extends SingleOutputStreamOperato
super(dataStream);
if (dataStream instanceof SingleOutputStreamOperator) {
this.isSplit = ((SingleOutputStreamOperator<OUT, ?>) dataStream).isSplit;
+ this.invokable = ((SingleOutputStreamOperator<OUT, ?>) dataStream).invokable;
}
}
@@ -192,4 +197,9 @@ public class SingleOutputStreamOperator<OUT, O extends SingleOutputStreamOperato
return new SingleOutputStreamOperator<OUT, O>(this);
}
+ public SingleOutputStreamOperator<OUT, O> setChainingStrategy(ChainingStrategy strategy) {
+ this.invokable.setChainingStrategy(strategy);
+ return this;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/flink/blob/3e30c6f7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
index 08fd19b..4194864 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
@@ -46,6 +46,7 @@ import org.apache.flink.streaming.api.function.source.RichParallelSourceFunction
import org.apache.flink.streaming.api.function.source.SocketTextStreamFunction;
import org.apache.flink.streaming.api.function.source.SourceFunction;
import org.apache.flink.streaming.api.invokable.SourceInvokable;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
/**
* {@link ExecutionEnvironment} for streaming jobs. An instance of it is
@@ -431,11 +432,13 @@ public abstract class StreamExecutionEnvironment {
boolean isParallel = function instanceof ParallelSourceFunction;
int dop = isParallel ? getDegreeOfParallelism() : 1;
+ StreamInvokable<OUT, OUT> sourceInvokable = new SourceInvokable<OUT>(function);
+
DataStreamSource<OUT> returnStream = new DataStreamSource<OUT>(this, sourceName,
- outTypeInfo, isParallel);
+ outTypeInfo, sourceInvokable, isParallel);
- jobGraphBuilder.addSourceVertex(returnStream.getId(), new SourceInvokable<OUT>(function),
- null, outTypeInfo, sourceName, dop);
+ jobGraphBuilder.addSourceVertex(returnStream.getId(), sourceInvokable, null, outTypeInfo,
+ sourceName, dop);
return returnStream;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/3e30c6f7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/ChainableInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/ChainableInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/ChainableInvokable.java
index 373b4e8..24c0319 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/ChainableInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/ChainableInvokable.java
@@ -28,6 +28,7 @@ public abstract class ChainableInvokable<IN, OUT> extends StreamInvokable<IN, OU
public ChainableInvokable(Function userFunction) {
super(userFunction);
+ setChainingStrategy(ChainingStrategy.ALWAYS);
}
public void setup(Collector<OUT> collector, StreamRecordSerializer<IN> inSerializer) {
http://git-wip-us.apache.org/repos/asf/flink/blob/3e30c6f7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java
index 25e9221..614b67f 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java
@@ -59,6 +59,8 @@ public abstract class StreamInvokable<IN, OUT> implements Serializable {
protected Function userFunction;
protected volatile boolean isRunning;
+ private ChainingStrategy chainingStrategy = ChainingStrategy.HEAD;
+
public StreamInvokable(Function userFunction) {
this.userFunction = userFunction;
}
@@ -160,7 +162,21 @@ public abstract class StreamInvokable<IN, OUT> implements Serializable {
return objectSerializer.copy(record);
}
- public boolean isChainable() {
- return this instanceof ChainableInvokable;
+ public void setChainingStrategy(ChainingStrategy strategy) {
+ if (strategy == ChainingStrategy.ALWAYS) {
+ if (!(this instanceof ChainableInvokable)) {
+ throw new RuntimeException(
+ "Invokable needs to extend ChainableInvokable to be chained");
+ }
+ }
+ this.chainingStrategy = strategy;
+ }
+
+ public ChainingStrategy getChainingStrategy() {
+ return chainingStrategy;
+ }
+
+ public static enum ChainingStrategy {
+ ALWAYS, NEVER, HEAD;
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/3e30c6f7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java
index 84258d6..01c0545 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java
@@ -34,6 +34,7 @@ public class GroupedReduceInvokable<IN> extends StreamReduceInvokable<IN> {
super(reducer);
this.keySelector = keySelector;
values = new HashMap<Object, IN>();
+ setChainingStrategy(ChainingStrategy.NEVER);
}
@Override
@@ -56,9 +57,4 @@ public class GroupedReduceInvokable<IN> extends StreamReduceInvokable<IN> {
reduced = reducer.reduce(currentValue, nextValue);
}
- @Override
- public boolean isChainable() {
- return false;
- }
-
}
http://git-wip-us.apache.org/repos/asf/flink/blob/3e30c6f7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
index 33348e4..bbd7b0c 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
@@ -154,6 +154,7 @@ public class GroupedWindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
LinkedList<EvictionPolicy<IN>> centralEvictionPolicies) {
super(userFunction);
+ setChainingStrategy(ChainingStrategy.NEVER);
this.keySelector = keySelector;
http://git-wip-us.apache.org/repos/asf/flink/blob/3e30c6f7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
index ea891c9..de9c664 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
@@ -61,7 +61,8 @@ public abstract class WindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT>
public WindowInvokable(Function userFunction, LinkedList<TriggerPolicy<IN>> triggerPolicies,
LinkedList<EvictionPolicy<IN>> evictionPolicies) {
super(userFunction);
-
+ setChainingStrategy(ChainingStrategy.NEVER);
+
this.triggerPolicies = triggerPolicies;
this.evictionPolicies = evictionPolicies;
http://git-wip-us.apache.org/repos/asf/flink/blob/3e30c6f7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/CoStreamVertex.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/CoStreamVertex.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/CoStreamVertex.java
index f065d9c..2b650be 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/CoStreamVertex.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/CoStreamVertex.java
@@ -80,7 +80,7 @@ public class CoStreamVertex<IN1, IN2, OUT> extends StreamVertex<IN1, OUT> {
ArrayList<BufferReader> inputList2 = new ArrayList<BufferReader>();
for (int i = 0; i < numberOfInputs; i++) {
- int inputType = configuration.getInputType(i);
+ int inputType = configuration.getInputIndex(i);
BufferReader reader = getEnvironment().getReader(i);
switch (inputType) {
case 1:
http://git-wip-us.apache.org/repos/asf/flink/blob/3e30c6f7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
index 457f3f8..60a7b14 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
@@ -26,7 +26,6 @@ import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
import org.apache.flink.runtime.plugable.SerializationDelegate;
import org.apache.flink.streaming.api.StreamConfig;
-import org.apache.flink.streaming.api.collector.CollectorWrapper;
import org.apache.flink.streaming.api.collector.DirectedOutputWrapper;
import org.apache.flink.streaming.api.collector.OutputSelector;
import org.apache.flink.streaming.api.collector.StreamOutput;
@@ -44,11 +43,11 @@ import org.slf4j.LoggerFactory;
public class OutputHandler<OUT> {
private static final Logger LOG = LoggerFactory.getLogger(OutputHandler.class);
- private StreamVertex<?, OUT> streamVertex;
+ private StreamVertex<?, OUT> vertex;
private StreamConfig configuration;
private List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> outputs;
- private Collector<OUT> endCollector;
+ private Collector<OUT> outerCollector;
TypeInformation<OUT> outTypeInfo = null;
StreamRecordSerializer<OUT> outSerializer = null;
@@ -58,18 +57,14 @@ public class OutputHandler<OUT> {
private int numberOfChainedTasks;
- public OutputHandler(StreamVertex<?, OUT> streamComponent) {
- this.streamVertex = streamComponent;
+ public OutputHandler(StreamVertex<?, OUT> vertex) {
+ this.vertex = vertex;
this.outputs = new LinkedList<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>>();
- this.configuration = new StreamConfig(streamComponent.getTaskConfiguration());
+ this.configuration = new StreamConfig(vertex.getTaskConfiguration());
this.chainedInvokables = new ArrayList<ChainableInvokable<?, ?>>();
+ this.numberOfChainedTasks = configuration.getNumberofChainedTasks();
- try {
- setConfigOutputs();
- } catch (StreamVertexException e) {
- throw new StreamVertexException("Cannot register outputs for "
- + streamComponent.getClass().getSimpleName(), e);
- }
+ this.outerCollector = createChainedCollector(0);
}
@@ -77,84 +72,83 @@ public class OutputHandler<OUT> {
return outputs;
}
- private void setConfigOutputs() {
- numberOfChainedTasks = configuration.getNumberofChainedTasks();
- endCollector = createChainedOutputs(0);
- }
-
- @SuppressWarnings("unchecked")
- private Collector<OUT> createChainedOutputs(int chainedTaskIndex) {
+ // We create the outer collector by nesting the chainable invokables into
+ // each other
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ private Collector<OUT> createChainedCollector(int chainedTaskIndex) {
if (numberOfChainedTasks == chainedTaskIndex) {
- return createEndCollector();
+ // At the end of the chain we create the collector that sends data
+ // to the recordwriters
+ return createNetworkCollector();
} else {
- CollectorWrapper<OUT> chainedCollector = new CollectorWrapper<OUT>();
- @SuppressWarnings("rawtypes")
ChainableInvokable chainableInvokable = configuration.getChainedInvokable(
- chainedTaskIndex, streamVertex.getUserCodeClassLoader());
+ chainedTaskIndex, vertex.getUserCodeClassLoader());
+ // The nesting is done by calling this method recursively when
+ // passing the collector to the invokable
chainableInvokable.setup(
- createChainedOutputs(chainedTaskIndex + 1),
+ createChainedCollector(chainedTaskIndex + 1),
configuration.getChainedInSerializer(chainedTaskIndex,
- streamVertex.getUserCodeClassLoader()));
+ vertex.getUserCodeClassLoader()));
+ // We hold a list of the chained invokables for initializaton
+ // afterwards
chainedInvokables.add(chainableInvokable);
- chainedCollector.addChainedOutput((Collector<OUT>) chainableInvokable);
-
- return chainedCollector;
+ return chainableInvokable;
}
}
- private Collector<OUT> createEndCollector() {
+ private Collector<OUT> createNetworkCollector() {
- setSerializers();
+ createOutSerializer();
StreamOutputWrapper<OUT> collector;
- if (streamVertex.configuration.getDirectedEmit()) {
- OutputSelector<OUT> outputSelector = streamVertex.configuration
- .getOutputSelector(streamVertex.userClassLoader);
+ if (vertex.configuration.isDirectedEmit()) {
+ OutputSelector<OUT> outputSelector = vertex.configuration
+ .getOutputSelector(vertex.userClassLoader);
- collector = new DirectedOutputWrapper<OUT>(streamVertex.getInstanceID(),
+ collector = new DirectedOutputWrapper<OUT>(vertex.getInstanceID(),
outSerializationDelegate, outputSelector);
} else {
- collector = new StreamOutputWrapper<OUT>(streamVertex.getInstanceID(),
+ collector = new StreamOutputWrapper<OUT>(vertex.getInstanceID(),
outSerializationDelegate);
}
int numberOfOutputs = configuration.getNumberOfOutputs();
for (int i = 0; i < numberOfOutputs; i++) {
- collector = (StreamOutputWrapper<OUT>) setPartitioner(i, collector);
+ collector = (StreamOutputWrapper<OUT>) addStreamOutput(i, collector);
}
return collector;
}
public Collector<OUT> getCollector() {
- return endCollector;
+ return outerCollector;
}
- void setSerializers() {
- outSerializer = configuration.getTypeSerializerOut1(streamVertex.userClassLoader);
+ void createOutSerializer() {
+ outSerializer = configuration.getTypeSerializerOut1(vertex.userClassLoader);
if (outSerializer != null) {
outSerializationDelegate = new SerializationDelegate<StreamRecord<OUT>>(outSerializer);
outSerializationDelegate.setInstance(outSerializer.createInstance());
}
}
- Collector<OUT> setPartitioner(int outputNumber, StreamOutputWrapper<OUT> endCollector) {
- StreamPartitioner<OUT> outputPartitioner = null;
+ Collector<OUT> addStreamOutput(int outputNumber, StreamOutputWrapper<OUT> networkCollector) {
+
+ StreamPartitioner<OUT> outputPartitioner;
try {
- outputPartitioner = configuration.getPartitioner(streamVertex.userClassLoader,
+ outputPartitioner = configuration.getPartitioner(vertex.userClassLoader,
outputNumber);
-
} catch (Exception e) {
throw new StreamVertexException("Cannot deserialize partitioner for "
- + streamVertex.getName() + " with " + outputNumber + " outputs", e);
+ + vertex.getName() + " with " + outputNumber + " outputs", e);
}
RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output;
@@ -162,37 +156,33 @@ public class OutputHandler<OUT> {
long bufferTimeout = configuration.getBufferTimeout();
if (bufferTimeout >= 0) {
- output = new StreamRecordWriter<SerializationDelegate<StreamRecord<OUT>>>(streamVertex
+ output = new StreamRecordWriter<SerializationDelegate<StreamRecord<OUT>>>(vertex
.getEnvironment().getWriter(outputNumber), outputPartitioner, bufferTimeout);
if (LOG.isTraceEnabled()) {
LOG.trace("StreamRecordWriter initiated with {} bufferTimeout for {}",
- bufferTimeout, streamVertex.getClass().getSimpleName());
+ bufferTimeout, vertex.getClass().getSimpleName());
}
} else {
- output = new RecordWriter<SerializationDelegate<StreamRecord<OUT>>>(streamVertex
+ output = new RecordWriter<SerializationDelegate<StreamRecord<OUT>>>(vertex
.getEnvironment().getWriter(outputNumber), outputPartitioner);
if (LOG.isTraceEnabled()) {
- LOG.trace("RecordWriter initiated for {}", streamVertex.getClass().getSimpleName());
+ LOG.trace("RecordWriter initiated for {}", vertex.getClass().getSimpleName());
}
}
outputs.add(output);
- List<String> outputNames = configuration.getOutputNames(outputNumber);
- boolean isSelectAllOutput = configuration.getSelectAll(outputNumber);
- if (endCollector != null) {
- endCollector.addOutput(new StreamOutput<OUT>(output, isSelectAllOutput ? null
- : outputNames));
- }
+ networkCollector.addOutput(new StreamOutput<OUT>(output, configuration
+ .isSelectAll(outputNumber) ? null : configuration.getOutputNames(outputNumber)));
if (LOG.isTraceEnabled()) {
LOG.trace("Partitioner set: {} with {} outputs for {}", outputPartitioner.getClass()
- .getSimpleName(), outputNumber, streamVertex.getClass().getSimpleName());
+ .getSimpleName(), outputNumber, vertex.getClass().getSimpleName());
}
- return endCollector;
+ return networkCollector;
}
public void flushOutputs() throws IOException, InterruptedException {
@@ -205,17 +195,15 @@ public class OutputHandler<OUT> {
}
}
- long startTime;
-
public void invokeUserFunction(String componentTypeName, StreamInvokable<?, OUT> userInvokable)
throws IOException, InterruptedException {
if (LOG.isDebugEnabled()) {
LOG.debug("{} {} invoked with instance id {}", componentTypeName,
- streamVertex.getName(), streamVertex.getInstanceID());
+ vertex.getName(), vertex.getInstanceID());
}
try {
- streamVertex.invokeUserFunction(userInvokable);
+ vertex.invokeUserFunction(userInvokable);
} catch (Exception e) {
flushOutputs();
throw new RuntimeException(e);
@@ -223,7 +211,7 @@ public class OutputHandler<OUT> {
if (LOG.isDebugEnabled()) {
LOG.debug("{} {} invoke finished instance id {}", componentTypeName,
- streamVertex.getName(), streamVertex.getInstanceID());
+ vertex.getName(), vertex.getInstanceID());
}
flushOutputs();
[2/8] flink git commit: [FLINK-1345] [streaming] Basic operator
chaining added
Posted by mb...@apache.org.
[FLINK-1345] [streaming] Basic operator chaining added
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/26535c48
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/26535c48
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/26535c48
Branch: refs/heads/master
Commit: 26535c487753975556aafc19c2ce4f9c24cc677b
Parents: d9b942b
Author: Gyula Fora <gy...@apache.org>
Authored: Thu Jan 15 22:32:13 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Wed Jan 21 16:06:33 2015 +0100
----------------------------------------------------------------------
.../flink/streaming/api/JobGraphBuilder.java | 222 +++++++++++++------
.../flink/streaming/api/StreamConfig.java | 56 +++--
.../api/collector/CollectorWrapper.java | 2 +-
.../api/datastream/IterativeDataStream.java | 2 +-
.../datastream/SingleOutputStreamOperator.java | 5 +
.../environment/StreamExecutionEnvironment.java | 2 +-
.../api/invokable/ChainableInvokable.java | 38 ++++
.../streaming/api/invokable/SinkInvokable.java | 10 +-
.../api/invokable/StreamInvokable.java | 23 +-
.../invokable/operator/CounterInvokable.java | 10 +-
.../api/invokable/operator/FilterInvokable.java | 18 +-
.../invokable/operator/FlatMapInvokable.java | 12 +-
.../operator/GroupedReduceInvokable.java | 7 +-
.../api/invokable/operator/MapInvokable.java | 12 +-
.../operator/StreamReduceInvokable.java | 15 +-
.../operator/co/CoWindowInvokable.java | 2 +-
.../api/streamvertex/OutputHandler.java | 86 +++++--
.../api/streamvertex/StreamVertex.java | 24 +-
.../examples/iteration/IterateExample.java | 3 +-
19 files changed, 406 insertions(+), 143 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
index 649072f..35e145e 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
@@ -22,7 +22,9 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+import org.apache.commons.lang3.StringUtils;
import org.apache.flink.api.common.io.InputFormat;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.runtime.jobgraph.AbstractJobVertex;
@@ -32,8 +34,7 @@ import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
import org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup;
import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
import org.apache.flink.streaming.api.collector.OutputSelector;
-import org.apache.flink.streaming.api.function.source.SourceFunction;
-import org.apache.flink.streaming.api.invokable.SourceInvokable;
+import org.apache.flink.streaming.api.invokable.ChainableInvokable;
import org.apache.flink.streaming.api.invokable.StreamInvokable;
import org.apache.flink.streaming.api.invokable.operator.co.CoInvokable;
import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
@@ -56,6 +57,8 @@ public class JobGraphBuilder {
private final static String DEAFULT_JOB_NAME = "Streaming Job";
private JobGraph jobGraph;
+ private boolean chaining = true;
+
// Graph attributes
private Map<String, AbstractJobVertex> streamVertices;
private Map<String, Integer> vertexParallelism;
@@ -65,7 +68,7 @@ public class JobGraphBuilder {
private Map<String, List<List<String>>> outEdgeNames;
private Map<String, List<Boolean>> outEdgeSelectAll;
private Map<String, List<String>> inEdgeList;
- private Map<String, List<StreamPartitioner<?>>> connectionTypes;
+ private Map<String, List<StreamPartitioner<?>>> outPartitioning;
private Map<String, String> operatorNames;
private Map<String, StreamInvokable<?, ?>> invokableObjects;
private Map<String, StreamRecordSerializer<?>> typeSerializersIn1;
@@ -81,6 +84,12 @@ public class JobGraphBuilder {
private Map<String, Long> iterationWaitTime;
private Map<String, Map<String, OperatorState<?>>> operatorStates;
private Map<String, InputFormat<String, ?>> inputFormatList;
+ private Map<String, List<String>> chainedVertices;
+ private Map<String, List<ChainableInvokable<?, ?>>> chainedInvokable;
+ private Map<String, List<StreamRecordSerializer<?>>> chainedSerializer;
+
+ private Set<String> sources;
+ private Set<String> builtVertices;
/**
* Creates an new {@link JobGraph} with the given name. A JobGraph is a DAG
@@ -96,7 +105,7 @@ public class JobGraphBuilder {
outEdgeNames = new HashMap<String, List<List<String>>>();
outEdgeSelectAll = new HashMap<String, List<Boolean>>();
inEdgeList = new HashMap<String, List<String>>();
- connectionTypes = new HashMap<String, List<StreamPartitioner<?>>>();
+ outPartitioning = new HashMap<String, List<StreamPartitioner<?>>>();
operatorNames = new HashMap<String, String>();
invokableObjects = new HashMap<String, StreamInvokable<?, ?>>();
typeSerializersIn1 = new HashMap<String, StreamRecordSerializer<?>>();
@@ -112,6 +121,12 @@ public class JobGraphBuilder {
iterationWaitTime = new HashMap<String, Long>();
operatorStates = new HashMap<String, Map<String, OperatorState<?>>>();
inputFormatList = new HashMap<String, InputFormat<String, ?>>();
+ chainedInvokable = new HashMap<String, List<ChainableInvokable<?, ?>>>();
+ chainedSerializer = new HashMap<String, List<StreamRecordSerializer<?>>>();
+ chainedVertices = new HashMap<String, List<String>>();
+
+ sources = new HashSet<String>();
+ builtVertices = new HashSet<String>();
if (LOG.isDebugEnabled()) {
LOG.debug("JobGraph created");
@@ -152,32 +167,12 @@ public class JobGraphBuilder {
}
}
- /**
- * Adds a source vertex to the streaming JobGraph with the given parameters
- *
- * @param vertexName
- * Name of the vertex
- * @param function
- * User defined function
- * @param inTypeInfo
- * Input type for serialization
- * @param outTypeInfo
- * Output type for serialization
- * @param operatorName
- * Operator type
- * @param parallelism
- * Number of parallel instances created
- */
- public <IN, OUT> void addSourceVertex(String vertexName, SourceFunction<OUT> function,
- TypeInformation<IN> inTypeInfo, TypeInformation<OUT> outTypeInfo, String operatorName,
- byte[] serializedFunction, int parallelism) {
-
- @SuppressWarnings("unchecked")
- StreamInvokable<IN, OUT> invokableObject = (StreamInvokable<IN, OUT>) new SourceInvokable<OUT>(
- function);
-
+ public <IN, OUT> void addSourceVertex(String vertexName,
+ StreamInvokable<IN, OUT> invokableObject, TypeInformation<IN> inTypeInfo,
+ TypeInformation<OUT> outTypeInfo, String operatorName, int parallelism) {
addStreamVertex(vertexName, invokableObject, inTypeInfo, outTypeInfo, operatorName,
parallelism);
+ sources.add(vertexName);
}
/**
@@ -205,7 +200,7 @@ public class JobGraphBuilder {
setBytesFrom(iterationHead, vertexName);
- setEdge(vertexName, iterationHead, connectionTypes
+ setEdge(vertexName, iterationHead, outPartitioning
.get(inEdgeList.get(iterationHead).get(0)).get(0), 0, new ArrayList<String>(),
false);
@@ -214,6 +209,8 @@ public class JobGraphBuilder {
if (LOG.isDebugEnabled()) {
LOG.debug("ITERATION SOURCE: {}", vertexName);
}
+
+ sources.add(vertexName);
}
/**
@@ -295,7 +292,7 @@ public class JobGraphBuilder {
outEdgeNames.put(vertexName, new ArrayList<List<String>>());
outEdgeSelectAll.put(vertexName, new ArrayList<Boolean>());
inEdgeList.put(vertexName, new ArrayList<String>());
- connectionTypes.put(vertexName, new ArrayList<StreamPartitioner<?>>());
+ outPartitioning.put(vertexName, new ArrayList<StreamPartitioner<?>>());
iterationTailCount.put(vertexName, 0);
}
@@ -317,50 +314,134 @@ public class JobGraphBuilder {
*/
private void createVertex(String vertexName) {
- // Get vertex attributes
- Class<? extends AbstractInvokable> vertexClass = vertexClasses.get(vertexName);
- StreamInvokable<?, ?> invokableObject = invokableObjects.get(vertexName);
- int parallelism = vertexParallelism.get(vertexName);
- byte[] outputSelector = outputSelectors.get(vertexName);
- Map<String, OperatorState<?>> state = operatorStates.get(vertexName);
+ if (!builtVertices.contains(vertexName)) {
+ if (!outEdgeList.get(vertexName).isEmpty()) {
- // Create vertex object
- AbstractJobVertex vertex = new AbstractJobVertex(vertexName);
+ for (String outName : outEdgeList.get(vertexName)) {
+ if (isChainable(vertexName, outName)) {
+ chainRecursively(vertexName, vertexName, outName);
+ } else {
+ createVertex(outName);
+ }
- this.jobGraph.addVertex(vertex);
+ }
+ }
+
+ // Get vertex attributes
+ Class<? extends AbstractInvokable> vertexClass = vertexClasses.get(vertexName);
+ StreamInvokable<?, ?> invokableObject = invokableObjects.get(vertexName);
+ int parallelism = vertexParallelism.get(vertexName);
+ byte[] outputSelector = outputSelectors.get(vertexName);
+ Map<String, OperatorState<?>> state = operatorStates.get(vertexName);
+
+ // Create vertex object
+ String cname = chainedVertices.get(vertexName) == null ? "" : " => "
+ + StringUtils.join(chainedVertices.get(vertexName), " => ");
+ AbstractJobVertex vertex = new AbstractJobVertex(vertexName + cname);
+
+ this.jobGraph.addVertex(vertex);
+
+ vertex.setInvokableClass(vertexClass);
+ vertex.setParallelism(parallelism);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Parallelism set: {} for {}", parallelism, vertexName);
+ }
+
+ StreamConfig config = new StreamConfig(vertex.getConfiguration());
+
+ config.setBufferTimeout(bufferTimeout.get(vertexName));
+
+ config.setTypeSerializerIn1(typeSerializersIn1.get(vertexName));
+ config.setTypeSerializerIn2(typeSerializersIn2.get(vertexName));
+ config.setTypeSerializerOut1(typeSerializersOut1.get(vertexName));
+ config.setTypeSerializerOut2(typeSerializersOut2.get(vertexName));
+
+ // Set vertex config
+ config.setUserInvokable(invokableObject);
+ config.setOutputSelector(outputSelector);
+ config.setOperatorStates(state);
+
+ if (vertexClass.equals(StreamIterationHead.class)
+ || vertexClass.equals(StreamIterationTail.class)) {
+ config.setIterationId(iterationIds.get(vertexName));
+ config.setIterationWaitTime(iterationWaitTime.get(vertexName));
+ }
+
+ if (inputFormatList.containsKey(vertexName)) {
+ vertex.setInputSplitSource(inputFormatList.get(vertexName));
+ }
+
+ List<ChainableInvokable<?, ?>> chainedInvokables = chainedInvokable.get(vertexName);
+ List<StreamRecordSerializer<?>> chainedSerializers = chainedSerializer.get(vertexName);
+
+ int numChained = chainedInvokables == null ? 0 : chainedInvokables.size();
+ config.setNumberofChainedTasks(numChained);
+
+ for (int i = 0; i < numChained; i++) {
+ config.setChainedInvokable(chainedInvokables.get(i), i);
+ config.setChainedSerializer(chainedSerializers.get(i), i);
+ }
+
+ streamVertices.put(vertexName, vertex);
+ builtVertices.add(vertexName);
- vertex.setInvokableClass(vertexClass);
- vertex.setParallelism(parallelism);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Parallelism set: {} for {}", parallelism, vertexName);
}
- StreamConfig config = new StreamConfig(vertex.getConfiguration());
+ }
- config.setBufferTimeout(bufferTimeout.get(vertexName));
+ private void chainRecursively(String chainStart, String current, String next) {
+ chainTasks(chainStart, next);
+ // Add multiple chaining here
+ for (String output : outEdgeList.get(next)) {
+ if (isChainable(next, output)) {
+ chainRecursively(chainStart, next, output);
+ } else {
+ createVertex(output);
+ }
+ }
+ }
- config.setTypeSerializerIn1(typeSerializersIn1.get(vertexName));
- config.setTypeSerializerIn2(typeSerializersIn2.get(vertexName));
- config.setTypeSerializerOut1(typeSerializersOut1.get(vertexName));
- config.setTypeSerializerOut2(typeSerializersOut2.get(vertexName));
+ private boolean isChainable(String vertexName, String outName) {
+ return outEdgeList.get(vertexName).size() == 1
+ && inEdgeList.get(outName).size() == 1
+ && outputSelectors.get(vertexName) == null
+ && invokableObjects.get(outName).isChainable()
+ && outPartitioning.get(vertexName).get(0).getStrategy() == PartitioningStrategy.FORWARD
+ && vertexParallelism.get(vertexName) == vertexParallelism.get(outName) && chaining;
+ }
- // Set vertex config
- config.setUserInvokable(invokableObject);
- config.setVertexName(vertexName);
- config.setOutputSelector(outputSelector);
- config.setOperatorStates(state);
+ private void chainTasks(String first, String second) {
+ List<ChainableInvokable<?, ?>> chainedInvokables = chainedInvokable.get(first);
+ if (chainedInvokables == null) {
+ chainedInvokables = new ArrayList<ChainableInvokable<?, ?>>();
+ }
+ chainedInvokables.add((ChainableInvokable<?, ?>) invokableObjects.get(second));
+ chainedInvokable.put(first, chainedInvokables);
- if (vertexClass.equals(StreamIterationHead.class)
- || vertexClass.equals(StreamIterationTail.class)) {
- config.setIterationId(iterationIds.get(vertexName));
- config.setIterationWaitTime(iterationWaitTime.get(vertexName));
+ List<StreamRecordSerializer<?>> chainedSerializers = chainedSerializer.get(first);
+ if (chainedSerializers == null) {
+ chainedSerializers = new ArrayList<StreamRecordSerializer<?>>();
}
+ chainedSerializers.add(typeSerializersIn1.get(second));
+ chainedSerializer.put(first, chainedSerializers);
- if (inputFormatList.containsKey(vertexName)) {
- vertex.setInputSplitSource(inputFormatList.get(vertexName));
+ List<String> chained = chainedVertices.get(first);
+ if (chained == null) {
+ chained = new ArrayList<String>();
}
+ chained.add(second);
+ chainedVertices.put(first, chained);
+
+ outEdgeList.put(first, outEdgeList.get(second));
+ typeSerializersOut1.put(first, typeSerializersOut1.get(second));
+ outPartitioning.put(first, outPartitioning.get(second));
+ outEdgeType.put(first, outEdgeType.get(second));
+ outEdgeNames.put(first, outEdgeNames.get(second));
+ outEdgeSelectAll.put(first, outEdgeSelectAll.get(second));
+ outPartitioning.put(first, outPartitioning.get(second));
+ bufferTimeout.put(first, bufferTimeout.get(second));
+ outputSelectors.put(first, outputSelectors.get(second));
- streamVertices.put(vertexName, vertex);
}
/**
@@ -385,8 +466,8 @@ public class JobGraphBuilder {
downStreamVertex
.connectNewDataSetAsInput(upStreamVertex, DistributionPattern.POINTWISE);
} else {
- downStreamVertex
- .connectNewDataSetAsInput(upStreamVertex, DistributionPattern.ALL_TO_ALL);
+ downStreamVertex.connectNewDataSetAsInput(upStreamVertex,
+ DistributionPattern.ALL_TO_ALL);
}
if (LOG.isDebugEnabled()) {
@@ -468,7 +549,7 @@ public class JobGraphBuilder {
outEdgeList.get(upStreamVertexName).add(downStreamVertexName);
outEdgeType.get(upStreamVertexName).add(typeNumber);
inEdgeList.get(downStreamVertexName).add(upStreamVertexName);
- connectionTypes.get(upStreamVertexName).add(partitionerObject);
+ outPartitioning.get(upStreamVertexName).add(partitionerObject);
outEdgeNames.get(upStreamVertexName).add(outputNames);
outEdgeSelectAll.get(upStreamVertexName).add(selectAll);
}
@@ -602,11 +683,12 @@ public class JobGraphBuilder {
* provided.
*/
private void buildJobGraph() {
- for (String vertexName : outEdgeList.keySet()) {
- createVertex(vertexName);
+
+ for (String sourceName : sources) {
+ createVertex(sourceName);
}
- for (String upStreamVertexName : outEdgeList.keySet()) {
+ for (String upStreamVertexName : builtVertices) {
int i = 0;
List<Integer> outEdgeTypeList = outEdgeType.get(upStreamVertexName);
@@ -621,7 +703,7 @@ public class JobGraphBuilder {
downStreamVertexConfig.setNumberOfInputs(inputNumber);
connect(upStreamVertexName, downStreamVertexName,
- connectionTypes.get(upStreamVertexName).get(i));
+ outPartitioning.get(upStreamVertexName).get(i));
i++;
}
}
@@ -631,4 +713,8 @@ public class JobGraphBuilder {
setNumberOfJobOutputs();
}
+ public void setChaining(boolean chaining) {
+ this.chaining = chaining;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
index 8837b85..22bcf92 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
@@ -27,6 +27,7 @@ import org.apache.commons.lang3.SerializationUtils;
import org.apache.flink.api.common.functions.AbstractRichFunction;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.collector.OutputSelector;
+import org.apache.flink.streaming.api.invokable.ChainableInvokable;
import org.apache.flink.streaming.api.invokable.StreamInvokable;
import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
import org.apache.flink.streaming.api.streamvertex.StreamVertexException;
@@ -39,6 +40,9 @@ public class StreamConfig {
private static final String INPUT_TYPE = "inputType_";
private static final String NUMBER_OF_OUTPUTS = "numberOfOutputs";
private static final String NUMBER_OF_INPUTS = "numberOfInputs";
+ private static final String NUMBER_OF_CHAINED_TASKS = "numOfChained";
+ private static final String CHAINED_IN_SERIALIZER = "chainedSerializer_";
+ private static final String CHAINED_INVOKABLE = "chainedInvokable_";
private static final String OUTPUT_NAME = "outputName_";
private static final String OUTPUT_SELECT_ALL = "outputSelectAll_";
private static final String PARTITIONER_OBJECT = "partitionerObject_";
@@ -46,8 +50,6 @@ public class StreamConfig {
private static final String ITERATION_ID = "iteration-id";
private static final String OUTPUT_SELECTOR = "outputSelector";
private static final String DIRECTED_EMIT = "directedEmit";
- private static final String FUNCTION_NAME = "operatorName";
- private static final String VERTEX_NAME = "vertexName";
private static final String SERIALIZEDUDF = "serializedudf";
private static final String USER_FUNCTION = "userfunction";
private static final String BUFFER_TIMEOUT = "bufferTimeout";
@@ -164,18 +166,6 @@ public class StreamConfig {
}
}
- public void setVertexName(String vertexName) {
- config.setString(VERTEX_NAME, vertexName);
- }
-
- public String getVertexName() {
- return config.getString(VERTEX_NAME, null);
- }
-
- public String getFunctionName() {
- return config.getString(FUNCTION_NAME, "");
- }
-
public void setDirectedEmit(boolean directedEmit) {
config.setBoolean(DIRECTED_EMIT, directedEmit);
}
@@ -261,7 +251,7 @@ public class StreamConfig {
}
@SuppressWarnings("unchecked")
- public List<String> getOutputName(int outputIndex) {
+ public List<String> getOutputNames(int outputIndex) {
return (List<String>) SerializationUtils.deserialize(config.getBytes(OUTPUT_NAME
+ outputIndex, null));
}
@@ -316,4 +306,40 @@ public class StreamConfig {
}
}
+ public int getNumberofChainedTasks() {
+ return config.getInteger(NUMBER_OF_CHAINED_TASKS, 0);
+ }
+
+ public void setNumberofChainedTasks(int n) {
+ config.setInteger(NUMBER_OF_CHAINED_TASKS, n);
+ }
+
+ public ChainableInvokable<?, ?> getChainedInvokable(int chainedTaskIndex, ClassLoader cl) {
+ try {
+ return (ChainableInvokable<?, ?>) InstantiationUtil.readObjectFromConfig(this.config,
+ CHAINED_INVOKABLE + chainedTaskIndex, cl);
+ } catch (Exception e) {
+ throw new RuntimeException("Could not instantiate invokable.");
+ }
+ }
+
+ public StreamRecordSerializer<?> getChainedInSerializer(int chainedTaskIndex, ClassLoader cl) {
+ try {
+ return (StreamRecordSerializer<?>) InstantiationUtil.readObjectFromConfig(this.config,
+ CHAINED_IN_SERIALIZER + chainedTaskIndex, cl);
+ } catch (Exception e) {
+ throw new RuntimeException("Could not instantiate serializer.");
+ }
+ }
+
+ public void setChainedSerializer(StreamRecordSerializer<?> typeWrapper, int chainedTaskIndex) {
+ config.setBytes(CHAINED_IN_SERIALIZER + chainedTaskIndex,
+ SerializationUtils.serialize(typeWrapper));
+ }
+
+ public void setChainedInvokable(ChainableInvokable<?, ?> invokable, int chainedTaskIndex) {
+ config.setBytes(CHAINED_INVOKABLE + chainedTaskIndex,
+ SerializationUtils.serialize(invokable));
+ }
+
}
http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
index d782d08..b7e57e0 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/collector/CollectorWrapper.java
@@ -37,7 +37,7 @@ public class CollectorWrapper<OUT> implements Collector<OUT> {
@Override
public void collect(OUT record) {
for(Collector<OUT> output: outputs){
- output.collect(record);;
+ output.collect(record);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
index d7467d1..88feb5d 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
@@ -67,7 +67,7 @@ public class IterativeDataStream<IN> extends
iterationTail.getParallelism(), waitTime);
jobGraphBuilder.setIterationSourceSettings(iterationID.toString(), iterationTail.getId());
- connectGraph(iterationTail, iterationSink.getId(), 0);
+ connectGraph(iterationTail.forward(), iterationSink.getId(), 0);
return iterationTail;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
index 5a8261e..7a0abe4 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
@@ -182,6 +182,11 @@ public class SingleOutputStreamOperator<OUT, O extends SingleOutputStreamOperato
return (SingleOutputStreamOperator<OUT, O>) super.distribute();
}
+ @SuppressWarnings("unchecked")
+ public SingleOutputStreamOperator<OUT, O> global() {
+ return (SingleOutputStreamOperator<OUT, O>) super.global();
+ }
+
@Override
public SingleOutputStreamOperator<OUT, O> copy() {
return new SingleOutputStreamOperator<OUT, O>(this);
http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
index 87f2287..08fd19b 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
@@ -434,7 +434,7 @@ public abstract class StreamExecutionEnvironment {
DataStreamSource<OUT> returnStream = new DataStreamSource<OUT>(this, sourceName,
outTypeInfo, isParallel);
- jobGraphBuilder.addStreamVertex(returnStream.getId(), new SourceInvokable<OUT>(function),
+ jobGraphBuilder.addSourceVertex(returnStream.getId(), new SourceInvokable<OUT>(function),
null, outTypeInfo, sourceName, dop);
return returnStream;
http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/ChainableInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/ChainableInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/ChainableInvokable.java
new file mode 100644
index 0000000..373b4e8
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/ChainableInvokable.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable;
+
+import org.apache.flink.api.common.functions.Function;
+import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
+import org.apache.flink.util.Collector;
+
+public abstract class ChainableInvokable<IN, OUT> extends StreamInvokable<IN, OUT> implements
+ Collector<IN> {
+
+ private static final long serialVersionUID = 1L;
+
+ public ChainableInvokable(Function userFunction) {
+ super(userFunction);
+ }
+
+ public void setup(Collector<OUT> collector, StreamRecordSerializer<IN> inSerializer) {
+ this.collector = collector;
+ this.inSerializer = inSerializer;
+ this.objectSerializer = inSerializer.getObjectSerializer();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/SinkInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/SinkInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/SinkInvokable.java
index 74591a8..13a6ba1 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/SinkInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/SinkInvokable.java
@@ -19,7 +19,7 @@ package org.apache.flink.streaming.api.invokable;
import org.apache.flink.streaming.api.function.sink.SinkFunction;
-public class SinkInvokable<IN> extends StreamInvokable<IN, IN> {
+public class SinkInvokable<IN> extends ChainableInvokable<IN, IN> {
private static final long serialVersionUID = 1L;
private SinkFunction<IN> sinkFunction;
@@ -38,7 +38,13 @@ public class SinkInvokable<IN> extends StreamInvokable<IN, IN> {
@Override
protected void callUserFunction() throws Exception {
- sinkFunction.invoke((IN) nextRecord.getObject());
+ sinkFunction.invoke(nextObject);
+ }
+
+ @Override
+ public void collect(IN record) {
+ nextObject = copy(record);
+ callUserFunctionAndLogException();
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java
index 87ad4e0..25e9221 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java
@@ -37,7 +37,7 @@ import org.slf4j.LoggerFactory;
/**
* The StreamInvokable represents the base class for all invokables in the
* streaming topology.
- *
+ *
* @param <OUT>
* The output type of the invokable
*/
@@ -52,6 +52,7 @@ public abstract class StreamInvokable<IN, OUT> implements Serializable {
protected StreamRecordSerializer<IN> inSerializer;
protected TypeSerializer<IN> objectSerializer;
protected StreamRecord<IN> nextRecord;
+ protected IN nextObject;
protected boolean isMutable;
protected Collector<OUT> collector;
@@ -92,7 +93,13 @@ public abstract class StreamInvokable<IN, OUT> implements Serializable {
protected StreamRecord<IN> readNext() {
this.nextRecord = inSerializer.createInstance();
try {
- return nextRecord = recordIterator.next(nextRecord);
+ nextRecord = recordIterator.next(nextRecord);
+ try {
+ nextObject = nextRecord.getObject();
+ } catch (NullPointerException e) {
+ // end of stream
+ }
+ return nextRecord;
} catch (IOException e) {
throw new RuntimeException("Could not read next record.");
}
@@ -135,10 +142,14 @@ public abstract class StreamInvokable<IN, OUT> implements Serializable {
* RichFunction class
*
*/
- public void close() throws Exception {
+ public void close() {
isRunning = false;
collector.close();
- FunctionUtils.closeFunction(userFunction);
+ try {
+ FunctionUtils.closeFunction(userFunction);
+ } catch (Exception e) {
+ throw new RuntimeException("Error when closing the function: " + e.getMessage());
+ }
}
public void setRuntimeContext(RuntimeContext t) {
@@ -148,4 +159,8 @@ public abstract class StreamInvokable<IN, OUT> implements Serializable {
protected IN copy(IN record) {
return objectSerializer.copy(record);
}
+
+ public boolean isChainable() {
+ return this instanceof ChainableInvokable;
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/CounterInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/CounterInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/CounterInvokable.java
index 0267253..3fc314c 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/CounterInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/CounterInvokable.java
@@ -17,9 +17,9 @@
package org.apache.flink.streaming.api.invokable.operator;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.invokable.ChainableInvokable;
-public class CounterInvokable<IN> extends StreamInvokable<IN, Long> {
+public class CounterInvokable<IN> extends ChainableInvokable<IN, Long> {
private static final long serialVersionUID = 1L;
Long count = 0L;
@@ -34,4 +34,10 @@ public class CounterInvokable<IN> extends StreamInvokable<IN, Long> {
collector.collect(++count);
}
}
+
+ @Override
+ public void collect(IN record) {
+ collector.collect(++count);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FilterInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FilterInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FilterInvokable.java
index 48b8ad0..0c8298e 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FilterInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FilterInvokable.java
@@ -18,9 +18,9 @@
package org.apache.flink.streaming.api.invokable.operator;
import org.apache.flink.api.common.functions.FilterFunction;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.invokable.ChainableInvokable;
-public class FilterInvokable<IN> extends StreamInvokable<IN, IN> {
+public class FilterInvokable<IN> extends ChainableInvokable<IN, IN> {
private static final long serialVersionUID = 1L;
@@ -36,14 +36,20 @@ public class FilterInvokable<IN> extends StreamInvokable<IN, IN> {
public void invoke() throws Exception {
while (readNext() != null) {
callUserFunctionAndLogException();
- if (collect) {
- collector.collect(nextRecord.getObject());
- }
}
}
@Override
protected void callUserFunction() throws Exception {
- collect = filterFunction.filter(copy(nextRecord.getObject()));
+ collect = filterFunction.filter(copy(nextObject));
+ if (collect) {
+ collector.collect(nextObject);
+ }
+ }
+
+ @Override
+ public void collect(IN record) {
+ nextObject = copy(record);
+ callUserFunctionAndLogException();
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FlatMapInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FlatMapInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FlatMapInvokable.java
index 8ff78eb..2a4081b 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FlatMapInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/FlatMapInvokable.java
@@ -18,9 +18,9 @@
package org.apache.flink.streaming.api.invokable.operator;
import org.apache.flink.api.common.functions.FlatMapFunction;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.invokable.ChainableInvokable;
-public class FlatMapInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
+public class FlatMapInvokable<IN, OUT> extends ChainableInvokable<IN, OUT> {
private static final long serialVersionUID = 1L;
private FlatMapFunction<IN, OUT> flatMapper;
@@ -39,7 +39,13 @@ public class FlatMapInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
@Override
protected void callUserFunction() throws Exception {
- flatMapper.flatMap(nextRecord.getObject(), collector);
+ flatMapper.flatMap(nextObject, collector);
+ }
+
+ @Override
+ public void collect(IN record) {
+ nextObject = copy(record);
+ callUserFunctionAndLogException();
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java
index fdcf520..84258d6 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java
@@ -40,7 +40,7 @@ public class GroupedReduceInvokable<IN> extends StreamReduceInvokable<IN> {
protected void reduce() throws Exception {
Object key = nextRecord.getKey(keySelector);
currentValue = values.get(key);
- nextValue = nextRecord.getObject();
+ nextValue = nextObject;
if (currentValue != null) {
callUserFunctionAndLogException();
values.put(key, reduced);
@@ -56,4 +56,9 @@ public class GroupedReduceInvokable<IN> extends StreamReduceInvokable<IN> {
reduced = reducer.reduce(currentValue, nextValue);
}
+ @Override
+ public boolean isChainable() {
+ return false;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/MapInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/MapInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/MapInvokable.java
index 6be96ec..7c8e577 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/MapInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/MapInvokable.java
@@ -18,9 +18,9 @@
package org.apache.flink.streaming.api.invokable.operator;
import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.invokable.ChainableInvokable;
-public class MapInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
+public class MapInvokable<IN, OUT> extends ChainableInvokable<IN, OUT> {
private static final long serialVersionUID = 1L;
private MapFunction<IN, OUT> mapper;
@@ -39,6 +39,12 @@ public class MapInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
@Override
protected void callUserFunction() throws Exception {
- collector.collect(mapper.map(nextRecord.getObject()));
+ collector.collect(mapper.map(nextObject));
+ }
+
+ @Override
+ public void collect(IN record) {
+ nextObject = copy(record);
+ callUserFunctionAndLogException();
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceInvokable.java
index 5f5cb12..e1a56cc 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceInvokable.java
@@ -18,9 +18,9 @@
package org.apache.flink.streaming.api.invokable.operator;
import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.invokable.ChainableInvokable;
-public class StreamReduceInvokable<IN> extends StreamInvokable<IN, IN> {
+public class StreamReduceInvokable<IN> extends ChainableInvokable<IN, IN> {
private static final long serialVersionUID = 1L;
protected ReduceFunction<IN> reducer;
@@ -41,13 +41,15 @@ public class StreamReduceInvokable<IN> extends StreamInvokable<IN, IN> {
}
protected void reduce() throws Exception {
- nextValue = nextRecord.getObject();
callUserFunctionAndLogException();
}
@Override
protected void callUserFunction() throws Exception {
+
+ nextValue = nextObject;
+
if (currentValue != null) {
currentValue = reducer.reduce(currentValue, nextValue);
} else {
@@ -57,4 +59,11 @@ public class StreamReduceInvokable<IN> extends StreamInvokable<IN, IN> {
collector.collect(currentValue);
}
+
+ @Override
+ public void collect(IN record) {
+ nextObject = copy(record);
+ callUserFunctionAndLogException();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowInvokable.java
index 03219b7..93f597f 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowInvokable.java
@@ -175,7 +175,7 @@ public class CoWindowInvokable<IN1, IN2, OUT> extends CoInvokable<IN1, IN2, OUT>
}
@Override
- public void close() throws Exception {
+ public void close() {
if (!window.miniBatchEnd()) {
callUserFunctionAndLogException();
}
http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
index 61a6eb4..457f3f8 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
@@ -17,26 +17,30 @@
package org.apache.flink.streaming.api.streamvertex;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
import org.apache.flink.runtime.plugable.SerializationDelegate;
import org.apache.flink.streaming.api.StreamConfig;
+import org.apache.flink.streaming.api.collector.CollectorWrapper;
import org.apache.flink.streaming.api.collector.DirectedOutputWrapper;
import org.apache.flink.streaming.api.collector.OutputSelector;
-import org.apache.flink.streaming.api.collector.StreamOutputWrapper;
import org.apache.flink.streaming.api.collector.StreamOutput;
+import org.apache.flink.streaming.api.collector.StreamOutputWrapper;
+import org.apache.flink.streaming.api.invokable.ChainableInvokable;
import org.apache.flink.streaming.api.invokable.StreamInvokable;
import org.apache.flink.streaming.api.streamrecord.StreamRecord;
import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
import org.apache.flink.streaming.io.StreamRecordWriter;
import org.apache.flink.streaming.partitioner.StreamPartitioner;
+import org.apache.flink.util.Collector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
-
public class OutputHandler<OUT> {
private static final Logger LOG = LoggerFactory.getLogger(OutputHandler.class);
@@ -44,17 +48,21 @@ public class OutputHandler<OUT> {
private StreamConfig configuration;
private List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> outputs;
- private StreamOutputWrapper<OUT> collector;
- private long bufferTimeout;
+ private Collector<OUT> endCollector;
TypeInformation<OUT> outTypeInfo = null;
StreamRecordSerializer<OUT> outSerializer = null;
SerializationDelegate<StreamRecord<OUT>> outSerializationDelegate = null;
+ public List<ChainableInvokable<?, ?>> chainedInvokables;
+
+ private int numberOfChainedTasks;
+
public OutputHandler(StreamVertex<?, OUT> streamComponent) {
this.streamVertex = streamComponent;
this.outputs = new LinkedList<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>>();
this.configuration = new StreamConfig(streamComponent.getTaskConfiguration());
+ this.chainedInvokables = new ArrayList<ChainableInvokable<?, ?>>();
try {
setConfigOutputs();
@@ -62,6 +70,7 @@ public class OutputHandler<OUT> {
throw new StreamVertexException("Cannot register outputs for "
+ streamComponent.getClass().getSimpleName(), e);
}
+
}
public List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> getOutputs() {
@@ -69,18 +78,42 @@ public class OutputHandler<OUT> {
}
private void setConfigOutputs() {
- setSerializers();
- setCollector();
+ numberOfChainedTasks = configuration.getNumberofChainedTasks();
+ endCollector = createChainedOutputs(0);
+ }
- int numberOfOutputs = configuration.getNumberOfOutputs();
- bufferTimeout = configuration.getBufferTimeout();
+ @SuppressWarnings("unchecked")
+ private Collector<OUT> createChainedOutputs(int chainedTaskIndex) {
- for (int i = 0; i < numberOfOutputs; i++) {
- setPartitioner(i, outputs);
+ if (numberOfChainedTasks == chainedTaskIndex) {
+ return createEndCollector();
+ } else {
+ CollectorWrapper<OUT> chainedCollector = new CollectorWrapper<OUT>();
+
+ @SuppressWarnings("rawtypes")
+ ChainableInvokable chainableInvokable = configuration.getChainedInvokable(
+ chainedTaskIndex, streamVertex.getUserCodeClassLoader());
+
+ chainableInvokable.setup(
+ createChainedOutputs(chainedTaskIndex + 1),
+ configuration.getChainedInSerializer(chainedTaskIndex,
+ streamVertex.getUserCodeClassLoader()));
+
+ chainedInvokables.add(chainableInvokable);
+
+ chainedCollector.addChainedOutput((Collector<OUT>) chainableInvokable);
+
+ return chainedCollector;
}
+
}
- private StreamOutputWrapper<OUT> setCollector() {
+ private Collector<OUT> createEndCollector() {
+
+ setSerializers();
+
+ StreamOutputWrapper<OUT> collector;
+
if (streamVertex.configuration.getDirectedEmit()) {
OutputSelector<OUT> outputSelector = streamVertex.configuration
.getOutputSelector(streamVertex.userClassLoader);
@@ -91,11 +124,17 @@ public class OutputHandler<OUT> {
collector = new StreamOutputWrapper<OUT>(streamVertex.getInstanceID(),
outSerializationDelegate);
}
+
+ int numberOfOutputs = configuration.getNumberOfOutputs();
+ for (int i = 0; i < numberOfOutputs; i++) {
+ collector = (StreamOutputWrapper<OUT>) setPartitioner(i, collector);
+ }
+
return collector;
}
- public StreamOutputWrapper<OUT> getCollector() {
- return collector;
+ public Collector<OUT> getCollector() {
+ return endCollector;
}
void setSerializers() {
@@ -106,8 +145,7 @@ public class OutputHandler<OUT> {
}
}
- void setPartitioner(int outputNumber,
- List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> outputs) {
+ Collector<OUT> setPartitioner(int outputNumber, StreamOutputWrapper<OUT> endCollector) {
StreamPartitioner<OUT> outputPartitioner = null;
try {
@@ -121,6 +159,8 @@ public class OutputHandler<OUT> {
RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output;
+ long bufferTimeout = configuration.getBufferTimeout();
+
if (bufferTimeout >= 0) {
output = new StreamRecordWriter<SerializationDelegate<StreamRecord<OUT>>>(streamVertex
.getEnvironment().getWriter(outputNumber), outputPartitioner, bufferTimeout);
@@ -139,18 +179,20 @@ public class OutputHandler<OUT> {
}
outputs.add(output);
- List<String> outputName = configuration.getOutputName(outputNumber);
+ List<String> outputNames = configuration.getOutputNames(outputNumber);
boolean isSelectAllOutput = configuration.getSelectAll(outputNumber);
- if (collector != null) {
- collector
- .addOutput(new StreamOutput<OUT>(output, isSelectAllOutput ? null : outputName));
+ if (endCollector != null) {
+ endCollector.addOutput(new StreamOutput<OUT>(output, isSelectAllOutput ? null
+ : outputNames));
}
if (LOG.isTraceEnabled()) {
LOG.trace("Partitioner set: {} with {} outputs for {}", outputPartitioner.getClass()
.getSimpleName(), outputNumber, streamVertex.getClass().getSimpleName());
}
+
+ return endCollector;
}
public void flushOutputs() throws IOException, InterruptedException {
http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertex.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertex.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertex.java
index 93e290f..994b1fa 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertex.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertex.java
@@ -22,6 +22,7 @@ import java.util.Map;
import org.apache.flink.runtime.execution.Environment;
import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
import org.apache.flink.streaming.api.StreamConfig;
+import org.apache.flink.streaming.api.invokable.ChainableInvokable;
import org.apache.flink.streaming.api.invokable.StreamInvokable;
import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
import org.apache.flink.streaming.io.CoReaderIterator;
@@ -35,11 +36,8 @@ public class StreamVertex<IN, OUT> extends AbstractInvokable implements StreamTa
protected StreamConfig configuration;
protected int instanceID;
- protected String name;
private static int numVertices = 0;
- protected String functionName;
-
private InputHandler<IN> inputHandler;
protected OutputHandler<OUT> outputHandler;
private StreamInvokable<IN, OUT> userInvokable;
@@ -70,17 +68,26 @@ public class StreamVertex<IN, OUT> extends AbstractInvokable implements StreamTa
protected void initialize() {
this.userClassLoader = getUserCodeClassLoader();
this.configuration = new StreamConfig(getTaskConfiguration());
- this.name = configuration.getVertexName();
- this.functionName = configuration.getFunctionName();
this.states = configuration.getOperatorStates(userClassLoader);
- this.context = createRuntimeContext(name, this.states);
+ this.context = createRuntimeContext(getEnvironment().getTaskName(), this.states);
}
protected <T> void invokeUserFunction(StreamInvokable<?, T> userInvokable) throws Exception {
userInvokable.setRuntimeContext(context);
userInvokable.open(getTaskConfiguration());
+
+ for (ChainableInvokable<?, ?> invokable : outputHandler.chainedInvokables) {
+ invokable.setRuntimeContext(context);
+ invokable.open(getTaskConfiguration());
+ }
+
userInvokable.invoke();
userInvokable.close();
+
+ for (ChainableInvokable<?, ?> invokable : outputHandler.chainedInvokables) {
+ invokable.close();
+ }
+
}
public void setInputsOutputs() {
@@ -94,14 +101,15 @@ public class StreamVertex<IN, OUT> extends AbstractInvokable implements StreamTa
}
public String getName() {
- return name;
+ return getEnvironment().getTaskName();
}
public int getInstanceID() {
return instanceID;
}
- public StreamingRuntimeContext createRuntimeContext(String taskName, Map<String, OperatorState<?>> states) {
+ public StreamingRuntimeContext createRuntimeContext(String taskName,
+ Map<String, OperatorState<?>> states) {
Environment env = getEnvironment();
return new StreamingRuntimeContext(taskName, env, getUserCodeClassLoader(), states);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/26535c48/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/iteration/IterateExample.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/iteration/IterateExample.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/iteration/IterateExample.java
index a5dc68a..998e818 100644
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/iteration/IterateExample.java
+++ b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/iteration/IterateExample.java
@@ -76,8 +76,7 @@ public class IterateExample {
// apply the step function to add new random value to the tuple and to
// increment the counter and split the output with the output selector
- SplitDataStream<Tuple2<Double, Integer>> step = it.map(new Step()).shuffle()
- .split(new MySelector());
+ SplitDataStream<Tuple2<Double, Integer>> step = it.map(new Step()).split(new MySelector());
// close the iteration by selecting the tuples that were directed to the
// 'iterate' channel in the output selector
[6/8] flink git commit: [streaming] JobGraphbuilder separated to
StreamGraph and StreamingJobGraphGenerator
Posted by mb...@apache.org.
[streaming] JobGraphbuilder separated to StreamGraph and StreamingJobGraphGenerator
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/e3b608ce
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/e3b608ce
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/e3b608ce
Branch: refs/heads/master
Commit: e3b608ce20f2c06b6c9ce8f231b6f53ab73f1de5
Parents: 7dbb55e
Author: Gyula Fora <gy...@apache.org>
Authored: Wed Jan 21 00:34:11 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Wed Jan 21 16:06:34 2015 +0100
----------------------------------------------------------------------
.../flink/streaming/api/JobGraphBuilder.java | 675 -------------------
.../flink/streaming/api/StreamConfig.java | 28 +-
.../apache/flink/streaming/api/StreamGraph.java | 545 +++++++++++++++
.../api/StreamingJobGraphGenerator.java | 286 ++++++++
.../api/datastream/ConnectedDataStream.java | 8 +-
.../streaming/api/datastream/DataStream.java | 18 +-
.../api/datastream/IterativeDataStream.java | 4 +-
.../datastream/SingleOutputStreamOperator.java | 20 +-
.../temporaloperator/StreamCrossOperator.java | 4 +-
.../temporaloperator/StreamJoinOperator.java | 2 +-
.../api/environment/LocalStreamEnvironment.java | 4 +-
.../environment/RemoteStreamEnvironment.java | 4 +-
.../environment/StreamContextEnvironment.java | 4 +-
.../environment/StreamExecutionEnvironment.java | 18 +-
.../operator/GroupedReduceInvokable.java | 1 -
.../operator/GroupedWindowInvokable.java | 3 +-
.../api/invokable/operator/WindowInvokable.java | 3 +-
.../api/streamvertex/OutputHandler.java | 64 +-
.../streaming/util/TestStreamEnvironment.java | 2 +-
.../api/scala/StreamCrossOperator.scala | 4 +-
.../api/scala/StreamJoinOperator.scala | 2 +-
21 files changed, 914 insertions(+), 785 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
deleted file mode 100644
index 6ae97c9..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
+++ /dev/null
@@ -1,675 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.flink.api.common.io.InputFormat;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.jobgraph.AbstractJobVertex;
-import org.apache.flink.runtime.jobgraph.DistributionPattern;
-import org.apache.flink.runtime.jobgraph.JobGraph;
-import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
-import org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup;
-import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
-import org.apache.flink.streaming.api.collector.OutputSelector;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
-import org.apache.flink.streaming.api.invokable.StreamInvokable.ChainingStrategy;
-import org.apache.flink.streaming.api.invokable.operator.co.CoInvokable;
-import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
-import org.apache.flink.streaming.api.streamvertex.CoStreamVertex;
-import org.apache.flink.streaming.api.streamvertex.StreamIterationHead;
-import org.apache.flink.streaming.api.streamvertex.StreamIterationTail;
-import org.apache.flink.streaming.api.streamvertex.StreamVertex;
-import org.apache.flink.streaming.partitioner.StreamPartitioner;
-import org.apache.flink.streaming.partitioner.StreamPartitioner.PartitioningStrategy;
-import org.apache.flink.streaming.state.OperatorState;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Object for building Apache Flink stream processing job graphs
- */
-public class JobGraphBuilder {
-
- private static final Logger LOG = LoggerFactory.getLogger(JobGraphBuilder.class);
- private final static String DEAFULT_JOB_NAME = "Streaming Job";
- private JobGraph jobGraph;
-
- private boolean chaining = true;
-
- // Graph attributes
- private Map<String, AbstractJobVertex> streamVertices;
- private Map<String, Integer> vertexParallelism;
- private Map<String, Long> bufferTimeout;
- private Map<String, List<String>> outEdgeList;
- private Map<String, List<Integer>> outEdgeIndex;
- private Map<String, List<List<String>>> outEdgeNames;
- private Map<String, List<Boolean>> outEdgeSelectAll;
- private Map<String, List<String>> inEdgeList;
- private Map<String, List<StreamPartitioner<?>>> outPartitioning;
- private Map<String, String> operatorNames;
- private Map<String, StreamInvokable<?, ?>> invokableObjects;
- private Map<String, StreamRecordSerializer<?>> typeSerializersIn1;
- private Map<String, StreamRecordSerializer<?>> typeSerializersIn2;
- private Map<String, StreamRecordSerializer<?>> typeSerializersOut1;
- private Map<String, StreamRecordSerializer<?>> typeSerializersOut2;
- private Map<String, byte[]> outputSelectors;
- private Map<String, Class<? extends AbstractInvokable>> vertexClasses;
- private Map<String, Integer> iterationIds;
- private Map<Integer, String> iterationIDtoHeadName;
- private Map<Integer, String> iterationIDtoTailName;
- private Map<String, Integer> iterationTailCount;
- private Map<String, Long> iterationWaitTime;
- private Map<String, Map<String, OperatorState<?>>> operatorStates;
- private Map<String, InputFormat<String, ?>> inputFormatList;
- private Map<String, Map<String, StreamConfig>> chainedConfigs;
- private Map<String, StreamConfig> vertexConfigs;
-
- private Set<String> sources;
- private Set<String> builtNodes;
-
- /**
- * Creates an new {@link JobGraph} with the given name. A JobGraph is a DAG
- * and consists of sources, tasks (intermediate vertices) and sinks.
- */
- public JobGraphBuilder() {
-
- initGraph();
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("JobGraph created");
- }
- }
-
- public void initGraph() {
- streamVertices = new HashMap<String, AbstractJobVertex>();
- vertexParallelism = new HashMap<String, Integer>();
- bufferTimeout = new HashMap<String, Long>();
- outEdgeList = new HashMap<String, List<String>>();
- outEdgeIndex = new HashMap<String, List<Integer>>();
- outEdgeNames = new HashMap<String, List<List<String>>>();
- outEdgeSelectAll = new HashMap<String, List<Boolean>>();
- inEdgeList = new HashMap<String, List<String>>();
- outPartitioning = new HashMap<String, List<StreamPartitioner<?>>>();
- operatorNames = new HashMap<String, String>();
- invokableObjects = new HashMap<String, StreamInvokable<?, ?>>();
- typeSerializersIn1 = new HashMap<String, StreamRecordSerializer<?>>();
- typeSerializersIn2 = new HashMap<String, StreamRecordSerializer<?>>();
- typeSerializersOut1 = new HashMap<String, StreamRecordSerializer<?>>();
- typeSerializersOut2 = new HashMap<String, StreamRecordSerializer<?>>();
- outputSelectors = new HashMap<String, byte[]>();
- vertexClasses = new HashMap<String, Class<? extends AbstractInvokable>>();
- iterationIds = new HashMap<String, Integer>();
- iterationIDtoHeadName = new HashMap<Integer, String>();
- iterationIDtoTailName = new HashMap<Integer, String>();
- iterationTailCount = new HashMap<String, Integer>();
- iterationWaitTime = new HashMap<String, Long>();
- operatorStates = new HashMap<String, Map<String, OperatorState<?>>>();
- inputFormatList = new HashMap<String, InputFormat<String, ?>>();
- chainedConfigs = new HashMap<String, Map<String, StreamConfig>>();
- vertexConfigs = new HashMap<String, StreamConfig>();
-
- sources = new HashSet<String>();
- builtNodes = new HashSet<String>();
- }
-
- /**
- * Adds a vertex to the streaming JobGraph with the given parameters
- *
- * @param vertexName
- * Name of the vertex
- * @param invokableObject
- * User defined operator
- * @param inTypeInfo
- * Input type for serialization
- * @param outTypeInfo
- * Output type for serialization
- * @param operatorName
- * Operator type
- * @param parallelism
- * Number of parallel instances created
- */
- public <IN, OUT> void addStreamVertex(String vertexName,
- StreamInvokable<IN, OUT> invokableObject, TypeInformation<IN> inTypeInfo,
- TypeInformation<OUT> outTypeInfo, String operatorName, int parallelism) {
-
- addVertex(vertexName, StreamVertex.class, invokableObject, operatorName, parallelism);
-
- StreamRecordSerializer<IN> inSerializer = inTypeInfo != null ? new StreamRecordSerializer<IN>(
- inTypeInfo) : null;
- StreamRecordSerializer<OUT> outSerializer = outTypeInfo != null ? new StreamRecordSerializer<OUT>(
- outTypeInfo) : null;
-
- addTypeSerializers(vertexName, inSerializer, null, outSerializer, null);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Vertex: {}", vertexName);
- }
- }
-
- public <IN, OUT> void addSourceVertex(String vertexName,
- StreamInvokable<IN, OUT> invokableObject, TypeInformation<IN> inTypeInfo,
- TypeInformation<OUT> outTypeInfo, String operatorName, int parallelism) {
- addStreamVertex(vertexName, invokableObject, inTypeInfo, outTypeInfo, operatorName,
- parallelism);
- sources.add(vertexName);
- }
-
- /**
- * Adds a vertex for the iteration head to the {@link JobGraph}. The
- * iterated values will be fed from this vertex back to the graph.
- *
- * @param vertexName
- * Name of the vertex
- * @param iterationHead
- * Id of the iteration head
- * @param iterationID
- * ID of iteration for multiple iterations
- * @param parallelism
- * Number of parallel instances created
- * @param waitTime
- * Max wait time for next record
- */
- public void addIterationHead(String vertexName, String iterationHead, Integer iterationID,
- int parallelism, long waitTime) {
-
- addVertex(vertexName, StreamIterationHead.class, null, null, parallelism);
-
- chaining = false;
-
- iterationIds.put(vertexName, iterationID);
- iterationIDtoHeadName.put(iterationID, vertexName);
-
- setSerializersFrom(iterationHead, vertexName);
-
- setEdge(vertexName, iterationHead, outPartitioning
- .get(inEdgeList.get(iterationHead).get(0)).get(0), 0, new ArrayList<String>(),
- false);
-
- iterationWaitTime.put(iterationIDtoHeadName.get(iterationID), waitTime);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("ITERATION SOURCE: {}", vertexName);
- }
-
- sources.add(vertexName);
- }
-
- /**
- * Adds a vertex for the iteration tail to the {@link JobGraph}. The values
- * intended to be iterated will be sent to this sink from the iteration
- * head.
- *
- * @param vertexName
- * Name of the vertex
- * @param iterationTail
- * Id of the iteration tail
- * @param iterationID
- * ID of iteration for mulitple iterations
- * @param parallelism
- * Number of parallel instances created
- * @param waitTime
- * Max waiting time for next record
- */
- public void addIterationTail(String vertexName, String iterationTail, Integer iterationID,
- int parallelism, long waitTime) {
-
- if (bufferTimeout.get(iterationTail) == 0) {
- throw new RuntimeException("Buffer timeout 0 at iteration tail is not supported.");
- }
-
- addVertex(vertexName, StreamIterationTail.class, null, null, parallelism);
-
- iterationIds.put(vertexName, iterationID);
- iterationIDtoTailName.put(iterationID, vertexName);
-
- setSerializersFrom(iterationTail, vertexName);
- iterationWaitTime.put(iterationIDtoTailName.get(iterationID), waitTime);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("ITERATION SINK: {}", vertexName);
- }
-
- }
-
- public <IN1, IN2, OUT> void addCoTask(String vertexName,
- CoInvokable<IN1, IN2, OUT> taskInvokableObject, TypeInformation<IN1> in1TypeInfo,
- TypeInformation<IN2> in2TypeInfo, TypeInformation<OUT> outTypeInfo,
- String operatorName, int parallelism) {
-
- addVertex(vertexName, CoStreamVertex.class, taskInvokableObject, operatorName, parallelism);
-
- addTypeSerializers(vertexName, new StreamRecordSerializer<IN1>(in1TypeInfo),
- new StreamRecordSerializer<IN2>(in2TypeInfo), new StreamRecordSerializer<OUT>(
- outTypeInfo), null);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("CO-TASK: {}", vertexName);
- }
- }
-
- /**
- * Sets vertex parameters in the JobGraph
- *
- * @param vertexName
- * Name of the vertex
- * @param vertexClass
- * The class of the vertex
- * @param invokableObjectject
- * The user defined invokable object
- * @param operatorName
- * Type of the user defined operator
- * @param parallelism
- * Number of parallel instances created
- */
- private void addVertex(String vertexName, Class<? extends AbstractInvokable> vertexClass,
- StreamInvokable<?, ?> invokableObject, String operatorName, int parallelism) {
-
- vertexClasses.put(vertexName, vertexClass);
- setParallelism(vertexName, parallelism);
- invokableObjects.put(vertexName, invokableObject);
- operatorNames.put(vertexName, operatorName);
- outEdgeList.put(vertexName, new ArrayList<String>());
- outEdgeIndex.put(vertexName, new ArrayList<Integer>());
- outEdgeNames.put(vertexName, new ArrayList<List<String>>());
- outEdgeSelectAll.put(vertexName, new ArrayList<Boolean>());
- inEdgeList.put(vertexName, new ArrayList<String>());
- outPartitioning.put(vertexName, new ArrayList<StreamPartitioner<?>>());
- iterationTailCount.put(vertexName, 0);
- }
-
- private void addTypeSerializers(String vertexName, StreamRecordSerializer<?> in1,
- StreamRecordSerializer<?> in2, StreamRecordSerializer<?> out1,
- StreamRecordSerializer<?> out2) {
- typeSerializersIn1.put(vertexName, in1);
- typeSerializersIn2.put(vertexName, in2);
- typeSerializersOut1.put(vertexName, out1);
- typeSerializersOut2.put(vertexName, out2);
- }
-
- private List<Tuple2<String, String>> createChain(String startNode, String current) {
-
- if (!builtNodes.contains(startNode)) {
-
- List<Tuple2<String, String>> transitiveOutEdges = new ArrayList<Tuple2<String, String>>();
- List<String> chainableOutputs = new ArrayList<String>();
- List<String> nonChainableOutputs = new ArrayList<String>();
-
- for (String outName : outEdgeList.get(current)) {
- if (isChainable(current, outName)) {
- chainableOutputs.add(outName);
- } else {
- nonChainableOutputs.add(outName);
- }
-
- }
-
- for (String chainable : chainableOutputs) {
- transitiveOutEdges.addAll(createChain(startNode, chainable));
- }
-
- for (String nonChainable : nonChainableOutputs) {
- transitiveOutEdges.add(new Tuple2<String, String>(current, nonChainable));
- transitiveOutEdges.addAll(createChain(nonChainable, nonChainable));
- }
-
- StreamConfig config = current.equals(startNode) ? createProcessingVertex(startNode)
- : new StreamConfig(new Configuration());
-
- setVertexConfig(current, config, chainableOutputs, nonChainableOutputs);
-
- if (current.equals(startNode)) {
-
- config.setChainStart();
- config.setRecordWriterOrder(transitiveOutEdges);
-
- for (Tuple2<String, String> edge : transitiveOutEdges) {
- connect(startNode, edge);
- }
-
- vertexConfigs.get(startNode).setTransitiveChainedTaskConfigs(
- chainedConfigs.get(startNode));
-
- } else {
-
- Map<String, StreamConfig> chainedConfs = chainedConfigs.get(startNode);
-
- if (chainedConfs == null) {
- chainedConfigs.put(startNode, new HashMap<String, StreamConfig>());
- }
- chainedConfigs.get(startNode).put(current, config);
- }
-
- return transitiveOutEdges;
-
- } else {
- return new ArrayList<Tuple2<String, String>>();
- }
- }
-
- private StreamConfig createProcessingVertex(String vertexName) {
-
- AbstractJobVertex vertex = new AbstractJobVertex(vertexName);
-
- this.jobGraph.addVertex(vertex);
-
- int parallelism = vertexParallelism.get(vertexName);
-
- vertex.setInvokableClass(vertexClasses.get(vertexName));
- vertex.setParallelism(parallelism);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Parallelism set: {} for {}", parallelism, vertexName);
- }
-
- if (inputFormatList.containsKey(vertexName)) {
- vertex.setInputSplitSource(inputFormatList.get(vertexName));
- }
-
- streamVertices.put(vertexName, vertex);
- builtNodes.add(vertexName);
-
- return new StreamConfig(vertex.getConfiguration());
- }
-
- private void setVertexConfig(String vertexName, StreamConfig config,
- List<String> chainableOutputs, List<String> nonChainableOutputs) {
-
- StreamInvokable<?, ?> invokableObject = invokableObjects.get(vertexName);
- byte[] outputSelector = outputSelectors.get(vertexName);
- Class<? extends AbstractInvokable> vertexClass = vertexClasses.get(vertexName);
- Map<String, OperatorState<?>> state = operatorStates.get(vertexName);
-
- config.setVertexName(vertexName);
-
- config.setBufferTimeout(bufferTimeout.get(vertexName));
-
- config.setTypeSerializerIn1(typeSerializersIn1.get(vertexName));
- config.setTypeSerializerIn2(typeSerializersIn2.get(vertexName));
- config.setTypeSerializerOut1(typeSerializersOut1.get(vertexName));
- config.setTypeSerializerOut2(typeSerializersOut2.get(vertexName));
-
- config.setUserInvokable(invokableObject);
- config.setOutputSelector(outputSelector);
- config.setOperatorStates(state);
-
- config.setNumberOfOutputs(nonChainableOutputs.size());
- config.setOutputs(nonChainableOutputs);
- config.setChainedOutputs(chainableOutputs);
-
- if (vertexClass.equals(StreamIterationHead.class)
- || vertexClass.equals(StreamIterationTail.class)) {
- config.setIterationId(iterationIds.get(vertexName));
- config.setIterationWaitTime(iterationWaitTime.get(vertexName));
- }
-
- vertexConfigs.put(vertexName, config);
- }
-
- private boolean isChainable(String vertexName, String outName) {
- return inEdgeList.get(outName).size() == 1
- && invokableObjects.get(outName) != null
- && outputSelectors.get(vertexName) == null
- && invokableObjects.get(outName).getChainingStrategy() == ChainingStrategy.ALWAYS
- && (invokableObjects.get(vertexName).getChainingStrategy() == ChainingStrategy.HEAD || invokableObjects
- .get(vertexName).getChainingStrategy() == ChainingStrategy.ALWAYS)
- && outPartitioning.get(vertexName)
- .get(outEdgeList.get(vertexName).indexOf(outName)).getStrategy() == PartitioningStrategy.FORWARD
- && vertexParallelism.get(vertexName) == vertexParallelism.get(outName) && chaining;
- }
-
- private <T> void connect(String headOfChain, Tuple2<String, String> edge) {
-
- String upStreamVertexName = edge.f0;
- String downStreamVertexName = edge.f1;
-
- int outputIndex = outEdgeList.get(upStreamVertexName).indexOf(downStreamVertexName);
-
- AbstractJobVertex headVertex = streamVertices.get(headOfChain);
- AbstractJobVertex downStreamVertex = streamVertices.get(downStreamVertexName);
-
- StreamConfig downStreamConfig = new StreamConfig(downStreamVertex.getConfiguration());
- StreamConfig upStreamConfig = new StreamConfig(headVertex.getConfiguration());
-
- List<Integer> outEdgeIndexList = outEdgeIndex.get(upStreamVertexName);
- int numOfInputs = downStreamConfig.getNumberOfInputs();
-
- downStreamConfig.setInputIndex(numOfInputs++, outEdgeIndexList.get(outputIndex));
- downStreamConfig.setNumberOfInputs(numOfInputs);
-
- StreamPartitioner<?> partitionerObject = outPartitioning.get(upStreamVertexName).get(
- outputIndex);
-
- upStreamConfig.setPartitioner(downStreamVertexName, partitionerObject);
-
- if (partitionerObject.getStrategy() == PartitioningStrategy.FORWARD) {
- downStreamVertex.connectNewDataSetAsInput(headVertex, DistributionPattern.POINTWISE);
- } else {
- downStreamVertex.connectNewDataSetAsInput(headVertex, DistributionPattern.ALL_TO_ALL);
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("CONNECTED: {} - {} -> {}", partitionerObject.getClass().getSimpleName(),
- headOfChain, downStreamVertexName);
- }
-
- upStreamConfig.setOutputNames(downStreamVertexName, outEdgeNames.get(upStreamVertexName)
- .get(outputIndex));
- upStreamConfig.setSelectAll(downStreamVertexName, outEdgeSelectAll.get(upStreamVertexName)
- .get(outputIndex));
- }
-
- /**
- * Sets the number of parallel instances created for the given vertex.
- *
- * @param vertexName
- * Name of the vertex
- * @param parallelism
- * Number of parallel instances created
- */
- public void setParallelism(String vertexName, int parallelism) {
- vertexParallelism.put(vertexName, parallelism);
- }
-
- /**
- * Sets the input format for the given vertex.
- *
- * @param vertexName
- * Name of the vertex
- * @param inputFormat
- * input format of the file source associated with the given
- * vertex
- */
- public void setInputFormat(String vertexName, InputFormat<String, ?> inputFormat) {
- inputFormatList.put(vertexName, inputFormat);
- }
-
- public void setBufferTimeout(String vertexName, long bufferTimeout) {
- this.bufferTimeout.put(vertexName, bufferTimeout);
- }
-
- public void addOperatorState(String veretxName, String stateName, OperatorState<?> state) {
- Map<String, OperatorState<?>> states = operatorStates.get(veretxName);
- if (states == null) {
- states = new HashMap<String, OperatorState<?>>();
- states.put(stateName, state);
- } else {
- if (states.containsKey(stateName)) {
- throw new RuntimeException("State has already been registered with this name: "
- + stateName);
- } else {
- states.put(stateName, state);
- }
- }
- operatorStates.put(veretxName, states);
- }
-
- /**
- * Connects two vertices in the JobGraph using the selected partitioner
- * settings
- *
- * @param upStreamVertexName
- * Name of the upstream(output) vertex
- * @param downStreamVertexName
- * Name of the downstream(input) vertex
- * @param partitionerObject
- * Partitioner object
- * @param typeNumber
- * Number of the type (used at co-functions)
- * @param outputNames
- * User defined names of the out edge
- */
- public void setEdge(String upStreamVertexName, String downStreamVertexName,
- StreamPartitioner<?> partitionerObject, int typeNumber, List<String> outputNames,
- boolean selectAll) {
- outEdgeList.get(upStreamVertexName).add(downStreamVertexName);
- outEdgeIndex.get(upStreamVertexName).add(typeNumber);
- inEdgeList.get(downStreamVertexName).add(upStreamVertexName);
- outPartitioning.get(upStreamVertexName).add(partitionerObject);
- outEdgeNames.get(upStreamVertexName).add(outputNames);
- outEdgeSelectAll.get(upStreamVertexName).add(selectAll);
- }
-
- /**
- * Sets the parallelism and buffertimeout of the iteration head of the given
- * iteration id to the parallelism given.
- *
- * @param iterationID
- * ID of the iteration
- * @param iterationTail
- * ID of the iteration tail
- */
- public void setIterationSourceSettings(String iterationID, String iterationTail) {
- setParallelism(iterationIDtoHeadName.get(iterationID), vertexParallelism.get(iterationTail));
- setBufferTimeout(iterationIDtoHeadName.get(iterationID), bufferTimeout.get(iterationTail));
- }
-
- /**
- * Sets a user defined {@link OutputSelector} for the given vertex. Used for
- * directed emits.
- *
- * @param vertexName
- * Name of the vertex for which the output selector will be set
- * @param serializedOutputSelector
- * Byte array representing the serialized output selector.
- */
- public <T> void setOutputSelector(String vertexName, byte[] serializedOutputSelector) {
- outputSelectors.put(vertexName, serializedOutputSelector);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Outputselector set for {}", vertexName);
- }
-
- }
-
- public <IN, OUT> void setInvokable(String id, StreamInvokable<IN, OUT> invokableObject) {
- invokableObjects.put(id, invokableObject);
- }
-
- public StreamInvokable<?, ?> getInvokable(String id) {
- return invokableObjects.get(id);
- }
-
- public <OUT> void setOutType(String id, TypeInformation<OUT> outType) {
- StreamRecordSerializer<OUT> serializer = new StreamRecordSerializer<OUT>(outType);
- typeSerializersOut1.put(id, serializer);
- }
-
- /**
- * Sets TypeSerializerWrapper from one vertex to another, used with some
- * sinks.
- *
- * @param from
- * from
- * @param to
- * to
- */
- public void setSerializersFrom(String from, String to) {
- operatorNames.put(to, operatorNames.get(from));
-
- typeSerializersIn1.put(to, typeSerializersOut1.get(from));
- typeSerializersIn2.put(to, typeSerializersOut2.get(from));
- typeSerializersOut1.put(to, typeSerializersOut1.get(from));
- typeSerializersOut2.put(to, typeSerializersOut2.get(from));
- }
-
- /**
- * Sets slot sharing for the vertices.
- */
- private void setSlotSharing() {
- SlotSharingGroup shareGroup = new SlotSharingGroup();
-
- for (AbstractJobVertex vertex : streamVertices.values()) {
- vertex.setSlotSharingGroup(shareGroup);
- }
-
- for (Integer iterID : new HashSet<Integer>(iterationIds.values())) {
- CoLocationGroup ccg = new CoLocationGroup();
- AbstractJobVertex tail = streamVertices.get(iterationIDtoTailName.get(iterID));
- AbstractJobVertex head = streamVertices.get(iterationIDtoHeadName.get(iterID));
-
- ccg.addVertex(head);
- ccg.addVertex(tail);
- }
- }
-
- /**
- * Gets the assembled {@link JobGraph} and adds a default name for it.
- */
- public JobGraph getJobGraph() {
- return getJobGraph(DEAFULT_JOB_NAME);
- }
-
- /**
- * Gets the assembled {@link JobGraph} and adds a user specified name for
- * it.
- *
- * @param jobGraphName
- * name of the jobGraph
- */
- public JobGraph getJobGraph(String jobGraphName) {
- jobGraph = new JobGraph(jobGraphName);
- buildJobGraph();
- return jobGraph;
- }
-
- /**
- * Builds the {@link JobGraph} from the vertices with the edges and settings
- * provided.
- */
- private void buildJobGraph() {
-
- for (String sourceName : sources) {
- createChain(sourceName, sourceName);
- }
-
- setSlotSharing();
- }
-
- public void setChaining(boolean chaining) {
- this.chaining = chaining;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
index 6fffaa6..213a892 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
@@ -18,7 +18,7 @@
package org.apache.flink.streaming.api;
import java.io.Serializable;
-import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -62,7 +62,7 @@ public class StreamConfig implements Serializable {
private static final String TYPE_SERIALIZER_OUT_2 = "typeSerializer_out_2";
private static final String ITERATON_WAIT = "iterationWait";
private static final String OUTPUTS = "outVertexNames";
- private static final String RW_ORDER = "rwOrder";
+ private static final String EDGES_IN_ORDER = "rwOrder";
// DEFAULT VALUES
@@ -186,10 +186,10 @@ public class StreamConfig implements Serializable {
return config.getBoolean(DIRECTED_EMIT, false);
}
- public void setOutputSelector(byte[] outputSelector) {
+ public void setOutputSelector(OutputSelector<?> outputSelector) {
if (outputSelector != null) {
setDirectedEmit(true);
- config.setBytes(OUTPUT_SELECTOR, outputSelector);
+ config.setBytes(OUTPUT_SELECTOR, SerializationUtils.serialize(outputSelector));
}
}
@@ -293,20 +293,16 @@ public class StreamConfig implements Serializable {
}
}
- public void setRecordWriterOrder(List<Tuple2<String, String>> outEdgeList) {
+ public void setOutEdgesInOrder(List<Tuple2<String, String>> outEdgeList) {
- List<String> outVertices = new ArrayList<String>();
- for (Tuple2<String, String> edge : outEdgeList) {
- outVertices.add(edge.f1);
- }
-
- config.setBytes(RW_ORDER, SerializationUtils.serialize((Serializable) outVertices));
+ config.setBytes(EDGES_IN_ORDER, SerializationUtils.serialize((Serializable) outEdgeList));
}
@SuppressWarnings("unchecked")
- public List<String> getRecordWriterOrder(ClassLoader cl) {
+ public List<Tuple2<String, String>> getOutEdgesInOrder(ClassLoader cl) {
try {
- return (List<String>) InstantiationUtil.readObjectFromConfig(this.config, RW_ORDER, cl);
+ return (List<Tuple2<String, String>>) InstantiationUtil.readObjectFromConfig(
+ this.config, EDGES_IN_ORDER, cl);
} catch (Exception e) {
throw new RuntimeException("Could not instantiate outputs.");
}
@@ -358,8 +354,10 @@ public class StreamConfig implements Serializable {
public Map<String, StreamConfig> getTransitiveChainedTaskConfigs(ClassLoader cl) {
try {
- return (Map<String, StreamConfig>) InstantiationUtil.readObjectFromConfig(this.config,
- CHAINED_TASK_CONFIG, cl);
+ Map<String, StreamConfig> confs = (Map<String, StreamConfig>) InstantiationUtil
+ .readObjectFromConfig(this.config, CHAINED_TASK_CONFIG, cl);
+
+ return confs == null ? new HashMap<String, StreamConfig>() : confs;
} catch (Exception e) {
throw new RuntimeException("Could not instantiate configuration.");
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamGraph.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamGraph.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamGraph.java
new file mode 100644
index 0000000..3dd1bdb
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamGraph.java
@@ -0,0 +1,545 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.flink.api.common.io.InputFormat;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
+import org.apache.flink.streaming.api.collector.OutputSelector;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.invokable.operator.co.CoInvokable;
+import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
+import org.apache.flink.streaming.api.streamvertex.CoStreamVertex;
+import org.apache.flink.streaming.api.streamvertex.StreamIterationHead;
+import org.apache.flink.streaming.api.streamvertex.StreamIterationTail;
+import org.apache.flink.streaming.api.streamvertex.StreamVertex;
+import org.apache.flink.streaming.partitioner.StreamPartitioner;
+import org.apache.flink.streaming.state.OperatorState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Object for building Apache Flink stream processing graphs
+ */
+public class StreamGraph {
+
+ private static final Logger LOG = LoggerFactory.getLogger(StreamGraph.class);
+ private final static String DEAFULT_JOB_NAME = "Flink Streaming Job";
+
+ protected boolean chaining = true;
+
+ // Graph attributes
+ private Map<String, Integer> operatorParallelisms;
+ private Map<String, Long> bufferTimeouts;
+ private Map<String, List<String>> outEdgeLists;
+ private Map<String, List<Integer>> outEdgeTypes;
+ private Map<String, List<List<String>>> selectedNames;
+ private Map<String, List<Boolean>> outEdgeSelectAlls;
+ private Map<String, List<String>> inEdgeLists;
+ private Map<String, List<StreamPartitioner<?>>> outputPartitioners;
+ private Map<String, String> operatorNames;
+ private Map<String, StreamInvokable<?, ?>> invokableObjects;
+ private Map<String, StreamRecordSerializer<?>> typeSerializersIn1;
+ private Map<String, StreamRecordSerializer<?>> typeSerializersIn2;
+ private Map<String, StreamRecordSerializer<?>> typeSerializersOut1;
+ private Map<String, StreamRecordSerializer<?>> typeSerializersOut2;
+ private Map<String, OutputSelector<?>> outputSelectors;
+ private Map<String, Class<? extends AbstractInvokable>> jobVertexClasses;
+ private Map<String, Integer> iterationIds;
+ private Map<Integer, String> iterationIDtoHeadName;
+ private Map<Integer, String> iterationIDtoTailName;
+ private Map<String, Integer> iterationTailCount;
+ private Map<String, Long> iterationTimeouts;
+ private Map<String, Map<String, OperatorState<?>>> operatorStates;
+ private Map<String, InputFormat<String, ?>> inputFormatLists;
+
+ private Set<String> sources;
+
+ public StreamGraph() {
+
+ initGraph();
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("StreamGraph created");
+ }
+ }
+
+ public void initGraph() {
+ operatorParallelisms = new HashMap<String, Integer>();
+ bufferTimeouts = new HashMap<String, Long>();
+ outEdgeLists = new HashMap<String, List<String>>();
+ outEdgeTypes = new HashMap<String, List<Integer>>();
+ selectedNames = new HashMap<String, List<List<String>>>();
+ outEdgeSelectAlls = new HashMap<String, List<Boolean>>();
+ inEdgeLists = new HashMap<String, List<String>>();
+ outputPartitioners = new HashMap<String, List<StreamPartitioner<?>>>();
+ operatorNames = new HashMap<String, String>();
+ invokableObjects = new HashMap<String, StreamInvokable<?, ?>>();
+ typeSerializersIn1 = new HashMap<String, StreamRecordSerializer<?>>();
+ typeSerializersIn2 = new HashMap<String, StreamRecordSerializer<?>>();
+ typeSerializersOut1 = new HashMap<String, StreamRecordSerializer<?>>();
+ typeSerializersOut2 = new HashMap<String, StreamRecordSerializer<?>>();
+ outputSelectors = new HashMap<String, OutputSelector<?>>();
+ jobVertexClasses = new HashMap<String, Class<? extends AbstractInvokable>>();
+ iterationIds = new HashMap<String, Integer>();
+ iterationIDtoHeadName = new HashMap<Integer, String>();
+ iterationIDtoTailName = new HashMap<Integer, String>();
+ iterationTailCount = new HashMap<String, Integer>();
+ iterationTimeouts = new HashMap<String, Long>();
+ operatorStates = new HashMap<String, Map<String, OperatorState<?>>>();
+ inputFormatLists = new HashMap<String, InputFormat<String, ?>>();
+ sources = new HashSet<String>();
+ }
+
+ /**
+ * Adds a vertex to the streaming graph with the given parameters
+ *
+ * @param vertexName
+ * Name of the vertex
+ * @param invokableObject
+ * User defined operator
+ * @param inTypeInfo
+ * Input type for serialization
+ * @param outTypeInfo
+ * Output type for serialization
+ * @param operatorName
+ * Operator type
+ * @param parallelism
+ * Number of parallel instances created
+ */
+ public <IN, OUT> void addStreamVertex(String vertexName,
+ StreamInvokable<IN, OUT> invokableObject, TypeInformation<IN> inTypeInfo,
+ TypeInformation<OUT> outTypeInfo, String operatorName, int parallelism) {
+
+ addVertex(vertexName, StreamVertex.class, invokableObject, operatorName, parallelism);
+
+ StreamRecordSerializer<IN> inSerializer = inTypeInfo != null ? new StreamRecordSerializer<IN>(
+ inTypeInfo) : null;
+ StreamRecordSerializer<OUT> outSerializer = outTypeInfo != null ? new StreamRecordSerializer<OUT>(
+ outTypeInfo) : null;
+
+ addTypeSerializers(vertexName, inSerializer, null, outSerializer, null);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Vertex: {}", vertexName);
+ }
+ }
+
+ public <IN, OUT> void addSourceVertex(String vertexName,
+ StreamInvokable<IN, OUT> invokableObject, TypeInformation<IN> inTypeInfo,
+ TypeInformation<OUT> outTypeInfo, String operatorName, int parallelism) {
+ addStreamVertex(vertexName, invokableObject, inTypeInfo, outTypeInfo, operatorName,
+ parallelism);
+ sources.add(vertexName);
+ }
+
+ /**
+ * Adds a vertex for the iteration head to the {@link JobGraph}. The
+ * iterated values will be fed from this vertex back to the graph.
+ *
+ * @param vertexName
+ * Name of the vertex
+ * @param iterationHead
+ * Id of the iteration head
+ * @param iterationID
+ * ID of iteration for multiple iterations
+ * @param parallelism
+ * Number of parallel instances created
+ * @param waitTime
+ * Max wait time for next record
+ */
+ public void addIterationHead(String vertexName, String iterationHead, Integer iterationID,
+ int parallelism, long waitTime) {
+
+ addVertex(vertexName, StreamIterationHead.class, null, null, parallelism);
+
+ chaining = false;
+
+ iterationIds.put(vertexName, iterationID);
+ iterationIDtoHeadName.put(iterationID, vertexName);
+
+ setSerializersFrom(iterationHead, vertexName);
+
+ setEdge(vertexName, iterationHead,
+ outputPartitioners.get(inEdgeLists.get(iterationHead).get(0)).get(0), 0,
+ new ArrayList<String>(), false);
+
+ iterationTimeouts.put(iterationIDtoHeadName.get(iterationID), waitTime);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("ITERATION SOURCE: {}", vertexName);
+ }
+
+ sources.add(vertexName);
+ }
+
+ /**
+ * Adds a vertex for the iteration tail to the {@link JobGraph}. The values
+ * intended to be iterated will be sent to this sink from the iteration
+ * head.
+ *
+ * @param vertexName
+ * Name of the vertex
+ * @param iterationTail
+ * Id of the iteration tail
+ * @param iterationID
+ * ID of iteration for mulitple iterations
+ * @param parallelism
+ * Number of parallel instances created
+ * @param waitTime
+ * Max waiting time for next record
+ */
+ public void addIterationTail(String vertexName, String iterationTail, Integer iterationID,
+ int parallelism, long waitTime) {
+
+ if (bufferTimeouts.get(iterationTail) == 0) {
+ throw new RuntimeException("Buffer timeout 0 at iteration tail is not supported.");
+ }
+
+ addVertex(vertexName, StreamIterationTail.class, null, null, parallelism);
+
+ iterationIds.put(vertexName, iterationID);
+ iterationIDtoTailName.put(iterationID, vertexName);
+
+ setSerializersFrom(iterationTail, vertexName);
+ iterationTimeouts.put(iterationIDtoTailName.get(iterationID), waitTime);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("ITERATION SINK: {}", vertexName);
+ }
+
+ }
+
+ public <IN1, IN2, OUT> void addCoTask(String vertexName,
+ CoInvokable<IN1, IN2, OUT> taskInvokableObject, TypeInformation<IN1> in1TypeInfo,
+ TypeInformation<IN2> in2TypeInfo, TypeInformation<OUT> outTypeInfo,
+ String operatorName, int parallelism) {
+
+ addVertex(vertexName, CoStreamVertex.class, taskInvokableObject, operatorName, parallelism);
+
+ addTypeSerializers(vertexName, new StreamRecordSerializer<IN1>(in1TypeInfo),
+ new StreamRecordSerializer<IN2>(in2TypeInfo), new StreamRecordSerializer<OUT>(
+ outTypeInfo), null);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("CO-TASK: {}", vertexName);
+ }
+ }
+
+ /**
+ * Sets vertex parameters in the JobGraph
+ *
+ * @param vertexName
+ * Name of the vertex
+ * @param vertexClass
+ * The class of the vertex
+ * @param invokableObjectject
+ * The user defined invokable object
+ * @param operatorName
+ * Type of the user defined operator
+ * @param parallelism
+ * Number of parallel instances created
+ */
+ private void addVertex(String vertexName, Class<? extends AbstractInvokable> vertexClass,
+ StreamInvokable<?, ?> invokableObject, String operatorName, int parallelism) {
+
+ jobVertexClasses.put(vertexName, vertexClass);
+ setParallelism(vertexName, parallelism);
+ invokableObjects.put(vertexName, invokableObject);
+ operatorNames.put(vertexName, operatorName);
+ outEdgeLists.put(vertexName, new ArrayList<String>());
+ outEdgeTypes.put(vertexName, new ArrayList<Integer>());
+ selectedNames.put(vertexName, new ArrayList<List<String>>());
+ outEdgeSelectAlls.put(vertexName, new ArrayList<Boolean>());
+ inEdgeLists.put(vertexName, new ArrayList<String>());
+ outputPartitioners.put(vertexName, new ArrayList<StreamPartitioner<?>>());
+ iterationTailCount.put(vertexName, 0);
+ }
+
+ /**
+ * Connects two vertices in the JobGraph using the selected partitioner
+ * settings
+ *
+ * @param upStreamVertexName
+ * Name of the upstream(output) vertex
+ * @param downStreamVertexName
+ * Name of the downstream(input) vertex
+ * @param partitionerObject
+ * Partitioner object
+ * @param typeNumber
+ * Number of the type (used at co-functions)
+ * @param outputNames
+ * User defined names of the out edge
+ */
+ public void setEdge(String upStreamVertexName, String downStreamVertexName,
+ StreamPartitioner<?> partitionerObject, int typeNumber, List<String> outputNames,
+ boolean selectAll) {
+ outEdgeLists.get(upStreamVertexName).add(downStreamVertexName);
+ outEdgeTypes.get(upStreamVertexName).add(typeNumber);
+ inEdgeLists.get(downStreamVertexName).add(upStreamVertexName);
+ outputPartitioners.get(upStreamVertexName).add(partitionerObject);
+ selectedNames.get(upStreamVertexName).add(outputNames);
+ outEdgeSelectAlls.get(upStreamVertexName).add(selectAll);
+ }
+
+ private void addTypeSerializers(String vertexName, StreamRecordSerializer<?> in1,
+ StreamRecordSerializer<?> in2, StreamRecordSerializer<?> out1,
+ StreamRecordSerializer<?> out2) {
+ typeSerializersIn1.put(vertexName, in1);
+ typeSerializersIn2.put(vertexName, in2);
+ typeSerializersOut1.put(vertexName, out1);
+ typeSerializersOut2.put(vertexName, out2);
+ }
+
+ /**
+ * Sets the number of parallel instances created for the given vertex.
+ *
+ * @param vertexName
+ * Name of the vertex
+ * @param parallelism
+ * Number of parallel instances created
+ */
+ public void setParallelism(String vertexName, int parallelism) {
+ operatorParallelisms.put(vertexName, parallelism);
+ }
+
+ public int getParallelism(String vertexName) {
+ return operatorParallelisms.get(vertexName);
+ }
+
+ /**
+ * Sets the input format for the given vertex.
+ *
+ * @param vertexName
+ * Name of the vertex
+ * @param inputFormat
+ * input format of the file source associated with the given
+ * vertex
+ */
+ public void setInputFormat(String vertexName, InputFormat<String, ?> inputFormat) {
+ inputFormatLists.put(vertexName, inputFormat);
+ }
+
+ public void setBufferTimeout(String vertexName, long bufferTimeout) {
+ this.bufferTimeouts.put(vertexName, bufferTimeout);
+ }
+
+ public long getBufferTimeout(String vertexName) {
+ return this.bufferTimeouts.get(vertexName);
+ }
+
+ public void addOperatorState(String veretxName, String stateName, OperatorState<?> state) {
+ Map<String, OperatorState<?>> states = operatorStates.get(veretxName);
+ if (states == null) {
+ states = new HashMap<String, OperatorState<?>>();
+ states.put(stateName, state);
+ } else {
+ if (states.containsKey(stateName)) {
+ throw new RuntimeException("State has already been registered with this name: "
+ + stateName);
+ } else {
+ states.put(stateName, state);
+ }
+ }
+ operatorStates.put(veretxName, states);
+ }
+
+ /**
+ * Sets the parallelism and buffertimeout of the iteration head of the given
+ * iteration id to the parallelism given.
+ *
+ * @param iterationID
+ * ID of the iteration
+ * @param iterationTail
+ * ID of the iteration tail
+ */
+ public void setIterationSourceSettings(String iterationID, String iterationTail) {
+ setParallelism(iterationIDtoHeadName.get(iterationID),
+ operatorParallelisms.get(iterationTail));
+ setBufferTimeout(iterationIDtoHeadName.get(iterationID), bufferTimeouts.get(iterationTail));
+ }
+
+ /**
+ * Sets a user defined {@link OutputSelector} for the given operator. Used
+ * for directed emits.
+ *
+ * @param vertexName
+ * Name of the vertex for which the output selector will be set
+ * @param outputSelector
+ * The outputselector object
+ */
+ public void setOutputSelector(String vertexName, OutputSelector<?> outputSelector) {
+ outputSelectors.put(vertexName, outputSelector);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Outputselector set for {}", vertexName);
+ }
+
+ }
+
+ public <IN, OUT> void setInvokable(String vertexName, StreamInvokable<IN, OUT> invokableObject) {
+ invokableObjects.put(vertexName, invokableObject);
+ }
+
+ public <OUT> void setOutType(String id, TypeInformation<OUT> outType) {
+ StreamRecordSerializer<OUT> serializer = new StreamRecordSerializer<OUT>(outType);
+ typeSerializersOut1.put(id, serializer);
+ }
+
+ public StreamInvokable<?, ?> getInvokable(String vertexName) {
+ return invokableObjects.get(vertexName);
+ }
+
+ @SuppressWarnings("unchecked")
+ public <OUT> StreamRecordSerializer<OUT> getOutSerializer1(String vertexName) {
+ return (StreamRecordSerializer<OUT>) typeSerializersOut1.get(vertexName);
+ }
+
+ @SuppressWarnings("unchecked")
+ public <OUT> StreamRecordSerializer<OUT> getOutSerializer2(String vertexName) {
+ return (StreamRecordSerializer<OUT>) typeSerializersOut2.get(vertexName);
+ }
+
+ @SuppressWarnings("unchecked")
+ public <IN> StreamRecordSerializer<IN> getInSerializer1(String vertexName) {
+ return (StreamRecordSerializer<IN>) typeSerializersIn1.get(vertexName);
+ }
+
+ @SuppressWarnings("unchecked")
+ public <IN> StreamRecordSerializer<IN> getInSerializer2(String vertexName) {
+ return (StreamRecordSerializer<IN>) typeSerializersIn2.get(vertexName);
+ }
+
+ /**
+ * Sets TypeSerializerWrapper from one vertex to another, used with some
+ * sinks.
+ *
+ * @param from
+ * from
+ * @param to
+ * to
+ */
+ public void setSerializersFrom(String from, String to) {
+ operatorNames.put(to, operatorNames.get(from));
+
+ typeSerializersIn1.put(to, typeSerializersOut1.get(from));
+ typeSerializersIn2.put(to, typeSerializersOut2.get(from));
+ typeSerializersOut1.put(to, typeSerializersOut1.get(from));
+ typeSerializersOut2.put(to, typeSerializersOut2.get(from));
+ }
+
+ /**
+ * Gets the assembled {@link JobGraph} and adds a default name for it.
+ */
+ public JobGraph getJobGraph() {
+ return getJobGraph(DEAFULT_JOB_NAME);
+ }
+
+ /**
+ * Gets the assembled {@link JobGraph} and adds a user specified name for
+ * it.
+ *
+ * @param jobGraphName
+ * name of the jobGraph
+ */
+ public JobGraph getJobGraph(String jobGraphName) {
+
+ StreamingJobGraphGenerator optimizer = new StreamingJobGraphGenerator(this);
+
+ return optimizer.createJobGraph(jobGraphName);
+ }
+
+ public void setChaining(boolean chaining) {
+ this.chaining = chaining;
+ }
+
+ public Collection<String> getSources() {
+ return sources;
+ }
+
+ public List<String> getOutEdges(String vertexName) {
+ return outEdgeLists.get(vertexName);
+ }
+
+ public List<String> getInEdges(String vertexName) {
+ return inEdgeLists.get(vertexName);
+ }
+
+ public List<Integer> getOutEdgeTypes(String vertexName) {
+
+ return outEdgeTypes.get(vertexName);
+ }
+
+ public StreamPartitioner<?> getOutPartitioner(String vertexName, int outputIndex) {
+ return outputPartitioners.get(vertexName).get(outputIndex);
+ }
+
+ public List<String> getSelectedNames(String vertexName, int outputIndex) {
+ return selectedNames.get(vertexName).get(outputIndex);
+ }
+
+ public Boolean isSelectAll(String vertexName, int outputIndex) {
+ return outEdgeSelectAlls.get(vertexName).get(outputIndex);
+ }
+
+ public Collection<Integer> getIterationIDs() {
+ return new HashSet<Integer>(iterationIds.values());
+ }
+
+ public String getIterationTail(int iterID) {
+ return iterationIDtoTailName.get(iterID);
+ }
+
+ public String getIterationHead(int iterID) {
+ return iterationIDtoHeadName.get(iterID);
+ }
+
+ public Class<? extends AbstractInvokable> getJobVertexClass(String vertexName) {
+ return jobVertexClasses.get(vertexName);
+ }
+
+ public InputFormat<String, ?> getInputFormat(String vertexName) {
+ return inputFormatLists.get(vertexName);
+ }
+
+ public OutputSelector<?> getOutputSelector(String vertexName) {
+ return outputSelectors.get(vertexName);
+ }
+
+ public Map<String, OperatorState<?>> getState(String vertexName) {
+ return operatorStates.get(vertexName);
+ }
+
+ public Integer getIterationID(String vertexName) {
+ return iterationIds.get(vertexName);
+ }
+
+ public long getIterationTimeout(String vertexName) {
+ return iterationTimeouts.get(vertexName);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamingJobGraphGenerator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamingJobGraphGenerator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamingJobGraphGenerator.java
new file mode 100644
index 0000000..6c0cc20
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamingJobGraphGenerator.java
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.jobgraph.AbstractJobVertex;
+import org.apache.flink.runtime.jobgraph.DistributionPattern;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
+import org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup;
+import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.invokable.StreamInvokable.ChainingStrategy;
+import org.apache.flink.streaming.api.streamvertex.StreamIterationHead;
+import org.apache.flink.streaming.api.streamvertex.StreamIterationTail;
+import org.apache.flink.streaming.partitioner.StreamPartitioner;
+import org.apache.flink.streaming.partitioner.StreamPartitioner.PartitioningStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class StreamingJobGraphGenerator {
+
+ private static final Logger LOG = LoggerFactory.getLogger(StreamingJobGraphGenerator.class);
+
+ private StreamGraph streamGraph;
+
+ private Map<String, AbstractJobVertex> streamVertices;
+ private JobGraph jobGraph;
+ private Collection<String> builtNodes;
+
+ private Map<String, Map<String, StreamConfig>> chainedConfigs;
+ private Map<String, StreamConfig> vertexConfigs;
+ private Map<String, String> chainedNames;
+
+ public StreamingJobGraphGenerator(StreamGraph streamGraph) {
+ this.streamGraph = streamGraph;
+ }
+
+ private void init() {
+ this.streamVertices = new HashMap<String, AbstractJobVertex>();
+ this.builtNodes = new HashSet<String>();
+ this.chainedConfigs = new HashMap<String, Map<String, StreamConfig>>();
+ this.vertexConfigs = new HashMap<String, StreamConfig>();
+ this.chainedNames = new HashMap<String, String>();
+ }
+
+ public JobGraph createJobGraph(String jobName) {
+ jobGraph = new JobGraph(jobName);
+ init();
+
+ for (String sourceName : streamGraph.getSources()) {
+ createChain(sourceName, sourceName);
+ }
+
+ setSlotSharing();
+
+ return jobGraph;
+ }
+
+ private List<Tuple2<String, String>> createChain(String startNode, String current) {
+
+ if (!builtNodes.contains(startNode)) {
+
+ List<Tuple2<String, String>> transitiveOutEdges = new ArrayList<Tuple2<String, String>>();
+ List<String> chainableOutputs = new ArrayList<String>();
+ List<String> nonChainableOutputs = new ArrayList<String>();
+
+ for (String outName : streamGraph.getOutEdges(current)) {
+ if (isChainable(current, outName)) {
+ chainableOutputs.add(outName);
+ } else {
+ nonChainableOutputs.add(outName);
+ }
+ }
+
+ for (String chainable : chainableOutputs) {
+ transitiveOutEdges.addAll(createChain(startNode, chainable));
+ }
+
+ for (String nonChainable : nonChainableOutputs) {
+ transitiveOutEdges.add(new Tuple2<String, String>(current, nonChainable));
+ createChain(nonChainable, nonChainable);
+ }
+
+ chainedNames.put(current, createChainedName(current, chainableOutputs));
+
+ StreamConfig config = current.equals(startNode) ? createProcessingVertex(startNode)
+ : new StreamConfig(new Configuration());
+
+ setVertexConfig(current, config, chainableOutputs, nonChainableOutputs);
+
+ if (current.equals(startNode)) {
+
+ config.setChainStart();
+ config.setOutEdgesInOrder(transitiveOutEdges);
+
+ for (Tuple2<String, String> edge : transitiveOutEdges) {
+ connect(startNode, edge);
+ }
+
+ config.setTransitiveChainedTaskConfigs(chainedConfigs.get(startNode));
+
+ } else {
+
+ Map<String, StreamConfig> chainedConfs = chainedConfigs.get(startNode);
+
+ if (chainedConfs == null) {
+ chainedConfigs.put(startNode, new HashMap<String, StreamConfig>());
+ }
+ chainedConfigs.get(startNode).put(current, config);
+ }
+
+ return transitiveOutEdges;
+
+ } else {
+ return new ArrayList<Tuple2<String, String>>();
+ }
+ }
+
+ private String createChainedName(String vertexName, List<String> chainedOutputs) {
+ if (chainedOutputs.size() > 1) {
+ List<String> outputChainedNames = new ArrayList<String>();
+ for (String chainable : chainedOutputs) {
+ outputChainedNames.add(chainedNames.get(chainable));
+ }
+
+ return vertexName + " -> (" + StringUtils.join(outputChainedNames, ", ") + ")";
+ } else if (chainedOutputs.size() == 1) {
+ return vertexName + " -> " + chainedNames.get(chainedOutputs.get(0));
+ } else {
+ return vertexName;
+ }
+
+ }
+
+ private StreamConfig createProcessingVertex(String vertexName) {
+
+ AbstractJobVertex vertex = new AbstractJobVertex(chainedNames.get(vertexName));
+
+ vertex.setInvokableClass(streamGraph.getJobVertexClass(vertexName));
+ vertex.setParallelism(streamGraph.getParallelism(vertexName));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Parallelism set: {} for {}", streamGraph.getParallelism(vertexName),
+ vertexName);
+ }
+
+ if (streamGraph.getInputFormat(vertexName) != null) {
+ vertex.setInputSplitSource(streamGraph.getInputFormat(vertexName));
+ }
+
+ streamVertices.put(vertexName, vertex);
+ builtNodes.add(vertexName);
+ jobGraph.addVertex(vertex);
+
+ return new StreamConfig(vertex.getConfiguration());
+ }
+
+ private void setVertexConfig(String vertexName, StreamConfig config,
+ List<String> chainableOutputs, List<String> nonChainableOutputs) {
+
+ config.setVertexName(vertexName);
+ config.setBufferTimeout(streamGraph.getBufferTimeout(vertexName));
+
+ config.setTypeSerializerIn1(streamGraph.getInSerializer1(vertexName));
+ config.setTypeSerializerIn2(streamGraph.getInSerializer2(vertexName));
+ config.setTypeSerializerOut1(streamGraph.getOutSerializer1(vertexName));
+ config.setTypeSerializerOut2(streamGraph.getOutSerializer2(vertexName));
+
+ config.setUserInvokable(streamGraph.getInvokable(vertexName));
+ config.setOutputSelector(streamGraph.getOutputSelector(vertexName));
+ config.setOperatorStates(streamGraph.getState(vertexName));
+
+ config.setNumberOfOutputs(nonChainableOutputs.size());
+ config.setOutputs(nonChainableOutputs);
+ config.setChainedOutputs(chainableOutputs);
+
+ Class<? extends AbstractInvokable> vertexClass = streamGraph.getJobVertexClass(vertexName);
+
+ if (vertexClass.equals(StreamIterationHead.class)
+ || vertexClass.equals(StreamIterationTail.class)) {
+ config.setIterationId(streamGraph.getIterationID(vertexName));
+ config.setIterationWaitTime(streamGraph.getIterationTimeout(vertexName));
+ }
+
+ vertexConfigs.put(vertexName, config);
+ }
+
+ private <T> void connect(String headOfChain, Tuple2<String, String> edge) {
+
+ String upStreamVertexName = edge.f0;
+ String downStreamVertexName = edge.f1;
+
+ int outputIndex = streamGraph.getOutEdges(upStreamVertexName).indexOf(downStreamVertexName);
+
+ AbstractJobVertex headVertex = streamVertices.get(headOfChain);
+ AbstractJobVertex downStreamVertex = streamVertices.get(downStreamVertexName);
+
+ StreamConfig downStreamConfig = new StreamConfig(downStreamVertex.getConfiguration());
+ StreamConfig upStreamConfig = new StreamConfig(headVertex.getConfiguration());
+
+ List<Integer> outEdgeIndexList = streamGraph.getOutEdgeTypes(upStreamVertexName);
+ int numOfInputs = downStreamConfig.getNumberOfInputs();
+
+ downStreamConfig.setInputIndex(numOfInputs++, outEdgeIndexList.get(outputIndex));
+ downStreamConfig.setNumberOfInputs(numOfInputs);
+
+ StreamPartitioner<?> partitioner = streamGraph.getOutPartitioner(upStreamVertexName,
+ outputIndex);
+
+ upStreamConfig.setPartitioner(downStreamVertexName, partitioner);
+
+ if (partitioner.getStrategy() == PartitioningStrategy.FORWARD) {
+ downStreamVertex.connectNewDataSetAsInput(headVertex, DistributionPattern.POINTWISE);
+ } else {
+ downStreamVertex.connectNewDataSetAsInput(headVertex, DistributionPattern.ALL_TO_ALL);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("CONNECTED: {} - {} -> {}", partitioner.getClass().getSimpleName(),
+ headOfChain, downStreamVertexName);
+ }
+
+ upStreamConfig.setOutputNames(downStreamVertexName,
+ streamGraph.getSelectedNames(upStreamVertexName, outputIndex));
+ upStreamConfig.setSelectAll(downStreamVertexName,
+ streamGraph.isSelectAll(upStreamVertexName, outputIndex));
+ }
+
+ private boolean isChainable(String vertexName, String outName) {
+
+ StreamInvokable<?, ?> headInvokable = streamGraph.getInvokable(vertexName);
+ StreamInvokable<?, ?> outInvokable = streamGraph.getInvokable(outName);
+
+ return streamGraph.getInEdges(outName).size() == 1
+ && outInvokable != null
+ && streamGraph.getOutputSelector(vertexName) == null
+ && outInvokable.getChainingStrategy() == ChainingStrategy.ALWAYS
+ && (headInvokable.getChainingStrategy() == ChainingStrategy.HEAD || headInvokable
+ .getChainingStrategy() == ChainingStrategy.ALWAYS)
+ && streamGraph.getOutPartitioner(vertexName,
+ streamGraph.getOutEdges(vertexName).indexOf(outName)).getStrategy() == PartitioningStrategy.FORWARD
+ && streamGraph.getParallelism(vertexName) == streamGraph.getParallelism(outName)
+ && streamGraph.chaining;
+ }
+
+ private void setSlotSharing() {
+ SlotSharingGroup shareGroup = new SlotSharingGroup();
+
+ for (AbstractJobVertex vertex : streamVertices.values()) {
+ vertex.setSlotSharingGroup(shareGroup);
+ }
+
+ for (Integer iterID : streamGraph.getIterationIDs()) {
+ CoLocationGroup ccg = new CoLocationGroup();
+ AbstractJobVertex tail = streamVertices.get(streamGraph.getIterationTail(iterID));
+ AbstractJobVertex head = streamVertices.get(streamGraph.getIterationHead(iterID));
+
+ ccg.addVertex(head);
+ ccg.addVertex(tail);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
index 80ea970..db8649b 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
@@ -22,7 +22,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.ClosureCleaner;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.apache.flink.streaming.api.JobGraphBuilder;
+import org.apache.flink.streaming.api.StreamGraph;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.function.co.CoFlatMapFunction;
import org.apache.flink.streaming.api.function.co.CoMapFunction;
@@ -52,7 +52,7 @@ import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
public class ConnectedDataStream<IN1, IN2> {
protected StreamExecutionEnvironment environment;
- protected JobGraphBuilder jobGraphBuilder;
+ protected StreamGraph jobGraphBuilder;
protected DataStream<IN1> dataStream1;
protected DataStream<IN2> dataStream2;
@@ -61,7 +61,7 @@ public class ConnectedDataStream<IN1, IN2> {
protected KeySelector<IN2, ?> keySelector2;
protected ConnectedDataStream(DataStream<IN1> input1, DataStream<IN2> input2) {
- this.jobGraphBuilder = input1.jobGraphBuilder;
+ this.jobGraphBuilder = input1.streamGraph;
this.environment = input1.environment;
this.dataStream1 = input1.copy();
this.dataStream2 = input2.copy();
@@ -402,7 +402,7 @@ public class ConnectedDataStream<IN1, IN2> {
SingleOutputStreamOperator<OUT, ?> returnStream = new SingleOutputStreamOperator(
environment, functionName, outTypeInfo, functionInvokable);
- dataStream1.jobGraphBuilder.addCoTask(returnStream.getId(), functionInvokable,
+ dataStream1.streamGraph.addCoTask(returnStream.getId(), functionInvokable,
getInputType1(), getInputType2(), outTypeInfo, functionName,
environment.getDegreeOfParallelism());
http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index f68ab68..8e7d823 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -43,7 +43,7 @@ import org.apache.flink.api.java.typeutils.TupleTypeInfo;
import org.apache.flink.api.java.typeutils.TypeExtractor;
import org.apache.flink.core.fs.FileSystem.WriteMode;
import org.apache.flink.core.fs.Path;
-import org.apache.flink.streaming.api.JobGraphBuilder;
+import org.apache.flink.streaming.api.StreamGraph;
import org.apache.flink.streaming.api.datastream.temporaloperator.StreamCrossOperator;
import org.apache.flink.streaming.api.datastream.temporaloperator.StreamJoinOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
@@ -102,7 +102,7 @@ public class DataStream<OUT> {
protected TypeInformation typeInfo;
protected List<DataStream<OUT>> mergedStreams;
- protected final JobGraphBuilder jobGraphBuilder;
+ protected final StreamGraph streamGraph;
/**
* Create a new {@link DataStream} in the given execution environment with
@@ -125,7 +125,7 @@ public class DataStream<OUT> {
this.id = operatorType + "-" + counter.toString();
this.environment = environment;
this.degreeOfParallelism = environment.getDegreeOfParallelism();
- this.jobGraphBuilder = environment.getJobGraphBuilder();
+ this.streamGraph = environment.getStreamGraph();
this.userDefinedNames = new ArrayList<String>();
this.selectAll = true;
this.partitioner = new DistributePartitioner<OUT>(true);
@@ -147,7 +147,7 @@ public class DataStream<OUT> {
this.userDefinedNames = new ArrayList<String>(dataStream.userDefinedNames);
this.selectAll = dataStream.selectAll;
this.partitioner = dataStream.partitioner;
- this.jobGraphBuilder = dataStream.jobGraphBuilder;
+ this.streamGraph = dataStream.streamGraph;
this.typeInfo = dataStream.typeInfo;
this.mergedStreams = new ArrayList<DataStream<OUT>>();
this.mergedStreams.add(this);
@@ -189,7 +189,7 @@ public class DataStream<OUT> {
@SuppressWarnings("unchecked")
public <R> DataStream<R> setType(TypeInformation<R> outType) {
- jobGraphBuilder.setOutType(id, outType);
+ streamGraph.setOutType(id, outType);
typeInfo = outType;
return (DataStream<R>) this;
}
@@ -1085,7 +1085,7 @@ public class DataStream<OUT> {
DataStream<R> returnStream = new DataStreamSource<R>(environment, "iterationSource", null,
null, true);
- jobGraphBuilder.addIterationHead(returnStream.getId(), this.getId(), iterationID,
+ streamGraph.addIterationHead(returnStream.getId(), this.getId(), iterationID,
degreeOfParallelism, waitTime);
return this.copy();
@@ -1112,7 +1112,7 @@ public class DataStream<OUT> {
SingleOutputStreamOperator<R, ?> returnStream = new SingleOutputStreamOperator(environment,
operatorName, outTypeInfo, invokable);
- jobGraphBuilder.addStreamVertex(returnStream.getId(), invokable, getType(), outTypeInfo,
+ streamGraph.addStreamVertex(returnStream.getId(), invokable, getType(), outTypeInfo,
operatorName, degreeOfParallelism);
connectGraph(inputStream, returnStream.getId(), 0);
@@ -1157,7 +1157,7 @@ public class DataStream<OUT> {
*/
protected <X> void connectGraph(DataStream<X> inputStream, String outputID, int typeNumber) {
for (DataStream<X> stream : inputStream.mergedStreams) {
- jobGraphBuilder.setEdge(stream.getId(), outputID, stream.partitioner, typeNumber,
+ streamGraph.setEdge(stream.getId(), outputID, stream.partitioner, typeNumber,
inputStream.userDefinedNames, inputStream.selectAll);
}
@@ -1179,7 +1179,7 @@ public class DataStream<OUT> {
DataStreamSink<OUT> returnStream = new DataStreamSink<OUT>(environment, "sink", getType(),
sinkInvokable);
- jobGraphBuilder.addStreamVertex(returnStream.getId(), sinkInvokable, getType(), null,
+ streamGraph.addStreamVertex(returnStream.getId(), sinkInvokable, getType(), null,
"sink", degreeOfParallelism);
this.connectGraph(this.copy(), returnStream.getId(), 0);
http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
index c4cd1e1..6f66b2c 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
@@ -64,10 +64,10 @@ public class IterativeDataStream<IN> extends
DataStream<IN> iterationSink = new DataStreamSink<IN>(environment, "iterationSink", null,
null);
- jobGraphBuilder.addIterationTail(iterationSink.getId(), iterationTail.getId(), iterationID,
+ streamGraph.addIterationTail(iterationSink.getId(), iterationTail.getId(), iterationID,
iterationTail.getParallelism(), waitTime);
- jobGraphBuilder.setIterationSourceSettings(iterationID.toString(), iterationTail.getId());
+ streamGraph.setIterationSourceSettings(iterationID.toString(), iterationTail.getId());
connectGraph(iterationTail.forward(), iterationSink.getId(), 0);
return iterationTail;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
index b8ca42c..dbfbc48 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
@@ -20,8 +20,6 @@ package org.apache.flink.streaming.api.datastream;
import java.util.Map;
import java.util.Map.Entry;
-import org.apache.commons.lang3.SerializationException;
-import org.apache.commons.lang3.SerializationUtils;
import org.apache.flink.api.common.functions.RichFunction;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.streaming.api.collector.OutputSelector;
@@ -65,7 +63,7 @@ public class SingleOutputStreamOperator<OUT, O extends SingleOutputStreamOperato
@SuppressWarnings("unchecked")
public <R> SingleOutputStreamOperator<R, ?> setType(TypeInformation<R> outType) {
- jobGraphBuilder.setOutType(id, outType);
+ streamGraph.setOutType(id, outType);
typeInfo = outType;
return (SingleOutputStreamOperator<R, ?>) this;
}
@@ -84,7 +82,7 @@ public class SingleOutputStreamOperator<OUT, O extends SingleOutputStreamOperato
}
this.degreeOfParallelism = dop;
- jobGraphBuilder.setParallelism(id, degreeOfParallelism);
+ streamGraph.setParallelism(id, degreeOfParallelism);
return this;
}
@@ -98,7 +96,7 @@ public class SingleOutputStreamOperator<OUT, O extends SingleOutputStreamOperato
* @return The operator with buffer timeout set.
*/
public SingleOutputStreamOperator<OUT, O> setBufferTimeout(long timeoutMillis) {
- jobGraphBuilder.setBufferTimeout(id, timeoutMillis);
+ streamGraph.setBufferTimeout(id, timeoutMillis);
return this;
}
@@ -115,13 +113,7 @@ public class SingleOutputStreamOperator<OUT, O extends SingleOutputStreamOperato
public SplitDataStream<OUT> split(OutputSelector<OUT> outputSelector) {
if (!isSplit) {
this.isSplit = true;
- try {
- jobGraphBuilder.setOutputSelector(id,
- SerializationUtils.serialize(clean(outputSelector)));
-
- } catch (SerializationException e) {
- throw new RuntimeException("Cannot serialize OutputSelector");
- }
+ streamGraph.setOutputSelector(id, clean(outputSelector));
return new SplitDataStream<OUT>(this);
} else {
@@ -144,7 +136,7 @@ public class SingleOutputStreamOperator<OUT, O extends SingleOutputStreamOperato
* @return The data stream with state registered.
*/
protected SingleOutputStreamOperator<OUT, O> registerState(String name, OperatorState<?> state) {
- jobGraphBuilder.addOperatorState(getId(), name, state);
+ streamGraph.addOperatorState(getId(), name, state);
return this;
}
@@ -161,7 +153,7 @@ public class SingleOutputStreamOperator<OUT, O extends SingleOutputStreamOperato
*/
protected SingleOutputStreamOperator<OUT, O> registerState(Map<String, OperatorState<?>> states) {
for (Entry<String, OperatorState<?>> entry : states.entrySet()) {
- jobGraphBuilder.addOperatorState(getId(), entry.getKey(), entry.getValue());
+ streamGraph.addOperatorState(getId(), entry.getKey(), entry.getValue());
}
return this;
http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamCrossOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamCrossOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamCrossOperator.java
index 8422400..03160c2 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamCrossOperator.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamCrossOperator.java
@@ -67,7 +67,7 @@ public class StreamCrossOperator<I1, I2> extends
@SuppressWarnings("unchecked")
public CrossWindow<I1, I2> every(long length) {
- ((CoWindowInvokable<I1, I2, ?>) jobGraphBuilder.getInvokable(id)).setSlideSize(length);
+ ((CoWindowInvokable<I1, I2, ?>) streamGraph.getInvokable(id)).setSlideSize(length);
return this;
}
@@ -90,7 +90,7 @@ public class StreamCrossOperator<I1, I2> extends
new CrossWindowFunction<I1, I2, R>(clean(function)), op.windowSize,
op.slideInterval, op.timeStamp1, op.timeStamp2);
- jobGraphBuilder.setInvokable(id, invokable);
+ streamGraph.setInvokable(id, invokable);
return setType(outTypeInfo);
http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamJoinOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamJoinOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamJoinOperator.java
index 626b9f1..d2b2032 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamJoinOperator.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamJoinOperator.java
@@ -245,7 +245,7 @@ public class StreamJoinOperator<I1, I2> extends
getJoinWindowFunction(joinFunction, predicate), predicate.op.windowSize,
predicate.op.slideInterval, predicate.op.timeStamp1, predicate.op.timeStamp2);
- jobGraphBuilder.setInvokable(id, invokable);
+ streamGraph.setInvokable(id, invokable);
return setType(outType);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java
index 3e44012..4824fca 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java
@@ -29,7 +29,7 @@ public class LocalStreamEnvironment extends StreamExecutionEnvironment {
*/
@Override
public void execute() throws Exception {
- ClusterUtil.runOnMiniCluster(this.jobGraphBuilder.getJobGraph(), getDegreeOfParallelism());
+ ClusterUtil.runOnMiniCluster(this.streamGraph.getJobGraph(), getDegreeOfParallelism());
}
/**
@@ -41,7 +41,7 @@ public class LocalStreamEnvironment extends StreamExecutionEnvironment {
*/
@Override
public void execute(String jobName) throws Exception {
- ClusterUtil.runOnMiniCluster(this.jobGraphBuilder.getJobGraph(jobName),
+ ClusterUtil.runOnMiniCluster(this.streamGraph.getJobGraph(jobName),
getDegreeOfParallelism());
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/RemoteStreamEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/RemoteStreamEnvironment.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/RemoteStreamEnvironment.java
index d833c8e..2eb05ad 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/RemoteStreamEnvironment.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/RemoteStreamEnvironment.java
@@ -81,14 +81,14 @@ public class RemoteStreamEnvironment extends StreamExecutionEnvironment {
@Override
public void execute() {
- JobGraph jobGraph = jobGraphBuilder.getJobGraph();
+ JobGraph jobGraph = streamGraph.getJobGraph();
executeRemotely(jobGraph);
}
@Override
public void execute(String jobName) {
- JobGraph jobGraph = jobGraphBuilder.getJobGraph(jobName);
+ JobGraph jobGraph = streamGraph.getJobGraph(jobName);
executeRemotely(jobGraph);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamContextEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamContextEnvironment.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamContextEnvironment.java
index c157435..a9a5bd3 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamContextEnvironment.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamContextEnvironment.java
@@ -54,9 +54,9 @@ public class StreamContextEnvironment extends StreamExecutionEnvironment {
JobGraph jobGraph;
if (jobName == null) {
- jobGraph = this.jobGraphBuilder.getJobGraph();
+ jobGraph = this.streamGraph.getJobGraph();
} else {
- jobGraph = this.jobGraphBuilder.getJobGraph(jobName);
+ jobGraph = this.streamGraph.getJobGraph(jobName);
}
for (File file : jars) {
http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
index 4194864..51dc0ae 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
@@ -33,7 +33,7 @@ import org.apache.flink.api.java.typeutils.TypeExtractor;
import org.apache.flink.client.program.Client;
import org.apache.flink.client.program.ContextEnvironment;
import org.apache.flink.core.fs.Path;
-import org.apache.flink.streaming.api.JobGraphBuilder;
+import org.apache.flink.streaming.api.StreamGraph;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.function.source.FileSourceFunction;
@@ -61,7 +61,7 @@ public abstract class StreamExecutionEnvironment {
private ExecutionConfig config = new ExecutionConfig();
- protected JobGraphBuilder jobGraphBuilder;
+ protected StreamGraph streamGraph;
// --------------------------------------------------------------------------------------------
// Constructor and Properties
@@ -71,7 +71,7 @@ public abstract class StreamExecutionEnvironment {
* Constructor for creating StreamExecutionEnvironment
*/
protected StreamExecutionEnvironment() {
- jobGraphBuilder = new JobGraphBuilder();
+ streamGraph = new StreamGraph();
}
/**
@@ -352,7 +352,7 @@ public abstract class StreamExecutionEnvironment {
TypeInformation<String> typeInfo) {
FileSourceFunction function = new FileSourceFunction(inputFormat, typeInfo);
DataStreamSource<String> returnStream = addSource(function, null, "fileSource");
- jobGraphBuilder.setInputFormat(returnStream.getId(), inputFormat);
+ streamGraph.setInputFormat(returnStream.getId(), inputFormat);
return returnStream;
}
@@ -437,7 +437,7 @@ public abstract class StreamExecutionEnvironment {
DataStreamSource<OUT> returnStream = new DataStreamSource<OUT>(this, sourceName,
outTypeInfo, sourceInvokable, isParallel);
- jobGraphBuilder.addSourceVertex(returnStream.getId(), sourceInvokable, null, outTypeInfo,
+ streamGraph.addSourceVertex(returnStream.getId(), sourceInvokable, null, outTypeInfo,
sourceName, dop);
return returnStream;
@@ -584,12 +584,12 @@ public abstract class StreamExecutionEnvironment {
public abstract void execute(String jobName) throws Exception;
/**
- * Getter of the {@link JobGraphBuilder} of the streaming job.
+ * Getter of the {@link StreamGraph} of the streaming job.
*
- * @return jobGraphBuilder
+ * @return The streamgraph representing the transformations
*/
- public JobGraphBuilder getJobGraphBuilder() {
- return jobGraphBuilder;
+ public StreamGraph getStreamGraph() {
+ return streamGraph;
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java
index 01c0545..c2177fa 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokable.java
@@ -34,7 +34,6 @@ public class GroupedReduceInvokable<IN> extends StreamReduceInvokable<IN> {
super(reducer);
this.keySelector = keySelector;
values = new HashMap<Object, IN>();
- setChainingStrategy(ChainingStrategy.NEVER);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
index bbd7b0c..a46fa96 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
@@ -67,7 +67,7 @@ import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
* method. Fake elements created on prenotification will be forwarded to all
* groups. The {@link ActiveTriggerCallback} is also implemented in a way, that
* it forwards/distributed calls all groups.
- *
+ *
* @param <IN>
* The type of input elements handled by this operator invokable.
*/
@@ -154,7 +154,6 @@ public class GroupedWindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
LinkedList<EvictionPolicy<IN>> centralEvictionPolicies) {
super(userFunction);
- setChainingStrategy(ChainingStrategy.NEVER);
this.keySelector = keySelector;
[5/8] flink git commit: [streaming] JobGraphbuilder separated to
StreamGraph and StreamingJobGraphGenerator
Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
index de9c664..ea891c9 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
@@ -61,8 +61,7 @@ public abstract class WindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT>
public WindowInvokable(Function userFunction, LinkedList<TriggerPolicy<IN>> triggerPolicies,
LinkedList<EvictionPolicy<IN>> evictionPolicies) {
super(userFunction);
- setChainingStrategy(ChainingStrategy.NEVER);
-
+
this.triggerPolicies = triggerPolicies;
this.evictionPolicies = evictionPolicies;
http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
index 99f826d..135f742 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/OutputHandler.java
@@ -24,6 +24,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
import org.apache.flink.runtime.plugable.SerializationDelegate;
import org.apache.flink.streaming.api.StreamConfig;
@@ -54,7 +55,7 @@ public class OutputHandler<OUT> {
private Map<String, StreamOutput<?>> outputMap;
private Map<String, StreamConfig> chainedConfigs;
- private List<String> recordWriterOrder;
+ private List<Tuple2<String, String>> outEdgesInOrder;
public OutputHandler(StreamVertex<?, OUT> vertex) {
@@ -68,24 +69,16 @@ public class OutputHandler<OUT> {
// We read the chained configs, and the order of record writer
// registrations by outputname
this.chainedConfigs = configuration.getTransitiveChainedTaskConfigs(cl);
- this.recordWriterOrder = configuration.getRecordWriterOrder(cl);
-
- // For the network outputs of the chain head we create the stream
- // outputs
- for (String outName : configuration.getOutputs(cl)) {
- StreamOutput<?> streamOutput = createStreamOutput(outName, configuration);
- outputMap.put(outName, streamOutput);
- }
-
- // If we have chained tasks we iterate through them and create the
- // stream outputs for the network outputs
- if (chainedConfigs != null) {
- for (StreamConfig config : chainedConfigs.values()) {
- for (String outName : config.getOutputs(cl)) {
- StreamOutput<?> streamOutput = createStreamOutput(outName, config);
- outputMap.put(outName, streamOutput);
- }
- }
+ this.chainedConfigs.put(configuration.getTaskName(), configuration);
+
+ this.outEdgesInOrder = configuration.getOutEdgesInOrder(cl);
+
+ // We iterate through all the out edges from this job vertex and create
+ // a stream output
+ for (Tuple2<String, String> outEdge : outEdgesInOrder) {
+ StreamOutput<?> streamOutput = createStreamOutput(outEdge.f1,
+ chainedConfigs.get(outEdge.f0), outEdgesInOrder.indexOf(outEdge));
+ outputMap.put(outEdge.f1, streamOutput);
}
// We create the outer collector that will be passed to the first task
@@ -196,40 +189,32 @@ public class OutputHandler<OUT> {
* We create the StreamOutput for the specific output given by the name, and
* the configuration of its source task
*
- * @param name
+ * @param outputVertex
* Name of the output to which the streamoutput will be set up
* @param configuration
* The config of upStream task
* @return
*/
- private <T> StreamOutput<T> createStreamOutput(String name, StreamConfig configuration) {
-
- int outputNumber = recordWriterOrder.indexOf(name);
-
- StreamPartitioner<T> outputPartitioner;
-
- try {
- outputPartitioner = configuration.getPartitioner(vertex.userClassLoader, name);
- } catch (Exception e) {
- throw new StreamVertexException("Cannot deserialize partitioner for "
- + vertex.getName() + " with " + name + " outputs", e);
- }
+ private <T> StreamOutput<T> createStreamOutput(String outputVertex, StreamConfig configuration,
+ int outputIndex) {
+
+ StreamPartitioner<T> outputPartitioner = configuration.getPartitioner(cl, outputVertex);
RecordWriter<SerializationDelegate<StreamRecord<T>>> output;
- long bufferTimeout = configuration.getBufferTimeout();
+ if (configuration.getBufferTimeout() >= 0) {
- if (bufferTimeout >= 0) {
output = new StreamRecordWriter<SerializationDelegate<StreamRecord<T>>>(vertex
- .getEnvironment().getWriter(outputNumber), outputPartitioner, bufferTimeout);
+ .getEnvironment().getWriter(outputIndex), outputPartitioner,
+ configuration.getBufferTimeout());
if (LOG.isTraceEnabled()) {
LOG.trace("StreamRecordWriter initiated with {} bufferTimeout for {}",
- bufferTimeout, vertex.getClass().getSimpleName());
+ configuration.getBufferTimeout(), vertex.getClass().getSimpleName());
}
} else {
output = new RecordWriter<SerializationDelegate<StreamRecord<T>>>(vertex
- .getEnvironment().getWriter(outputNumber), outputPartitioner);
+ .getEnvironment().getWriter(outputIndex), outputPartitioner);
if (LOG.isTraceEnabled()) {
LOG.trace("RecordWriter initiated for {}", vertex.getClass().getSimpleName());
@@ -237,11 +222,12 @@ public class OutputHandler<OUT> {
}
StreamOutput<T> streamOutput = new StreamOutput<T>(output,
- configuration.isSelectAll(name) ? null : configuration.getOutputNames(name));
+ configuration.isSelectAll(outputVertex) ? null
+ : configuration.getOutputNames(outputVertex));
if (LOG.isTraceEnabled()) {
LOG.trace("Partitioner set: {} with {} outputs for {}", outputPartitioner.getClass()
- .getSimpleName(), outputNumber, vertex.getClass().getSimpleName());
+ .getSimpleName(), outputIndex, vertex.getClass().getSimpleName());
}
return streamOutput;
http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/TestStreamEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/TestStreamEnvironment.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/TestStreamEnvironment.java
index e7f15b6..95a5d9b 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/TestStreamEnvironment.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/TestStreamEnvironment.java
@@ -46,7 +46,7 @@ public class TestStreamEnvironment extends StreamExecutionEnvironment {
@Override
public void execute(String jobName) throws Exception {
- JobGraph jobGraph = jobGraphBuilder.getJobGraph(jobName);
+ JobGraph jobGraph = streamGraph.getJobGraph(jobName);
Configuration configuration = jobGraph.getJobConfiguration();
http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamCrossOperator.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamCrossOperator.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamCrossOperator.scala
index a69454c..a408ec0 100644
--- a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamCrossOperator.scala
+++ b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamCrossOperator.scala
@@ -83,7 +83,7 @@ object StreamCrossOperator {
clean(getCrossWindowFunction(op, fun)), op.windowSize, op.slideInterval, op.timeStamp1,
op.timeStamp2)
- javaStream.getExecutionEnvironment().getJobGraphBuilder().setInvokable(javaStream.getId(),
+ javaStream.getExecutionEnvironment().getStreamGraph().setInvokable(javaStream.getId(),
invokable)
javaStream.setType(implicitly[TypeInformation[R]])
@@ -94,7 +94,7 @@ object StreamCrossOperator {
}
override def every(length: Long): CrossWindow[I1, I2] = {
- val builder = javaStream.getExecutionEnvironment().getJobGraphBuilder()
+ val builder = javaStream.getExecutionEnvironment().getStreamGraph()
val invokable = builder.getInvokable(javaStream.getId())
invokable.asInstanceOf[CoWindowInvokable[_,_,_]].setSlideSize(length)
this
http://git-wip-us.apache.org/repos/asf/flink/blob/e3b608ce/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamJoinOperator.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamJoinOperator.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamJoinOperator.scala
index 7ecd79a..1bd1bfb 100644
--- a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamJoinOperator.scala
+++ b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamJoinOperator.scala
@@ -186,7 +186,7 @@ object StreamJoinOperator {
clean(getJoinWindowFunction(jp, fun)), op.windowSize, op.slideInterval, op.timeStamp1,
op.timeStamp2)
- javaStream.getExecutionEnvironment().getJobGraphBuilder().setInvokable(javaStream.getId(),
+ javaStream.getExecutionEnvironment().getStreamGraph().setInvokable(javaStream.getId(),
invokable)
javaStream.setType(implicitly[TypeInformation[R]])