You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by mb...@apache.org on 2015/06/12 10:00:46 UTC

[1/3] flink git commit: [streaming] Logging update for checkpointed streaming topologies

Repository: flink
Updated Branches:
  refs/heads/master e966a0dd1 -> 32ddc9e60


[streaming] Logging update for checkpointed streaming topologies

Also adding a log message when delaying the retry of the execution graph.

Closes #819


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/078e6ef3
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/078e6ef3
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/078e6ef3

Branch: refs/heads/master
Commit: 078e6ef3ef0d455d6f405c21c1ca209b80418a23
Parents: e966a0d
Author: mbalassi <mb...@apache.org>
Authored: Wed Jun 10 14:46:11 2015 +0200
Committer: mbalassi <mb...@apache.org>
Committed: Fri Jun 12 09:54:51 2015 +0200

----------------------------------------------------------------------
 .../org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java | 1 -
 .../org/apache/flink/runtime/executiongraph/ExecutionGraph.java    | 1 +
 .../java/org/apache/flink/streaming/runtime/tasks/StreamTask.java  | 2 +-
 3 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/078e6ef3/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
index b52e732..424a9ed 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
@@ -60,7 +60,6 @@ public class CheckpointCoordinator {
 	/** The number of recent checkpoints whose IDs are remembered */
 	private static final int NUM_GHOST_CHECKPOINT_IDS = 16;
 	
-	
 	/** Coordinator-wide lock to safeguard the checkpoint updates */
 	private final Object lock = new Object();
 	

http://git-wip-us.apache.org/repos/asf/flink/blob/078e6ef3/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
index 705496d..17eb3f6 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
@@ -710,6 +710,7 @@ public class ExecutionGraph implements Serializable {
 								@Override
 								public Object call() throws Exception {
 									try {
+										LOG.info("Delaying retry of job execution for {} ms ...", delayBeforeRetrying);
 										Thread.sleep(delayBeforeRetrying);
 									}
 									catch(InterruptedException e){

http://git-wip-us.apache.org/repos/asf/flink/blob/078e6ef3/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
index b55272c..db95dcc 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
@@ -233,7 +233,7 @@ public abstract class StreamTask<OUT, O extends StreamOperator<OUT>> extends Abs
 		synchronized (checkpointLock) {
 			if (isRunning) {
 				try {
-					LOG.info("Starting checkpoint {} on task {}", checkpointId, getName());
+					LOG.debug("Starting checkpoint {} on task {}", checkpointId, getName());
 					
 					// first draw the state that should go into checkpoint
 					StateHandle<Serializable> state;


[2/3] flink git commit: [build] [streaming] Streaming parents dependencies pushed to children

Posted by mb...@apache.org.
[build] [streaming] Streaming parents dependencies pushed to children

Closes #820


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/9d6290e0
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/9d6290e0
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/9d6290e0

Branch: refs/heads/master
Commit: 9d6290e0e236f71835583fbe3ff28c893822f352
Parents: 078e6ef
Author: mbalassi <mb...@apache.org>
Authored: Wed Jun 10 16:09:14 2015 +0200
Committer: mbalassi <mb...@apache.org>
Committed: Fri Jun 12 09:57:24 2015 +0200

----------------------------------------------------------------------
 .../flink-streaming-core/pom.xml                | 18 ++++++++++++++
 flink-staging/flink-streaming/pom.xml           | 26 --------------------
 2 files changed, 18 insertions(+), 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/9d6290e0/flink-staging/flink-streaming/flink-streaming-core/pom.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/pom.xml b/flink-staging/flink-streaming/flink-streaming-core/pom.xml
index bc6c55c..1beadae 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/pom.xml
+++ b/flink-staging/flink-streaming/flink-streaming-core/pom.xml
@@ -37,6 +37,24 @@ under the License.
 	<dependencies>
 
 		<dependency>
+			<groupId>org.apache.flink</groupId>
+			<artifactId>flink-core</artifactId>
+			<version>${project.version}</version>
+		</dependency>
+
+		<dependency>
+			<groupId>org.apache.flink</groupId>
+			<artifactId>flink-runtime</artifactId>
+			<version>${project.version}</version>
+		</dependency>
+
+		<dependency>
+			<groupId>org.apache.flink</groupId>
+			<artifactId>flink-clients</artifactId>
+			<version>${project.version}</version>
+		</dependency>
+
+		<dependency>
 			<groupId>org.apache.commons</groupId>
 			<artifactId>commons-math</artifactId>
 			<version>2.2</version>

http://git-wip-us.apache.org/repos/asf/flink/blob/9d6290e0/flink-staging/flink-streaming/pom.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/pom.xml b/flink-staging/flink-streaming/pom.xml
index e7c23db..c2ff19d 100644
--- a/flink-staging/flink-streaming/pom.xml
+++ b/flink-staging/flink-streaming/pom.xml
@@ -40,30 +40,4 @@ under the License.
 		<module>flink-streaming-connectors</module>
 	</modules>
 	
-	<dependencies>
-		<dependency>
-			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-core</artifactId>
-			<version>${project.version}</version>
-		</dependency>
-
-		<dependency>
-			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-runtime</artifactId>
-			<version>${project.version}</version>
-		</dependency>
-
-		<dependency>
-			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-clients</artifactId>
-			<version>${project.version}</version>
-		</dependency>
-
-		<dependency>
-			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-java</artifactId>
-			<version>${project.version}</version>
-		</dependency>
-	</dependencies>
-
 </project>


[3/3] flink git commit: [streaming] [api-breaking] Minor DataStream cleanups

Posted by mb...@apache.org.
[streaming] [api-breaking] Minor DataStream cleanups

- Removed unused constructor parameter.
- Updated outdated and wrong connection javadocs.

Closes #825


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/32ddc9e6
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/32ddc9e6
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/32ddc9e6

Branch: refs/heads/master
Commit: 32ddc9e602327c0814a10626778796c1e3008078
Parents: 9d6290e
Author: mbalassi <mb...@apache.org>
Authored: Thu Jun 11 12:27:21 2015 +0200
Committer: mbalassi <mb...@apache.org>
Committed: Fri Jun 12 09:59:04 2015 +0200

----------------------------------------------------------------------
 .../api/datastream/ConnectedDataStream.java     |  2 +-
 .../streaming/api/datastream/DataStream.java    | 49 +++++++++++---------
 .../api/datastream/DataStreamSink.java          |  2 +-
 .../api/datastream/DataStreamSource.java        |  2 +-
 .../datastream/SingleOutputStreamOperator.java  |  4 +-
 5 files changed, 33 insertions(+), 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/32ddc9e6/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
index 0176277..2626d9c 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
@@ -482,7 +482,7 @@ public class ConnectedDataStream<IN1, IN2> {
 
 		@SuppressWarnings({ "unchecked", "rawtypes" })
 		SingleOutputStreamOperator<OUT, ?> returnStream = new SingleOutputStreamOperator(
-				environment, functionName, outTypeInfo, operator);
+				environment, outTypeInfo, operator);
 
 		dataStream1.streamGraph.addCoOperator(returnStream.getId(), operator, getType1(),
 				getType2(), outTypeInfo, functionName);

http://git-wip-us.apache.org/repos/asf/flink/blob/32ddc9e6/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index 09f8155..6964a07 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -125,12 +125,10 @@ public class DataStream<OUT> {
 	 * 
 	 * @param environment
 	 *            StreamExecutionEnvironment
-	 * @param operatorType
-	 *            The type of the operator in the component
 	 * @param typeInfo
 	 *            Type of the datastream
 	 */
-	public DataStream(StreamExecutionEnvironment environment, String operatorType, TypeInformation<OUT> typeInfo) {
+	public DataStream(StreamExecutionEnvironment environment, TypeInformation<OUT> typeInfo) {
 		if (environment == null) {
 			throw new NullPointerException("context is null");
 		}
@@ -174,7 +172,7 @@ public class DataStream<OUT> {
 	}
 
 	/**
-	 * Returns the ID of the {@link DataStream}.
+	 * Returns the ID of the {@link DataStream} in the current {@link StreamExecutionEnvironment}.
 	 * 
 	 * @return ID of the DataStream
 	 */
@@ -420,9 +418,11 @@ public class DataStream<OUT> {
 
 	/**
 	 * Sets the partitioning of the {@link DataStream} so that the output tuples
-	 * are broadcasted to every parallel instance of the next component. This
-	 * setting only effects the how the outputs will be distributed between the
-	 * parallel instances of the next processing operator.
+	 * are broadcasted to every parallel instance of the next component.
+	 *
+	 * <p>
+	 * This setting only effects the how the outputs will be distributed between
+	 * the parallel instances of the next processing operator.
 	 * 
 	 * @return The DataStream with broadcast partitioning set.
 	 */
@@ -432,9 +432,11 @@ public class DataStream<OUT> {
 
 	/**
 	 * Sets the partitioning of the {@link DataStream} so that the output tuples
-	 * are shuffled to the next component. This setting only effects the how the
-	 * outputs will be distributed between the parallel instances of the next
-	 * processing operator.
+	 * are shuffled uniformly randomly to the next component.
+	 *
+	 * <p>
+	 * This setting only effects the how the outputs will be distributed between
+	 * the parallel instances of the next processing operator.
 	 * 
 	 * @return The DataStream with shuffle partitioning set.
 	 */
@@ -445,11 +447,13 @@ public class DataStream<OUT> {
 	/**
 	 * Sets the partitioning of the {@link DataStream} so that the output tuples
 	 * are forwarded to the local subtask of the next component (whenever
-	 * possible). This is the default partitioner setting. This setting only
-	 * effects the how the outputs will be distributed between the parallel
-	 * instances of the next processing operator.
+	 * possible).
+	 *
+	 * <p>
+	 * This setting only effects the how the outputs will be distributed between
+	 * the parallel instances of the next processing operator.
 	 * 
-	 * @return The DataStream with shuffle partitioning set.
+	 * @return The DataStream with forward partitioning set.
 	 */
 	public DataStream<OUT> forward() {
 		return setConnectionType(new RebalancePartitioner<OUT>(true));
@@ -457,11 +461,14 @@ public class DataStream<OUT> {
 
 	/**
 	 * Sets the partitioning of the {@link DataStream} so that the output tuples
-	 * are distributed evenly to the next component.This setting only effects
-	 * the how the outputs will be distributed between the parallel instances of
-	 * the next processing operator.
+	 * are distributed evenly to instances of the next component in a Round-robin
+	 * fashion.
+	 *
+	 * <p>
+	 * This setting only effects the how the outputs will be distributed between
+	 * the parallel instances of the next processing operator.
 	 * 
-	 * @return The DataStream with shuffle partitioning set.
+	 * @return The DataStream with rebalance partitioning set.
 	 */
 	public DataStream<OUT> rebalance() {
 		return setConnectionType(new RebalancePartitioner<OUT>(false));
@@ -1237,7 +1244,7 @@ public class DataStream<OUT> {
 
 	/**
 	 * Method for passing user defined operators along with the type
-	 * informations that will transform the DataStream.
+	 * information that will transform the DataStream.
 	 * 
 	 * @param operatorName
 	 *            name of the operator, for logging purposes
@@ -1254,7 +1261,7 @@ public class DataStream<OUT> {
 		DataStream<OUT> inputStream = this.copy();
 		@SuppressWarnings({ "unchecked", "rawtypes" })
 		SingleOutputStreamOperator<R, ?> returnStream = new SingleOutputStreamOperator(environment,
-				operatorName, outTypeInfo, operator);
+				outTypeInfo, operator);
 
 		streamGraph.addOperator(returnStream.getId(), operator, getType(), outTypeInfo,
 				operatorName);
@@ -1337,7 +1344,7 @@ public class DataStream<OUT> {
 	}
 
 	/**
-	 * Gets the class of the field at the given position
+	 * Gets the class of the field at the given position.
 	 * 
 	 * @param pos
 	 *            Position of the field

http://git-wip-us.apache.org/repos/asf/flink/blob/32ddc9e6/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
index dc457e0..60dc367 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
@@ -31,7 +31,7 @@ public class DataStreamSink<IN> extends SingleOutputStreamOperator<IN, DataStrea
 
 	protected DataStreamSink(StreamExecutionEnvironment environment, String operatorType,
 			TypeInformation<IN> outTypeInfo, OneInputStreamOperator<IN, ?> operator) {
-		super(environment, operatorType, outTypeInfo, operator);
+		super(environment, outTypeInfo, operator);
 	}
 
 	protected DataStreamSink(DataStream<IN> dataStream) {

http://git-wip-us.apache.org/repos/asf/flink/blob/32ddc9e6/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java
index 2bb70df..0dd7701 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java
@@ -34,7 +34,7 @@ public class DataStreamSource<OUT> extends SingleOutputStreamOperator<OUT, DataS
 	public DataStreamSource(StreamExecutionEnvironment environment, String operatorType,
 			TypeInformation<OUT> outTypeInfo, StreamOperator<OUT> operator,
 			boolean isParallel, String sourceName) {
-		super(environment, operatorType, outTypeInfo, operator);
+		super(environment, outTypeInfo, operator);
 
 		environment.getStreamGraph().addSource(getId(), operator, null, outTypeInfo,
 				sourceName);

http://git-wip-us.apache.org/repos/asf/flink/blob/32ddc9e6/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
index 1e5b5cf..b4a99c8 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
@@ -64,8 +64,8 @@ public class SingleOutputStreamOperator<OUT, O extends SingleOutputStreamOperato
 	}
 
 	protected SingleOutputStreamOperator(StreamExecutionEnvironment environment,
-			String operatorType, TypeInformation<OUT> outTypeInfo, StreamOperator<?> operator) {
-		super(environment, operatorType, outTypeInfo);
+			TypeInformation<OUT> outTypeInfo, StreamOperator<?> operator) {
+		super(environment, outTypeInfo);
 		this.isSplit = false;
 		this.operator = operator;
 	}