You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by mb...@apache.org on 2015/06/12 10:00:46 UTC
[1/3] flink git commit: [streaming] Logging update for checkpointed
streaming topologies
Repository: flink
Updated Branches:
refs/heads/master e966a0dd1 -> 32ddc9e60
[streaming] Logging update for checkpointed streaming topologies
Also adding a log message when delaying the retry of the execution graph.
Closes #819
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/078e6ef3
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/078e6ef3
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/078e6ef3
Branch: refs/heads/master
Commit: 078e6ef3ef0d455d6f405c21c1ca209b80418a23
Parents: e966a0d
Author: mbalassi <mb...@apache.org>
Authored: Wed Jun 10 14:46:11 2015 +0200
Committer: mbalassi <mb...@apache.org>
Committed: Fri Jun 12 09:54:51 2015 +0200
----------------------------------------------------------------------
.../org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java | 1 -
.../org/apache/flink/runtime/executiongraph/ExecutionGraph.java | 1 +
.../java/org/apache/flink/streaming/runtime/tasks/StreamTask.java | 2 +-
3 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/078e6ef3/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
index b52e732..424a9ed 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
@@ -60,7 +60,6 @@ public class CheckpointCoordinator {
/** The number of recent checkpoints whose IDs are remembered */
private static final int NUM_GHOST_CHECKPOINT_IDS = 16;
-
/** Coordinator-wide lock to safeguard the checkpoint updates */
private final Object lock = new Object();
http://git-wip-us.apache.org/repos/asf/flink/blob/078e6ef3/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
index 705496d..17eb3f6 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
@@ -710,6 +710,7 @@ public class ExecutionGraph implements Serializable {
@Override
public Object call() throws Exception {
try {
+ LOG.info("Delaying retry of job execution for {} ms ...", delayBeforeRetrying);
Thread.sleep(delayBeforeRetrying);
}
catch(InterruptedException e){
http://git-wip-us.apache.org/repos/asf/flink/blob/078e6ef3/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
index b55272c..db95dcc 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
@@ -233,7 +233,7 @@ public abstract class StreamTask<OUT, O extends StreamOperator<OUT>> extends Abs
synchronized (checkpointLock) {
if (isRunning) {
try {
- LOG.info("Starting checkpoint {} on task {}", checkpointId, getName());
+ LOG.debug("Starting checkpoint {} on task {}", checkpointId, getName());
// first draw the state that should go into checkpoint
StateHandle<Serializable> state;
[2/3] flink git commit: [build] [streaming] Streaming parents
dependencies pushed to children
Posted by mb...@apache.org.
[build] [streaming] Streaming parents dependencies pushed to children
Closes #820
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/9d6290e0
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/9d6290e0
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/9d6290e0
Branch: refs/heads/master
Commit: 9d6290e0e236f71835583fbe3ff28c893822f352
Parents: 078e6ef
Author: mbalassi <mb...@apache.org>
Authored: Wed Jun 10 16:09:14 2015 +0200
Committer: mbalassi <mb...@apache.org>
Committed: Fri Jun 12 09:57:24 2015 +0200
----------------------------------------------------------------------
.../flink-streaming-core/pom.xml | 18 ++++++++++++++
flink-staging/flink-streaming/pom.xml | 26 --------------------
2 files changed, 18 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/9d6290e0/flink-staging/flink-streaming/flink-streaming-core/pom.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/pom.xml b/flink-staging/flink-streaming/flink-streaming-core/pom.xml
index bc6c55c..1beadae 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/pom.xml
+++ b/flink-staging/flink-streaming/flink-streaming-core/pom.xml
@@ -37,6 +37,24 @@ under the License.
<dependencies>
<dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-runtime</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-clients</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-math</artifactId>
<version>2.2</version>
http://git-wip-us.apache.org/repos/asf/flink/blob/9d6290e0/flink-staging/flink-streaming/pom.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/pom.xml b/flink-staging/flink-streaming/pom.xml
index e7c23db..c2ff19d 100644
--- a/flink-staging/flink-streaming/pom.xml
+++ b/flink-staging/flink-streaming/pom.xml
@@ -40,30 +40,4 @@ under the License.
<module>flink-streaming-connectors</module>
</modules>
- <dependencies>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-core</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-runtime</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-clients</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-java</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
-
</project>
[3/3] flink git commit: [streaming] [api-breaking] Minor DataStream
cleanups
Posted by mb...@apache.org.
[streaming] [api-breaking] Minor DataStream cleanups
- Removed unused constructor parameter.
- Updated outdated and wrong connection javadocs.
Closes #825
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/32ddc9e6
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/32ddc9e6
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/32ddc9e6
Branch: refs/heads/master
Commit: 32ddc9e602327c0814a10626778796c1e3008078
Parents: 9d6290e
Author: mbalassi <mb...@apache.org>
Authored: Thu Jun 11 12:27:21 2015 +0200
Committer: mbalassi <mb...@apache.org>
Committed: Fri Jun 12 09:59:04 2015 +0200
----------------------------------------------------------------------
.../api/datastream/ConnectedDataStream.java | 2 +-
.../streaming/api/datastream/DataStream.java | 49 +++++++++++---------
.../api/datastream/DataStreamSink.java | 2 +-
.../api/datastream/DataStreamSource.java | 2 +-
.../datastream/SingleOutputStreamOperator.java | 4 +-
5 files changed, 33 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/32ddc9e6/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
index 0176277..2626d9c 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
@@ -482,7 +482,7 @@ public class ConnectedDataStream<IN1, IN2> {
@SuppressWarnings({ "unchecked", "rawtypes" })
SingleOutputStreamOperator<OUT, ?> returnStream = new SingleOutputStreamOperator(
- environment, functionName, outTypeInfo, operator);
+ environment, outTypeInfo, operator);
dataStream1.streamGraph.addCoOperator(returnStream.getId(), operator, getType1(),
getType2(), outTypeInfo, functionName);
http://git-wip-us.apache.org/repos/asf/flink/blob/32ddc9e6/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index 09f8155..6964a07 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -125,12 +125,10 @@ public class DataStream<OUT> {
*
* @param environment
* StreamExecutionEnvironment
- * @param operatorType
- * The type of the operator in the component
* @param typeInfo
* Type of the datastream
*/
- public DataStream(StreamExecutionEnvironment environment, String operatorType, TypeInformation<OUT> typeInfo) {
+ public DataStream(StreamExecutionEnvironment environment, TypeInformation<OUT> typeInfo) {
if (environment == null) {
throw new NullPointerException("context is null");
}
@@ -174,7 +172,7 @@ public class DataStream<OUT> {
}
/**
- * Returns the ID of the {@link DataStream}.
+ * Returns the ID of the {@link DataStream} in the current {@link StreamExecutionEnvironment}.
*
* @return ID of the DataStream
*/
@@ -420,9 +418,11 @@ public class DataStream<OUT> {
/**
* Sets the partitioning of the {@link DataStream} so that the output tuples
- * are broadcasted to every parallel instance of the next component. This
- * setting only effects the how the outputs will be distributed between the
- * parallel instances of the next processing operator.
+ * are broadcasted to every parallel instance of the next component.
+ *
+ * <p>
+ * This setting only effects the how the outputs will be distributed between
+ * the parallel instances of the next processing operator.
*
* @return The DataStream with broadcast partitioning set.
*/
@@ -432,9 +432,11 @@ public class DataStream<OUT> {
/**
* Sets the partitioning of the {@link DataStream} so that the output tuples
- * are shuffled to the next component. This setting only effects the how the
- * outputs will be distributed between the parallel instances of the next
- * processing operator.
+ * are shuffled uniformly randomly to the next component.
+ *
+ * <p>
+ * This setting only effects the how the outputs will be distributed between
+ * the parallel instances of the next processing operator.
*
* @return The DataStream with shuffle partitioning set.
*/
@@ -445,11 +447,13 @@ public class DataStream<OUT> {
/**
* Sets the partitioning of the {@link DataStream} so that the output tuples
* are forwarded to the local subtask of the next component (whenever
- * possible). This is the default partitioner setting. This setting only
- * effects the how the outputs will be distributed between the parallel
- * instances of the next processing operator.
+ * possible).
+ *
+ * <p>
+ * This setting only effects the how the outputs will be distributed between
+ * the parallel instances of the next processing operator.
*
- * @return The DataStream with shuffle partitioning set.
+ * @return The DataStream with forward partitioning set.
*/
public DataStream<OUT> forward() {
return setConnectionType(new RebalancePartitioner<OUT>(true));
@@ -457,11 +461,14 @@ public class DataStream<OUT> {
/**
* Sets the partitioning of the {@link DataStream} so that the output tuples
- * are distributed evenly to the next component.This setting only effects
- * the how the outputs will be distributed between the parallel instances of
- * the next processing operator.
+ * are distributed evenly to instances of the next component in a Round-robin
+ * fashion.
+ *
+ * <p>
+ * This setting only effects the how the outputs will be distributed between
+ * the parallel instances of the next processing operator.
*
- * @return The DataStream with shuffle partitioning set.
+ * @return The DataStream with rebalance partitioning set.
*/
public DataStream<OUT> rebalance() {
return setConnectionType(new RebalancePartitioner<OUT>(false));
@@ -1237,7 +1244,7 @@ public class DataStream<OUT> {
/**
* Method for passing user defined operators along with the type
- * informations that will transform the DataStream.
+ * information that will transform the DataStream.
*
* @param operatorName
* name of the operator, for logging purposes
@@ -1254,7 +1261,7 @@ public class DataStream<OUT> {
DataStream<OUT> inputStream = this.copy();
@SuppressWarnings({ "unchecked", "rawtypes" })
SingleOutputStreamOperator<R, ?> returnStream = new SingleOutputStreamOperator(environment,
- operatorName, outTypeInfo, operator);
+ outTypeInfo, operator);
streamGraph.addOperator(returnStream.getId(), operator, getType(), outTypeInfo,
operatorName);
@@ -1337,7 +1344,7 @@ public class DataStream<OUT> {
}
/**
- * Gets the class of the field at the given position
+ * Gets the class of the field at the given position.
*
* @param pos
* Position of the field
http://git-wip-us.apache.org/repos/asf/flink/blob/32ddc9e6/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
index dc457e0..60dc367 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
@@ -31,7 +31,7 @@ public class DataStreamSink<IN> extends SingleOutputStreamOperator<IN, DataStrea
protected DataStreamSink(StreamExecutionEnvironment environment, String operatorType,
TypeInformation<IN> outTypeInfo, OneInputStreamOperator<IN, ?> operator) {
- super(environment, operatorType, outTypeInfo, operator);
+ super(environment, outTypeInfo, operator);
}
protected DataStreamSink(DataStream<IN> dataStream) {
http://git-wip-us.apache.org/repos/asf/flink/blob/32ddc9e6/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java
index 2bb70df..0dd7701 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java
@@ -34,7 +34,7 @@ public class DataStreamSource<OUT> extends SingleOutputStreamOperator<OUT, DataS
public DataStreamSource(StreamExecutionEnvironment environment, String operatorType,
TypeInformation<OUT> outTypeInfo, StreamOperator<OUT> operator,
boolean isParallel, String sourceName) {
- super(environment, operatorType, outTypeInfo, operator);
+ super(environment, outTypeInfo, operator);
environment.getStreamGraph().addSource(getId(), operator, null, outTypeInfo,
sourceName);
http://git-wip-us.apache.org/repos/asf/flink/blob/32ddc9e6/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
index 1e5b5cf..b4a99c8 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
@@ -64,8 +64,8 @@ public class SingleOutputStreamOperator<OUT, O extends SingleOutputStreamOperato
}
protected SingleOutputStreamOperator(StreamExecutionEnvironment environment,
- String operatorType, TypeInformation<OUT> outTypeInfo, StreamOperator<?> operator) {
- super(environment, operatorType, outTypeInfo);
+ TypeInformation<OUT> outTypeInfo, StreamOperator<?> operator) {
+ super(environment, outTypeInfo);
this.isSplit = false;
this.operator = operator;
}