You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by mx...@apache.org on 2015/11/03 16:27:16 UTC
[1/4] flink git commit: [FLINK-2958] Remove hard coded number of
execution retries
Repository: flink
Updated Branches:
refs/heads/master ede80e227 -> 8f74718b1
[FLINK-2958] Remove hard coded number of execution retries
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/8f74718b
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/8f74718b
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/8f74718b
Branch: refs/heads/master
Commit: 8f74718b1e0a7309e2c29aad1f99891b56fb845e
Parents: aa1c145
Author: Ufuk Celebi <uc...@apache.org>
Authored: Tue Nov 3 12:06:46 2015 +0100
Committer: Maximilian Michels <mx...@apache.org>
Committed: Tue Nov 3 16:25:49 2015 +0100
----------------------------------------------------------------------
.../java/org/apache/flink/api/common/ExecutionConfig.java | 6 +++---
.../flink/streaming/api/graph/StreamingJobGraphGenerator.java | 7 +------
2 files changed, 4 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/8f74718b/flink-core/src/main/java/org/apache/flink/api/common/ExecutionConfig.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/ExecutionConfig.java b/flink-core/src/main/java/org/apache/flink/api/common/ExecutionConfig.java
index b620796..b031441 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/ExecutionConfig.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/ExecutionConfig.java
@@ -246,7 +246,7 @@ public class ExecutionConfig implements Serializable {
public int getNumberOfExecutionRetries() {
return numberOfExecutionRetries;
}
-
+
/**
* Returns the delay between execution retries.
*/
@@ -269,9 +269,9 @@ public class ExecutionConfig implements Serializable {
this.numberOfExecutionRetries = numberOfExecutionRetries;
return this;
}
-
+
/**
- * Sets the delay between executions. A value of {@code -1} indicates that the default value
+ * Sets the delay between executions. A value of {@code -1} indicates that the default value
* should be used.
* @param executionRetryDelay The number of milliseconds the system will wait to retry.
*/
http://git-wip-us.apache.org/repos/asf/flink/blob/8f74718b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
index d7d20af..a829a8d 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
@@ -421,12 +421,7 @@ public class StreamingJobGraphGenerator {
private void configureExecutionRetries() {
int executionRetries = streamGraph.getExecutionConfig().getNumberOfExecutionRetries();
- if (executionRetries != -1) {
- jobGraph.setNumberOfExecutionRetries(executionRetries);
- } else {
- // if the user didn't configure anything, the number of retries is 0.
- jobGraph.setNumberOfExecutionRetries(0);
- }
+ jobGraph.setNumberOfExecutionRetries(executionRetries);
}
private void configureExecutionRetryDelay() {
[4/4] flink git commit: [release][scripts] deploy Scala 2.11 version
to Maven
Posted by mx...@apache.org.
[release][scripts] deploy Scala 2.11 version to Maven
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/66b44209
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/66b44209
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/66b44209
Branch: refs/heads/master
Commit: 66b4420934ce3d30e4b3476bfcc5c3183de4f405
Parents: ede80e2
Author: Maximilian Michels <mx...@apache.org>
Authored: Tue Nov 3 10:26:03 2015 +0100
Committer: Maximilian Michels <mx...@apache.org>
Committed: Tue Nov 3 16:25:49 2015 +0100
----------------------------------------------------------------------
tools/create_release_files.sh | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/66b44209/tools/create_release_files.sh
----------------------------------------------------------------------
diff --git a/tools/create_release_files.sh b/tools/create_release_files.sh
index 7ba2206..7c71bad 100755
--- a/tools/create_release_files.sh
+++ b/tools/create_release_files.sh
@@ -173,6 +173,9 @@ deploy_to_maven() {
cp ../../deploysettings.xml .
echo "For your reference, the command:\n\t $MVN clean deploy -Prelease --settings deploysettings.xml -DskipTests -Dgpg.keyname=$GPG_KEY -Dgpg.passphrase=$GPG_PASSPHRASE ./tools/generate_specific_pom.sh $NEW_VERSION $NEW_VERSION_HADOOP1 pom.xml"
$MVN clean deploy -Prelease,docs-and-source --settings deploysettings.xml -DskipTests -Dgpg.executable=$GPG -Dgpg.keyname=$GPG_KEY -Dgpg.passphrase=$GPG_PASSPHRASE -DretryFailedDeploymentCount=10
+ cd tools && ./change-scala-version.sh 2.11 && cd ..
+ $MVN clean deploy -Dgpg.executable=$GPG -Prelease,docs-and-source --settings deploysettings.xml -DskipTests -Dgpg.keyname=$GPG_KEY -Dgpg.passphrase=$GPG_PASSPHRASE -DretryFailedDeploymentCount=10
+ cd tools && ../change-scala-version.sh 2.10 && cd ..
../generate_specific_pom.sh $NEW_VERSION $NEW_VERSION_HADOOP1 pom.xml
sleep 4
$MVN clean deploy -Dgpg.executable=$GPG -Prelease,docs-and-source --settings deploysettings.xml -DskipTests -Dgpg.keyname=$GPG_KEY -Dgpg.passphrase=$GPG_PASSPHRASE -DretryFailedDeploymentCount=10
[2/4] flink git commit: [docker] move the #!bash to first line
Posted by mx...@apache.org.
[docker] move the #!bash to first line
This closes #1314.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/f340bc72
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/f340bc72
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/f340bc72
Branch: refs/heads/master
Commit: f340bc727d7431d3ae02b70290d7055338a255f5
Parents: 66b4420
Author: Patrice Freydiere <fr...@gmail.com>
Authored: Sun Nov 1 22:37:01 2015 +0100
Committer: Maximilian Michels <mx...@apache.org>
Committed: Tue Nov 3 16:25:49 2015 +0100
----------------------------------------------------------------------
flink-contrib/docker-flink/flink/config-flink.sh | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/f340bc72/flink-contrib/docker-flink/flink/config-flink.sh
----------------------------------------------------------------------
diff --git a/flink-contrib/docker-flink/flink/config-flink.sh b/flink-contrib/docker-flink/flink/config-flink.sh
index 72f2828..ba77039 100755
--- a/flink-contrib/docker-flink/flink/config-flink.sh
+++ b/flink-contrib/docker-flink/flink/config-flink.sh
@@ -1,3 +1,5 @@
+#!/bin/bash
+
################################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -17,7 +19,7 @@
################################################################################
-#!/bin/bash
+
CONF=/usr/local/flink/conf
EXEC=/usr/local/flink/bin
@@ -52,4 +54,4 @@ echo "export F=/usr/local/flink/;" >> ~/.profile
#echo 'export FLINK_SSH_OPTS="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"' >> ~/.profile
#run ssh server and supervisor to keep container running.
-/usr/sbin/sshd && supervisord -c /etc/supervisor/supervisor.conf
\ No newline at end of file
+/usr/sbin/sshd && supervisord -c /etc/supervisor/supervisor.conf
[3/4] flink git commit: [FLINK-2930] Respect ExecutionConfig
execution retry delay
Posted by mx...@apache.org.
[FLINK-2930] Respect ExecutionConfig execution retry delay
- fix hard-coded defaults
This closes #1304.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/aa1c1454
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/aa1c1454
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/aa1c1454
Branch: refs/heads/master
Commit: aa1c145481f26320d4b1a2b3b207868bd23d8d04
Parents: f340bc7
Author: Ufuk Celebi <uc...@apache.org>
Authored: Tue Oct 27 18:15:15 2015 +0100
Committer: Maximilian Michels <mx...@apache.org>
Committed: Tue Nov 3 16:25:49 2015 +0100
----------------------------------------------------------------------
.../flink/api/common/ExecutionConfig.java | 2 +-
.../flink/api/java/ExecutionEnvironment.java | 25 --------------------
.../flink/runtime/jobmanager/JobManager.scala | 10 +++++++-
.../flink/api/scala/ExecutionEnvironment.scala | 16 -------------
.../environment/StreamExecutionEnvironment.java | 24 -------------------
.../api/graph/StreamingJobGraphGenerator.java | 14 +++--------
.../api/scala/StreamExecutionEnvironment.scala | 17 -------------
7 files changed, 13 insertions(+), 95 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/aa1c1454/flink-core/src/main/java/org/apache/flink/api/common/ExecutionConfig.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/ExecutionConfig.java b/flink-core/src/main/java/org/apache/flink/api/common/ExecutionConfig.java
index b9ebaf7..b620796 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/ExecutionConfig.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/ExecutionConfig.java
@@ -248,7 +248,7 @@ public class ExecutionConfig implements Serializable {
}
/**
- * @return The delay between retires.
+ * Returns the delay between execution retries.
*/
public long getExecutionRetryDelay() {
return executionRetryDelay;
http://git-wip-us.apache.org/repos/asf/flink/blob/aa1c1454/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java b/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
index 283d6d4..5405d4e 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
@@ -199,31 +199,6 @@ public abstract class ExecutionEnvironment {
}
/**
- * Sets the delay that failed tasks are re-executed in milliseconds. A value of
- * zero effectively disables fault tolerance. A value of {@code -1}
- * indicates that the system default value (as defined in the configuration)
- * should be used.
- *
- * @param executionRetryDelay
- * The delay of time the system will wait to re-execute failed
- * tasks.
- */
- public void setExecutionRetryDelay(long executionRetryDelay) {
- config.setExecutionRetryDelay(executionRetryDelay);
- }
-
- /**
- * Gets the delay time in milliseconds the system will wait to re-execute failed tasks.
- * A value of {@code -1} indicates that the system default value (as defined
- * in the configuration) should be used.
- *
- * @return The delay time the system will wait to re-execute failed tasks.
- */
- public long getExecutionRetryDelay() {
- return config.getExecutionRetryDelay();
- }
-
- /**
* Returns the {@link org.apache.flink.api.common.JobExecutionResult} of the last executed job.
*
* @return The execution result from the latest job execution.
http://git-wip-us.apache.org/repos/asf/flink/blob/aa1c1454/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
index d9b69ad..567e67b 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
@@ -818,8 +818,16 @@ class JobManager(
} else {
defaultExecutionRetries
}
+
+ val executionRetryDelay = if (jobGraph.getExecutionRetryDelay() >= 0) {
+ jobGraph.getExecutionRetryDelay()
+ }
+ else {
+ delayBetweenRetries
+ }
+
executionGraph.setNumberOfRetriesLeft(jobNumberRetries)
- executionGraph.setDelayBeforeRetrying(delayBetweenRetries)
+ executionGraph.setDelayBeforeRetrying(executionRetryDelay)
executionGraph.setScheduleMode(jobGraph.getScheduleMode())
executionGraph.setQueuedSchedulingAllowed(jobGraph.getAllowQueuedScheduling())
http://git-wip-us.apache.org/repos/asf/flink/blob/aa1c1454/flink-scala/src/main/scala/org/apache/flink/api/scala/ExecutionEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/ExecutionEnvironment.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/ExecutionEnvironment.scala
index e27d55a..9238f5c 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/ExecutionEnvironment.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/ExecutionEnvironment.scala
@@ -110,22 +110,6 @@ class ExecutionEnvironment(javaEnv: JavaEnv) {
def getNumberOfExecutionRetries = javaEnv.getNumberOfExecutionRetries
/**
- * Sets the delay that failed tasks are re-executed. A value of
- * zero effectively disables fault tolerance. A value of "-1"
- * indicates that the system default value (as defined in the configuration)
- * should be used.
- */
- def setExecutionRetryDelay(executionRetryDelay: Long): Unit = {
- javaEnv.setExecutionRetryDelay(executionRetryDelay)
- }
-
- /**
- * Gets the delay time in milliseconds the system will wait to re-execute failed tasks.
- * A value of "-1" indicates that the system default value (as defined
- * in the configuration) should be used.
- */
- def getExecutionRetryDelay = javaEnv.getExecutionRetryDelay
- /**
* Gets the UUID by which this environment is identified. The UUID sets the execution context
* in the cluster or local environment.
*/
http://git-wip-us.apache.org/repos/asf/flink/blob/aa1c1454/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
index 210447d..3c961f0 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
@@ -426,30 +426,6 @@ public abstract class StreamExecutionEnvironment {
}
/**
- * Sets the delay that failed tasks are re-executed. A value of
- * zero effectively disables fault tolerance. A value of {@code -1}
- * indicates that the system default value (as defined in the configuration)
- * should be used.
- *
- * @param executionRetryDelay
- * The delay of time the system will wait to re-execute failed
- * tasks.
- */
- public void setExecutionRetryDelay(long executionRetryDelay) {
- config.setExecutionRetryDelay(executionRetryDelay);
- }
-
- /**
- * Gets the delay time in milliseconds the system will wait to re-execute failed tasks.
- * A value of {@code -1} indicates that the system default value (as defined
- * in the configuration) should be used.
- *
- * @return The delay time the system will wait to re-execute failed tasks.
- */
- public long getExecutionRetryDelay() {
- return config.getExecutionRetryDelay();
- }
- /**
* Sets the default parallelism that will be used for the local execution
* environment created by {@link #createLocalEnvironment()}.
*
http://git-wip-us.apache.org/repos/asf/flink/blob/aa1c1454/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
index 49cbd11..d7d20af 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
@@ -99,7 +99,7 @@ public class StreamingJobGraphGenerator {
configureCheckpointing();
configureExecutionRetries();
-
+
configureExecutionRetryDelay();
try {
@@ -416,10 +416,6 @@ public class StreamingJobGraphGenerator {
if(executionRetries == -1) {
streamGraph.getExecutionConfig().setNumberOfExecutionRetries(Integer.MAX_VALUE);
}
- long executionRetryDelay = streamGraph.getExecutionConfig().getExecutionRetryDelay();
- if(executionRetryDelay == -1) {
- streamGraph.getExecutionConfig().setExecutionRetryDelay(100 * 1000);
- }
}
}
@@ -432,13 +428,9 @@ public class StreamingJobGraphGenerator {
jobGraph.setNumberOfExecutionRetries(0);
}
}
-
+
private void configureExecutionRetryDelay() {
long executionRetryDelay = streamGraph.getExecutionConfig().getExecutionRetryDelay();
- if (executionRetryDelay != -1) {
- jobGraph.setExecutionRetryDelay(executionRetryDelay);
- } else {
- jobGraph.setExecutionRetryDelay(100 * 1000);
- }
+ jobGraph.setExecutionRetryDelay(executionRetryDelay);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/aa1c1454/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
index e953696..f632240 100644
--- a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
+++ b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
@@ -233,23 +233,6 @@ class StreamExecutionEnvironment(javaEnv: JavaEnv) {
*/
def getNumberOfExecutionRetries = javaEnv.getNumberOfExecutionRetries
- /**
- * Sets the delay that failed tasks are re-executed. A value of
- * zero effectively disables fault tolerance. A value of "-1"
- * indicates that the system default value (as defined in the configuration)
- * should be used.
- */
- def setExecutionRetryDelay(executionRetryDelay: Long): Unit = {
- javaEnv.setExecutionRetryDelay(executionRetryDelay)
- }
-
- /**
- * Gets the delay time in milliseconds the system will wait to re-execute failed tasks.
- * A value of "-1" indicates that the system default value (as defined
- * in the configuration) should be used.
- */
- def getExecutionRetryDelay = javaEnv.getExecutionRetryDelay
-
// --------------------------------------------------------------------------------------------
// Registry for types and serializers
// --------------------------------------------------------------------------------------------