You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tr...@apache.org on 2019/05/14 09:15:30 UTC
[flink] 01/05: [FLINK-12472][yarn] Support setting
attemptFailuresValidityInterval of jobs on Yarn
This is an automated email from the ASF dual-hosted git repository.
trohrmann pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 6231b18645dd41031299b7071d20ddea32c60457
Author: Victor Wong <ji...@gmail.com>
AuthorDate: Fri May 10 16:01:50 2019 +0800
[FLINK-12472][yarn] Support setting attemptFailuresValidityInterval of jobs on Yarn
This closes #8400.
---
docs/_includes/generated/yarn_config_configuration.html | 5 +++++
.../apache/flink/yarn/AbstractYarnClusterDescriptor.java | 6 ++++--
.../apache/flink/yarn/configuration/YarnConfigOptions.java | 13 +++++++++++++
3 files changed, 22 insertions(+), 2 deletions(-)
diff --git a/docs/_includes/generated/yarn_config_configuration.html b/docs/_includes/generated/yarn_config_configuration.html
index a6b04b6..40dfc09 100644
--- a/docs/_includes/generated/yarn_config_configuration.html
+++ b/docs/_includes/generated/yarn_config_configuration.html
@@ -8,6 +8,11 @@
</thead>
<tbody>
<tr>
+ <td><h5>yarn.application-attempt-failures-validity-interval</h5></td>
+ <td style="word-wrap: break-word;">10000</td>
+ <td>Time window in milliseconds which defines the number of application attempt failures when restarting the AM. Failures which fall outside of this window are not being considered. Set this value to -1 in order to count globally. See <a href="https://hortonworks.com/blog/apache-hadoop-yarn-hdp-2-2-fault-tolerance-features-long-running-services/">here</a> for more information.</td>
+ </tr>
+ <tr>
<td><h5>yarn.application-attempts</h5></td>
<td style="word-wrap: break-word;">(none)</td>
<td>Number of ApplicationMaster restarts. Note that that the entire Flink cluster will restart and the YARN Client will loose the connection. Also, the JobManager address will change and you’ll need to set the JM host:port manually. It is recommended to leave this option at 1.</td>
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java b/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java
index 20b5417..0f24496 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java
@@ -35,7 +35,6 @@ import org.apache.flink.configuration.RestOptions;
import org.apache.flink.configuration.SecurityOptions;
import org.apache.flink.configuration.TaskManagerOptions;
import org.apache.flink.core.plugin.PluginUtils;
-import org.apache.flink.runtime.akka.AkkaUtils;
import org.apache.flink.runtime.clusterframework.BootstrapTools;
import org.apache.flink.runtime.clusterframework.ContaineredTaskManagerParameters;
import org.apache.flink.runtime.entrypoint.ClusterEntrypoint;
@@ -1283,7 +1282,10 @@ public abstract class AbstractYarnClusterDescriptor implements ClusterDescriptor
ApplicationSubmissionContextReflector reflector = ApplicationSubmissionContextReflector.getInstance();
reflector.setKeepContainersAcrossApplicationAttempts(appContext, true);
- reflector.setAttemptFailuresValidityInterval(appContext, AkkaUtils.getTimeout(flinkConfiguration).toMillis());
+
+ reflector.setAttemptFailuresValidityInterval(
+ appContext,
+ flinkConfiguration.getLong(YarnConfigOptions.APPLICATION_ATTEMPT_FAILURE_VALIDITY_INTERVAL));
}
private void setApplicationTags(final ApplicationSubmissionContext appContext) throws InvocationTargetException,
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/configuration/YarnConfigOptions.java b/flink-yarn/src/main/java/org/apache/flink/yarn/configuration/YarnConfigOptions.java
index 40a5929..0f46a57 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/configuration/YarnConfigOptions.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/configuration/YarnConfigOptions.java
@@ -102,6 +102,19 @@ public class YarnConfigOptions {
" to set the JM host:port manually. It is recommended to leave this option at 1.");
/**
+ * The config parameter defining the attemptFailuresValidityInterval of Yarn application.
+ */
+ public static final ConfigOption<Long> APPLICATION_ATTEMPT_FAILURE_VALIDITY_INTERVAL =
+ key("yarn.application-attempt-failures-validity-interval")
+ .defaultValue(10000L)
+ .withDescription(Description.builder()
+ .text("Time window in milliseconds which defines the number of application attempt failures when restarting the AM. " +
+ "Failures which fall outside of this window are not being considered. " +
+ "Set this value to -1 in order to count globally. " +
+ "See %s for more information.", link("https://hortonworks.com/blog/apache-hadoop-yarn-hdp-2-2-fault-tolerance-features-long-running-services/", "here"))
+ .build());
+
+ /**
* The heartbeat interval between the Application Master and the YARN Resource Manager.
*/
public static final ConfigOption<Integer> HEARTBEAT_DELAY_SECONDS =