You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2018/02/13 11:46:09 UTC
[1/2] flink git commit: [FLINK-8475][config][docs] Integrate HA-ZK
options
Repository: flink
Updated Branches:
refs/heads/master 9e139a72b -> 528cca5cf
[FLINK-8475][config][docs] Integrate HA-ZK options
This closes #5462.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/8c8033eb
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/8c8033eb
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/8c8033eb
Branch: refs/heads/master
Commit: 8c8033eb6665d7e352fed4d067930c91b9507d82
Parents: 9e139a7
Author: zentol <ch...@apache.org>
Authored: Tue Jan 23 13:50:32 2018 +0100
Committer: zentol <ch...@apache.org>
Committed: Tue Feb 13 12:33:29 2018 +0100
----------------------------------------------------------------------
...gh_availability_zookeeper_configuration.html | 81 ++++++++++++++++++++
docs/ops/config.md | 22 +-----
.../configuration/HighAvailabilityOptions.java | 45 +++++++----
3 files changed, 114 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/8c8033eb/docs/_includes/generated/high_availability_zookeeper_configuration.html
----------------------------------------------------------------------
diff --git a/docs/_includes/generated/high_availability_zookeeper_configuration.html b/docs/_includes/generated/high_availability_zookeeper_configuration.html
new file mode 100644
index 0000000..8e57deb
--- /dev/null
+++ b/docs/_includes/generated/high_availability_zookeeper_configuration.html
@@ -0,0 +1,81 @@
+<table class="table table-bordered">
+ <thead>
+ <tr>
+ <th class="text-left" style="width: 20%">Key</th>
+ <th class="text-left" style="width: 15%">Default</th>
+ <th class="text-left" style="width: 65%">Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><h5>high-availability.zookeeper.client.acl</h5></td>
+ <td>"open"</td>
+ <td>Defines the ACL (open|creator) to be configured on ZK node. The configuration value can be set to “creator” if the ZooKeeper server configuration has the “authProvider” property mapped to use SASLAuthenticationProvider and the cluster is configured to run in secure mode (Kerberos).</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.client.connection-timeout</h5></td>
+ <td>15000</td>
+ <td>Defines the connection timeout for ZooKeeper in ms.</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.client.max-retry-attempts</h5></td>
+ <td>3</td>
+ <td>Defines the number of connection retries before the client gives up.</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.client.retry-wait</h5></td>
+ <td>5000</td>
+ <td>Defines the pause between consecutive retries in ms.</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.client.session-timeout</h5></td>
+ <td>60000</td>
+ <td>Defines the session timeout for the ZooKeeper session in ms.</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.path.checkpoint-counter</h5></td>
+ <td>"/checkpoint-counter"</td>
+ <td>ZooKeeper root path (ZNode) for checkpoint counters.</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.path.checkpoints</h5></td>
+ <td>"/checkpoints"</td>
+ <td>ZooKeeper root path (ZNode) for completed checkpoints.</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.path.jobgraphs</h5></td>
+ <td>"/jobgraphs"</td>
+ <td>ZooKeeper root path (ZNode) for job graphs</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.path.latch</h5></td>
+ <td>"/leaderlatch"</td>
+ <td> Defines the znode of the leader latch which is used to elect the leader.</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.path.leader</h5></td>
+ <td>"/leader"</td>
+ <td>Defines the znode of the leader which contains the URL to the leader and the current leader session ID.</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.path.mesos-workers</h5></td>
+ <td>"/mesos-workers"</td>
+ <td>ZooKeeper root path (ZNode) for Mesos workers.</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.path.root</h5></td>
+ <td>"/flink"</td>
+ <td>The root path under which Flink stores its entries in ZooKeeper.</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.path.running-registry</h5></td>
+ <td>"/running_job_registry/"</td>
+ <td></td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.quorum</h5></td>
+ <td>(none)</td>
+ <td>The ZooKeeper quorum to use, when running Flink in a high-availability mode with ZooKeeper.</td>
+ </tr>
+ </tbody>
+</table>
http://git-wip-us.apache.org/repos/asf/flink/blob/8c8033eb/docs/ops/config.md
----------------------------------------------------------------------
diff --git a/docs/ops/config.md b/docs/ops/config.md
index ab3ef38..0cbd935 100644
--- a/docs/ops/config.md
+++ b/docs/ops/config.md
@@ -431,27 +431,7 @@ Previously this key was named `recovery.mode` and the default value was `standal
#### ZooKeeper-based HA Mode
-- `high-availability.zookeeper.quorum`: Defines the ZooKeeper quorum URL which is used to connect to the ZooKeeper cluster when the 'zookeeper' HA mode is selected. Previously this key was named `recovery.zookeeper.quorum`.
-
-- `high-availability.zookeeper.path.root`: (Default `/flink`) Defines the root dir under which the ZooKeeper HA mode will create namespace directories. Previously this ket was named `recovery.zookeeper.path.root`.
-
-- `high-availability.zookeeper.path.latch`: (Default `/leaderlatch`) Defines the znode of the leader latch which is used to elect the leader. Previously this key was named `recovery.zookeeper.path.latch`.
-
-- `high-availability.zookeeper.path.leader`: (Default `/leader`) Defines the znode of the leader which contains the URL to the leader and the current leader session ID. Previously this key was named `recovery.zookeeper.path.leader`.
-
-- `high-availability.storageDir`: Defines the directory in the state backend where the JobManager metadata will be stored (ZooKeeper only keeps pointers to it). Required for HA. Previously this key was named `recovery.zookeeper.storageDir` and `high-availability.zookeeper.storageDir`.
-
-- `high-availability.zookeeper.client.session-timeout`: (Default `60000`) Defines the session timeout for the ZooKeeper session in ms. Previously this key was named `recovery.zookeeper.client.session-timeout`
-
-- `high-availability.zookeeper.client.connection-timeout`: (Default `15000`) Defines the connection timeout for ZooKeeper in ms. Previously this key was named `recovery.zookeeper.client.connection-timeout`.
-
-- `high-availability.zookeeper.client.retry-wait`: (Default `5000`) Defines the pause between consecutive retries in ms. Previously this key was named `recovery.zookeeper.client.retry-wait`.
-
-- `high-availability.zookeeper.client.max-retry-attempts`: (Default `3`) Defines the number of connection retries before the client gives up. Previously this key was named `recovery.zookeeper.client.max-retry-attempts`.
-
-- `high-availability.job.delay`: (Default `akka.ask.timeout`) Defines the delay before persisted jobs are recovered in case of a master recovery situation. Previously this key was named `recovery.job.delay`.
-
-- `high-availability.zookeeper.client.acl`: (Default `open`) Defines the ACL (open\|creator) to be configured on ZK node. The configuration value can be set to "creator" if the ZooKeeper server configuration has the "authProvider" property mapped to use SASLAuthenticationProvider and the cluster is configured to run in secure mode (Kerberos). The ACL options are based on https://zookeeper.apache.org/doc/r3.1.2/zookeeperProgrammers.html#sc_BuiltinACLSchemes
+{% include generated/high_availability_zookeeper_configuration.html %}
### ZooKeeper Security
http://git-wip-us.apache.org/repos/asf/flink/blob/8c8033eb/flink-core/src/main/java/org/apache/flink/configuration/HighAvailabilityOptions.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/HighAvailabilityOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/HighAvailabilityOptions.java
index 0039fdd..590271e 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/HighAvailabilityOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/HighAvailabilityOptions.java
@@ -26,6 +26,9 @@ import static org.apache.flink.configuration.ConfigOptions.key;
* The set of configuration options relating to high-availability settings.
*/
@PublicEvolving
+@ConfigGroups(groups = {
+ @ConfigGroup(name = "HighAvailabilityZookeeper", keyPrefix = "high-availability.zookeeper")
+})
public class HighAvailabilityOptions {
// ------------------------------------------------------------------------
@@ -90,7 +93,8 @@ public class HighAvailabilityOptions {
public static final ConfigOption<String> HA_ZOOKEEPER_QUORUM =
key("high-availability.zookeeper.quorum")
.noDefaultValue()
- .withDeprecatedKeys("recovery.zookeeper.quorum");
+ .withDeprecatedKeys("recovery.zookeeper.quorum")
+ .withDescription("The ZooKeeper quorum to use, when running Flink in a high-availability mode with ZooKeeper.");
/**
* The root path under which Flink stores its entries in ZooKeeper.
@@ -98,42 +102,50 @@ public class HighAvailabilityOptions {
public static final ConfigOption<String> HA_ZOOKEEPER_ROOT =
key("high-availability.zookeeper.path.root")
.defaultValue("/flink")
- .withDeprecatedKeys("recovery.zookeeper.path.root");
+ .withDeprecatedKeys("recovery.zookeeper.path.root")
+ .withDescription("The root path under which Flink stores its entries in ZooKeeper.");
public static final ConfigOption<String> HA_ZOOKEEPER_LATCH_PATH =
key("high-availability.zookeeper.path.latch")
.defaultValue("/leaderlatch")
- .withDeprecatedKeys("recovery.zookeeper.path.latch");
+ .withDeprecatedKeys("recovery.zookeeper.path.latch")
+ .withDescription(" Defines the znode of the leader latch which is used to elect the leader.");
/** ZooKeeper root path (ZNode) for job graphs. */
public static final ConfigOption<String> HA_ZOOKEEPER_JOBGRAPHS_PATH =
key("high-availability.zookeeper.path.jobgraphs")
.defaultValue("/jobgraphs")
- .withDeprecatedKeys("recovery.zookeeper.path.jobgraphs");
+ .withDeprecatedKeys("recovery.zookeeper.path.jobgraphs")
+ .withDescription("ZooKeeper root path (ZNode) for job graphs");
public static final ConfigOption<String> HA_ZOOKEEPER_LEADER_PATH =
key("high-availability.zookeeper.path.leader")
.defaultValue("/leader")
- .withDeprecatedKeys("recovery.zookeeper.path.leader");
+ .withDeprecatedKeys("recovery.zookeeper.path.leader")
+ .withDescription("Defines the znode of the leader which contains the URL to the leader and the current" +
+ " leader session ID.");
/** ZooKeeper root path (ZNode) for completed checkpoints. */
public static final ConfigOption<String> HA_ZOOKEEPER_CHECKPOINTS_PATH =
key("high-availability.zookeeper.path.checkpoints")
.defaultValue("/checkpoints")
- .withDeprecatedKeys("recovery.zookeeper.path.checkpoints");
+ .withDeprecatedKeys("recovery.zookeeper.path.checkpoints")
+ .withDescription("ZooKeeper root path (ZNode) for completed checkpoints.");
/** ZooKeeper root path (ZNode) for checkpoint counters. */
public static final ConfigOption<String> HA_ZOOKEEPER_CHECKPOINT_COUNTER_PATH =
key("high-availability.zookeeper.path.checkpoint-counter")
.defaultValue("/checkpoint-counter")
- .withDeprecatedKeys("recovery.zookeeper.path.checkpoint-counter");
+ .withDeprecatedKeys("recovery.zookeeper.path.checkpoint-counter")
+ .withDescription("ZooKeeper root path (ZNode) for checkpoint counters.");
/** ZooKeeper root path (ZNode) for Mesos workers. */
@PublicEvolving
public static final ConfigOption<String> HA_ZOOKEEPER_MESOS_WORKERS_PATH =
key("high-availability.zookeeper.path.mesos-workers")
.defaultValue("/mesos-workers")
- .withDeprecatedKeys("recovery.zookeeper.path.mesos-workers");
+ .withDeprecatedKeys("recovery.zookeeper.path.mesos-workers")
+ .withDescription("ZooKeeper root path (ZNode) for Mesos workers.");
// ------------------------------------------------------------------------
// ZooKeeper Client Settings
@@ -142,22 +154,26 @@ public class HighAvailabilityOptions {
public static final ConfigOption<Integer> ZOOKEEPER_SESSION_TIMEOUT =
key("high-availability.zookeeper.client.session-timeout")
.defaultValue(60000)
- .withDeprecatedKeys("recovery.zookeeper.client.session-timeout");
+ .withDeprecatedKeys("recovery.zookeeper.client.session-timeout")
+ .withDescription("Defines the session timeout for the ZooKeeper session in ms.");
public static final ConfigOption<Integer> ZOOKEEPER_CONNECTION_TIMEOUT =
key("high-availability.zookeeper.client.connection-timeout")
.defaultValue(15000)
- .withDeprecatedKeys("recovery.zookeeper.client.connection-timeout");
+ .withDeprecatedKeys("recovery.zookeeper.client.connection-timeout")
+ .withDescription("Defines the connection timeout for ZooKeeper in ms.");
public static final ConfigOption<Integer> ZOOKEEPER_RETRY_WAIT =
key("high-availability.zookeeper.client.retry-wait")
.defaultValue(5000)
- .withDeprecatedKeys("recovery.zookeeper.client.retry-wait");
+ .withDeprecatedKeys("recovery.zookeeper.client.retry-wait")
+ .withDescription("Defines the pause between consecutive retries in ms.");
public static final ConfigOption<Integer> ZOOKEEPER_MAX_RETRY_ATTEMPTS =
key("high-availability.zookeeper.client.max-retry-attempts")
.defaultValue(3)
- .withDeprecatedKeys("recovery.zookeeper.client.max-retry-attempts");
+ .withDeprecatedKeys("recovery.zookeeper.client.max-retry-attempts")
+ .withDescription("Defines the number of connection retries before the client gives up.");
public static final ConfigOption<String> ZOOKEEPER_RUNNING_JOB_REGISTRY_PATH =
key("high-availability.zookeeper.path.running-registry")
@@ -165,7 +181,10 @@ public class HighAvailabilityOptions {
public static final ConfigOption<String> ZOOKEEPER_CLIENT_ACL =
key("high-availability.zookeeper.client.acl")
- .defaultValue("open");
+ .defaultValue("open")
+ .withDescription("Defines the ACL (open|creator) to be configured on ZK node. The configuration value can be" +
+ " set to “creator” if the ZooKeeper server configuration has the “authProvider” property mapped to use" +
+ " SASLAuthenticationProvider and the cluster is configured to run in secure mode (Kerberos).");
// ------------------------------------------------------------------------
[2/2] flink git commit: [FLINK-8475][config][docs] Integrate YARN
options
Posted by ch...@apache.org.
[FLINK-8475][config][docs] Integrate YARN options
This closes #5463.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/528cca5c
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/528cca5c
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/528cca5c
Branch: refs/heads/master
Commit: 528cca5cf68ea27eeed3553fa18d85b69a93a8ff
Parents: 8c8033e
Author: zentol <ch...@apache.org>
Authored: Tue Jan 23 14:04:36 2018 +0100
Committer: zentol <ch...@apache.org>
Committed: Tue Feb 13 12:34:26 2018 +0100
----------------------------------------------------------------------
.../generated/yarn_config_configuration.html | 61 ++++++++++++++++++++
docs/ops/config.md | 33 +----------
.../yarn/configuration/YarnConfigOptions.java | 44 ++++++++++----
3 files changed, 96 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/528cca5c/docs/_includes/generated/yarn_config_configuration.html
----------------------------------------------------------------------
diff --git a/docs/_includes/generated/yarn_config_configuration.html b/docs/_includes/generated/yarn_config_configuration.html
new file mode 100644
index 0000000..56fb6fc
--- /dev/null
+++ b/docs/_includes/generated/yarn_config_configuration.html
@@ -0,0 +1,61 @@
+<table class="table table-bordered">
+ <thead>
+ <tr>
+ <th class="text-left" style="width: 20%">Key</th>
+ <th class="text-left" style="width: 15%">Default</th>
+ <th class="text-left" style="width: 65%">Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><h5>yarn.application-attempts</h5></td>
+ <td>(none)</td>
+ <td>Number of ApplicationMaster restarts. Note that that the entire Flink cluster will restart and the YARN Client will loose the connection. Also, the JobManager address will change and you’ll need to set the JM host:port manually. It is recommended to leave this option at 1.</td>
+ </tr>
+ <tr>
+ <td><h5>yarn.application-master.port</h5></td>
+ <td>"0"</td>
+ <td>With this configuration option, users can specify a port, a range of ports or a list of ports for the Application Master (and JobManager) RPC port. By default we recommend using the default value (0) to let the operating system choose an appropriate port. In particular when multiple AMs are running on the same physical host, fixed port assignments prevent the AM from starting. For example when running Flink on YARN on an environment with a restrictive firewall, this option allows specifying a range of allowed ports.</td>
+ </tr>
+ <tr>
+ <td><h5>yarn.appmaster.rpc.address</h5></td>
+ <td>(none)</td>
+ <td>The hostname or address where the application master RPC system is listening.</td>
+ </tr>
+ <tr>
+ <td><h5>yarn.appmaster.rpc.port</h5></td>
+ <td>-1</td>
+ <td>The port where the application master RPC system is listening.</td>
+ </tr>
+ <tr>
+ <td><h5>yarn.containers.vcores</h5></td>
+ <td>-1</td>
+ <td>The number of virtual cores (vcores) per YARN container. By default, the number of vcores is set to the number of slots per TaskManager, if set, or to 1, otherwise.</td>
+ </tr>
+ <tr>
+ <td><h5>yarn.heartbeat-delay</h5></td>
+ <td>5</td>
+ <td>Time between heartbeats with the ResourceManager in seconds.</td>
+ </tr>
+ <tr>
+ <td><h5>yarn.maximum-failed-containers</h5></td>
+ <td>(none)</td>
+ <td>Maximum number of containers the system is going to reallocate in case of a failure.</td>
+ </tr>
+ <tr>
+ <td><h5>yarn.per-job-cluster.include-user-jar</h5></td>
+ <td>"ORDER"</td>
+ <td>Defines whether user-jars are included in the system class path for per-job-clusters as well as their positioning in the path. They can be positioned at the beginning ("FIRST"), at the end ("LAST"), or be positioned based on their name ("ORDER").</td>
+ </tr>
+ <tr>
+ <td><h5>yarn.properties-file.location</h5></td>
+ <td>(none)</td>
+ <td>When a Flink job is submitted to YARN, the JobManager’s host and the number of available processing slots is written into a properties file, so that the Flink client is able to pick those details up. This configuration parameter allows changing the default location of that file (for example for environments sharing a Flink installation between users).</td>
+ </tr>
+ <tr>
+ <td><h5>yarn.tags</h5></td>
+ <td>(none)</td>
+ <td>A comma-separated list of tags to apply to the Flink YARN application.</td>
+ </tr>
+ </tbody>
+</table>
http://git-wip-us.apache.org/repos/asf/flink/blob/528cca5c/docs/ops/config.md
----------------------------------------------------------------------
diff --git a/docs/ops/config.md b/docs/ops/config.md
index 0cbd935..adead19 100644
--- a/docs/ops/config.md
+++ b/docs/ops/config.md
@@ -378,38 +378,7 @@ of the JobManager, because the same ActorSystem is used. Its not possible to use
### YARN
-- `containerized.heap-cutoff-ratio`: (Default 0.25) Percentage of heap space to remove from containers started by YARN. When a user requests a certain amount of memory for each TaskManager container (for example 4 GB), we can not pass this amount as the maximum heap space for the JVM (`-Xmx` argument) because the JVM is also allocating memory outside the heap. YARN is very strict with killing containers which are using more memory than requested. Therefore, we remove this fraction of the memory from the requested heap as a safety margin and add it to the memory used off-heap.
-
-- `containerized.heap-cutoff-min`: (Default 600 MB) Minimum amount of memory to cut off the requested heap size.
-
-- `yarn.maximum-failed-containers` (Default: number of requested containers). Maximum number of containers the system is going to reallocate in case of a failure.
-
-- `yarn.application-attempts` (Default: 1). Number of ApplicationMaster restarts. Note that that the entire Flink cluster will restart and the YARN Client will loose the connection. Also, the JobManager address will change and you'll need to set the JM host:port manually. It is recommended to leave this option at 1.
-
-- `yarn.heartbeat-delay` (Default: 5 seconds). Time between heartbeats with the ResourceManager.
-
-- `yarn.properties-file.location` (Default: temp directory). When a Flink job is submitted to YARN, the JobManager's host and the number of available processing slots is written into a properties file, so that the Flink client is able to pick those details up. This configuration parameter allows changing the default location of that file (for example for environments sharing a Flink installation between users)
-
-- `yarn.containers.vcores` The number of virtual cores (vcores) per YARN container. By default, the number of `vcores` is set to the number of slots per TaskManager, if set, or to 1, otherwise.
-
-- `containerized.master.env.`*ENV_VAR1=value* Configuration values prefixed with `containerized.master.env.` will be passed as environment variables to the ApplicationMaster/JobManager process. For example for passing `LD_LIBRARY_PATH` as an env variable to the ApplicationMaster, set:
-
- `containerized.master.env.LD_LIBRARY_PATH: "/usr/lib/native"`
-
-- `containerized.taskmanager.env.` Similar to the configuration prefix about, this prefix allows setting custom environment variables for the TaskManager processes.
-
-- `yarn.container-start-command-template`: Flink uses the following template when starting on YARN:
-`%java% %jvmmem% %jvmopts% %logging% %class% %args% %redirects%`. This configuration parameter allows users
-to pass custom settings (such as JVM paths, arguments etc.). Note that in most cases, it is sufficient to
-use the `env.java.opts` setting, which is the `%jvmopts%` variable in the String.
-
-- `yarn.application-master.port` (Default: 0, which lets the OS choose an ephemeral port) With this configuration option, users can specify a port, a range of ports or a list of ports for the Application Master (and JobManager) RPC port. By default we recommend using the default value (0) to let the operating system choose an appropriate port. In particular when multiple AMs are running on the same physical host, fixed port assignments prevent the AM from starting.
-
- For example when running Flink on YARN on an environment with a restrictive firewall, this option allows specifying a range of allowed ports.
-
-- `yarn.tags` A comma-separated list of tags to apply to the Flink YARN application.
-
-- `yarn.per-job-cluster.include-user-jar` (Default: ORDER) Control whether and how the user-jar is included in the system class path for per-job clusters. Setting this parameter to `DISABLED` causes the jar to be included in the user class path instead. Setting this parameter to one of `FIRST`, `LAST` or `ORDER` causes the jar to be included in the system class path, with the jar either being placed at the beginning of the class path (`FIRST`), at the end (`LAST`), or based on the lexicographic order (`ORDER`).
+{% include generated/yarn_config_configuration.html %}
### Mesos
http://git-wip-us.apache.org/repos/asf/flink/blob/528cca5c/flink-yarn/src/main/java/org/apache/flink/yarn/configuration/YarnConfigOptions.java
----------------------------------------------------------------------
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/configuration/YarnConfigOptions.java b/flink-yarn/src/main/java/org/apache/flink/yarn/configuration/YarnConfigOptions.java
index 02ab6a0..255a6c7 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/configuration/YarnConfigOptions.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/configuration/YarnConfigOptions.java
@@ -34,14 +34,16 @@ public class YarnConfigOptions {
*/
public static final ConfigOption<String> APP_MASTER_RPC_ADDRESS =
key("yarn.appmaster.rpc.address")
- .noDefaultValue();
+ .noDefaultValue()
+ .withDescription("The hostname or address where the application master RPC system is listening.");
/**
* The port where the application master RPC system is listening.
*/
public static final ConfigOption<Integer> APP_MASTER_RPC_PORT =
key("yarn.appmaster.rpc.port")
- .defaultValue(-1);
+ .defaultValue(-1)
+ .withDescription("The port where the application master RPC system is listening.");
/**
* Defines whether user-jars are included in the system class path for per-job-clusters as well as their positioning
@@ -50,14 +52,20 @@ public class YarnConfigOptions {
*/
public static final ConfigOption<String> CLASSPATH_INCLUDE_USER_JAR =
key("yarn.per-job-cluster.include-user-jar")
- .defaultValue("ORDER");
+ .defaultValue("ORDER")
+ .withDescription("Defines whether user-jars are included in the system class path for per-job-clusters as" +
+ " well as their positioning in the path. They can be positioned at the beginning (\"FIRST\"), at the" +
+ " end (\"LAST\"), or be positioned based on their name (\"ORDER\"). Setting this parameter to" +
+ " \"DISABLED\" causes the jar to be included in the user class path instead.");
/**
* The vcores exposed by YARN.
*/
public static final ConfigOption<Integer> VCORES =
key("yarn.containers.vcores")
- .defaultValue(-1);
+ .defaultValue(-1)
+ .withDescription("The number of virtual cores (vcores) per YARN container. By default, the number of vcores" +
+ " is set to the number of slots per TaskManager, if set, or to 1, otherwise.");
/**
* The maximum number of failed YARN containers before entirely stopping
@@ -68,7 +76,8 @@ public class YarnConfigOptions {
*/
public static final ConfigOption<String> MAX_FAILED_CONTAINERS =
key("yarn.maximum-failed-containers")
- .noDefaultValue();
+ .noDefaultValue()
+ .withDescription("Maximum number of containers the system is going to reallocate in case of a failure.");
/**
* Set the number of retries for failed YARN ApplicationMasters/JobManagers in high
@@ -79,14 +88,18 @@ public class YarnConfigOptions {
*/
public static final ConfigOption<String> APPLICATION_ATTEMPTS =
key("yarn.application-attempts")
- .noDefaultValue();
+ .noDefaultValue()
+ .withDescription("Number of ApplicationMaster restarts. Note that that the entire Flink cluster will restart" +
+ " and the YARN Client will loose the connection. Also, the JobManager address will change and you’ll need" +
+ " to set the JM host:port manually. It is recommended to leave this option at 1.");
/**
* The heartbeat interval between the Application Master and the YARN Resource Manager.
*/
public static final ConfigOption<Integer> HEARTBEAT_DELAY_SECONDS =
key("yarn.heartbeat-delay")
- .defaultValue(5);
+ .defaultValue(5)
+ .withDescription("Time between heartbeats with the ResourceManager in seconds.");
/**
* When a Flink job is submitted to YARN, the JobManager's host and the number of available
@@ -97,7 +110,11 @@ public class YarnConfigOptions {
*/
public static final ConfigOption<String> PROPERTIES_FILE_LOCATION =
key("yarn.properties-file.location")
- .noDefaultValue();
+ .noDefaultValue()
+ .withDescription("When a Flink job is submitted to YARN, the JobManager’s host and the number of available" +
+ " processing slots is written into a properties file, so that the Flink client is able to pick those" +
+ " details up. This configuration parameter allows changing the default location of that file" +
+ " (for example for environments sharing a Flink installation between users).");
/**
* The config parameter defining the Akka actor system port for the ApplicationMaster and
@@ -109,14 +126,21 @@ public class YarnConfigOptions {
*/
public static final ConfigOption<String> APPLICATION_MASTER_PORT =
key("yarn.application-master.port")
- .defaultValue("0");
+ .defaultValue("0")
+ .withDescription("With this configuration option, users can specify a port, a range of ports or a list of ports" +
+ " for the Application Master (and JobManager) RPC port. By default we recommend using the default value (0)" +
+ " to let the operating system choose an appropriate port. In particular when multiple AMs are running on" +
+ " the same physical host, fixed port assignments prevent the AM from starting. For example when running" +
+ " Flink on YARN on an environment with a restrictive firewall, this option allows specifying a range of" +
+ " allowed ports.");
/**
* A comma-separated list of strings to use as YARN application tags.
*/
public static final ConfigOption<String> APPLICATION_TAGS =
key("yarn.tags")
- .defaultValue("");
+ .defaultValue("")
+ .withDescription("A comma-separated list of tags to apply to the Flink YARN application.");
// ------------------------------------------------------------------------