You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2020/02/03 14:16:11 UTC
[flink] 06/10: [FLINK-15698][docs] Restructure the Configuration
docs
This is an automated email from the ASF dual-hosted git repository.
sewen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit afb11a9a4f2d47f961b62d335e7773d3b11d28f2
Author: Stephan Ewen <se...@apache.org>
AuthorDate: Wed Jan 29 20:08:58 2020 +0100
[FLINK-15698][docs] Restructure the Configuration docs
- Grouping options by semantics and functionality, rather than by defined class.
- Splitting between "normal/common options" and options that should only be necessary
for trouble-shooting.
---
...figuration.html => all_jobmanager_section.html} | 14 +-
...n.html => all_taskmanager_network_section.html} | 54 ++-
...iguration.html => all_taskmanager_section.html} | 20 +-
....html => common_high_availability_section.html} | 6 -
.../common_high_availability_zk_section.html | 24 ++
.../generated/common_host_port_section.html | 66 ++++
.../_includes/generated/common_memory_section.html | 102 ++++++
.../generated/common_miscellaneous_section.html | 24 ++
docs/_includes/generated/common_section.html | 84 -----
.../generated/common_state_backends_section.html | 54 +++
docs/_includes/generated/core_configuration.html | 18 +
.../generated/deprecated_file_sinks_section.html | 24 ++
...tion.html => expert_class_loading_section.html} | 12 -
.../generated/expert_fault_tolerance_section.html | 60 ++++
.../expert_high_availability_section.html | 18 +
.../expert_high_availability_zk_section.html | 84 +++++
docs/_includes/generated/expert_rest_section.html | 66 ++++
.../generated/expert_rocksdb_section.html | 36 ++
.../generated/expert_scheduling_section.html | 30 ++
.../generated/expert_security_ssl_section.html | 42 +++
.../generated/expert_state_backends_section.html | 30 ++
.../generated/high_availability_configuration.html | 84 +++++
.../generated/job_manager_configuration.html | 2 +-
.../netty_shuffle_environment_configuration.html | 42 +++
.../generated/resource_manager_configuration.html | 6 -
.../generated/security_auth_kerberos_section.html | 36 ++
.../generated/security_auth_zk_section.html | 30 ++
.../generated/security_configuration.html | 70 ++--
...onfiguration.html => security_ssl_section.html} | 60 ----
.../generated/state_backend_rocksdb_section.html | 42 +++
.../generated/task_manager_configuration.html | 6 -
docs/_includes/generated/web_configuration.html | 12 -
docs/ops/config.md | 394 +++++++++++++++-----
docs/ops/config.zh.md | 396 +++++++++++++++------
.../flink/annotation/docs/Documentation.java | 44 ++-
.../flink/configuration/CheckpointingOptions.java | 19 +-
.../apache/flink/configuration/ClusterOptions.java | 7 +
.../apache/flink/configuration/CoreOptions.java | 13 +-
.../configuration/HeartbeatManagerOptions.java | 3 +
.../configuration/HighAvailabilityOptions.java | 30 +-
.../flink/configuration/JobManagerOptions.java | 15 +-
.../NettyShuffleEnvironmentOptions.java | 19 +-
.../configuration/ResourceManagerOptions.java | 3 +-
.../apache/flink/configuration/RestOptions.java | 14 +
.../flink/configuration/SecurityOptions.java | 49 ++-
.../flink/configuration/TaskManagerOptions.java | 40 ++-
.../org/apache/flink/configuration/WebOptions.java | 2 +
.../contrib/streaming/state/RocksDBOptions.java | 10 +
48 files changed, 1812 insertions(+), 504 deletions(-)
diff --git a/docs/_includes/generated/job_manager_configuration.html b/docs/_includes/generated/all_jobmanager_section.html
similarity index 87%
copy from docs/_includes/generated/job_manager_configuration.html
copy to docs/_includes/generated/all_jobmanager_section.html
index 76671e5..ab01a85 100644
--- a/docs/_includes/generated/job_manager_configuration.html
+++ b/docs/_includes/generated/all_jobmanager_section.html
@@ -22,7 +22,7 @@
</tr>
<tr>
<td><h5>jobmanager.execution.failover-strategy</h5></td>
- <td style="word-wrap: break-word;">"full"</td>
+ <td style="word-wrap: break-word;">region</td>
<td>String</td>
<td>This option specifies how the job computation recovers from task failures. Accepted values are:<ul><li>'full': Restarts all tasks to recover the job.</li><li>'region': Restarts all tasks that could be affected by the task failure. More details can be found <a href="../dev/task_failure_recovery.html#restart-pipelined-region-failover-strategy">here</a>.</li></ul></td>
</tr>
@@ -62,17 +62,5 @@
<td>Integer</td>
<td>The max number of completed jobs that can be kept in the job store.</td>
</tr>
- <tr>
- <td><h5>slot.idle.timeout</h5></td>
- <td style="word-wrap: break-word;">50000</td>
- <td>Long</td>
- <td>The timeout in milliseconds for a idle slot in Slot Pool.</td>
- </tr>
- <tr>
- <td><h5>slot.request.timeout</h5></td>
- <td style="word-wrap: break-word;">300000</td>
- <td>Long</td>
- <td>The timeout in milliseconds for requesting a slot from Slot Pool.</td>
- </tr>
</tbody>
</table>
diff --git a/docs/_includes/generated/netty_shuffle_environment_configuration.html b/docs/_includes/generated/all_taskmanager_network_section.html
similarity index 67%
copy from docs/_includes/generated/netty_shuffle_environment_configuration.html
copy to docs/_includes/generated/all_taskmanager_network_section.html
index 8a5b3ba..04a8566 100644
--- a/docs/_includes/generated/netty_shuffle_environment_configuration.html
+++ b/docs/_includes/generated/all_taskmanager_network_section.html
@@ -9,18 +9,6 @@
</thead>
<tbody>
<tr>
- <td><h5>taskmanager.data.port</h5></td>
- <td style="word-wrap: break-word;">0</td>
- <td>Integer</td>
- <td>The task manager’s port used for data exchange operations.</td>
- </tr>
- <tr>
- <td><h5>taskmanager.data.ssl.enabled</h5></td>
- <td style="word-wrap: break-word;">true</td>
- <td>Boolean</td>
- <td>Enable SSL support for the taskmanager data transport. This is applicable only when the global flag for internal SSL (security.ssl.internal.enabled) is set to true</td>
- </tr>
- <tr>
<td><h5>taskmanager.network.blocking-shuffle.compression.enabled</h5></td>
<td style="word-wrap: break-word;">false</td>
<td>Boolean</td>
@@ -51,6 +39,48 @@
<td>Number of extra network buffers to use for each outgoing/incoming gate (result partition/input gate). In credit-based flow control mode, this indicates how many floating credits are shared among all the input channels. The floating buffers are distributed based on backlog (real-time output buffers in the subpartition) feedback, and can help relieve back-pressure caused by unbalanced data distribution among the subpartitions. This value should be increased in case of highe [...]
</tr>
<tr>
+ <td><h5>taskmanager.network.netty.client.connectTimeoutSec</h5></td>
+ <td style="word-wrap: break-word;">120</td>
+ <td>Integer</td>
+ <td>The Netty client connection timeout.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.network.netty.client.numThreads</h5></td>
+ <td style="word-wrap: break-word;">-1</td>
+ <td>Integer</td>
+ <td>The number of Netty client threads.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.network.netty.num-arenas</h5></td>
+ <td style="word-wrap: break-word;">-1</td>
+ <td>Integer</td>
+ <td>The number of Netty arenas.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.network.netty.sendReceiveBufferSize</h5></td>
+ <td style="word-wrap: break-word;">0</td>
+ <td>Integer</td>
+ <td>The Netty send and receive buffer size. This defaults to the system buffer size (cat /proc/sys/net/ipv4/tcp_[rw]mem) and is 4 MiB in modern Linux.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.network.netty.server.backlog</h5></td>
+ <td style="word-wrap: break-word;">0</td>
+ <td>Integer</td>
+ <td>The netty server connection backlog.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.network.netty.server.numThreads</h5></td>
+ <td style="word-wrap: break-word;">-1</td>
+ <td>Integer</td>
+ <td>The number of Netty server threads.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.network.netty.transport</h5></td>
+ <td style="word-wrap: break-word;">"auto"</td>
+ <td>String</td>
+ <td>The Netty transport type, either "nio" or "epoll". The "auto" means selecting the property mode automatically based on the platform. Note that the "epoll" mode can get better performance, less GC and have more advanced features which are only available on modern Linux.</td>
+ </tr>
+ <tr>
<td><h5>taskmanager.network.request-backoff.initial</h5></td>
<td style="word-wrap: break-word;">100</td>
<td>Integer</td>
diff --git a/docs/_includes/generated/task_manager_configuration.html b/docs/_includes/generated/all_taskmanager_section.html
similarity index 86%
copy from docs/_includes/generated/task_manager_configuration.html
copy to docs/_includes/generated/all_taskmanager_section.html
index fed0f21..692c38b 100644
--- a/docs/_includes/generated/task_manager_configuration.html
+++ b/docs/_includes/generated/all_taskmanager_section.html
@@ -27,10 +27,16 @@
<td>Time we wait for the timers in milliseconds to finish all pending timer threads when the stream task is cancelled.</td>
</tr>
<tr>
- <td><h5>task.checkpoint.alignment.max-size</h5></td>
- <td style="word-wrap: break-word;">-1</td>
- <td>Long</td>
- <td>The maximum number of bytes that a checkpoint alignment may buffer. If the checkpoint alignment buffers more than the configured amount of data, the checkpoint is aborted (skipped). A value of -1 indicates that there is no limit.</td>
+ <td><h5>taskmanager.data.port</h5></td>
+ <td style="word-wrap: break-word;">0</td>
+ <td>Integer</td>
+ <td>The task manager’s port used for data exchange operations.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.data.ssl.enabled</h5></td>
+ <td style="word-wrap: break-word;">true</td>
+ <td>Boolean</td>
+ <td>Enable SSL support for the taskmanager data transport. This is applicable only when the global flag for internal SSL (security.ssl.internal.enabled) is set to true</td>
</tr>
<tr>
<td><h5>taskmanager.debug.memory.log</h5></td>
@@ -57,6 +63,12 @@
<td>Whether to kill the TaskManager when the task thread throws an OutOfMemoryError.</td>
</tr>
<tr>
+ <td><h5>taskmanager.memory.segment-size</h5></td>
+ <td style="word-wrap: break-word;">32 kb</td>
+ <td>MemorySize</td>
+ <td>Size of memory buffers used by the network stack and the memory manager.</td>
+ </tr>
+ <tr>
<td><h5>taskmanager.network.bind-policy</h5></td>
<td style="word-wrap: break-word;">"ip"</td>
<td>String</td>
diff --git a/docs/_includes/generated/high_availability_configuration.html b/docs/_includes/generated/common_high_availability_section.html
similarity index 83%
copy from docs/_includes/generated/high_availability_configuration.html
copy to docs/_includes/generated/common_high_availability_section.html
index bf30e91..4c06ca7 100644
--- a/docs/_includes/generated/high_availability_configuration.html
+++ b/docs/_includes/generated/common_high_availability_section.html
@@ -21,12 +21,6 @@
<td>The ID of the Flink cluster, used to separate multiple Flink clusters from each other. Needs to be set for standalone clusters but is automatically inferred in YARN and Mesos.</td>
</tr>
<tr>
- <td><h5>high-availability.jobmanager.port</h5></td>
- <td style="word-wrap: break-word;">"0"</td>
- <td>String</td>
- <td>Optional port (range) used by the job manager in high-availability mode.</td>
- </tr>
- <tr>
<td><h5>high-availability.storageDir</h5></td>
<td style="word-wrap: break-word;">(none)</td>
<td>String</td>
diff --git a/docs/_includes/generated/common_high_availability_zk_section.html b/docs/_includes/generated/common_high_availability_zk_section.html
new file mode 100644
index 0000000..18175fb
--- /dev/null
+++ b/docs/_includes/generated/common_high_availability_zk_section.html
@@ -0,0 +1,24 @@
+<table class="table table-bordered">
+ <thead>
+ <tr>
+ <th class="text-left" style="width: 20%">Key</th>
+ <th class="text-left" style="width: 15%">Default</th>
+ <th class="text-left" style="width: 10%">Type</th>
+ <th class="text-left" style="width: 55%">Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><h5>high-availability.zookeeper.path.root</h5></td>
+ <td style="word-wrap: break-word;">"/flink"</td>
+ <td>String</td>
+ <td>The root path under which Flink stores its entries in ZooKeeper.</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.quorum</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>The ZooKeeper quorum to use, when running Flink in a high-availability mode with ZooKeeper.</td>
+ </tr>
+ </tbody>
+</table>
diff --git a/docs/_includes/generated/common_host_port_section.html b/docs/_includes/generated/common_host_port_section.html
new file mode 100644
index 0000000..ce9b92f
--- /dev/null
+++ b/docs/_includes/generated/common_host_port_section.html
@@ -0,0 +1,66 @@
+<table class="table table-bordered">
+ <thead>
+ <tr>
+ <th class="text-left" style="width: 20%">Key</th>
+ <th class="text-left" style="width: 15%">Default</th>
+ <th class="text-left" style="width: 10%">Type</th>
+ <th class="text-left" style="width: 55%">Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><h5>jobmanager.rpc.address</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>The config parameter defining the network address to connect to for communication with the job manager. This value is only interpreted in setups where a single JobManager with static name or address exists (simple standalone setups, or container setups with dynamic service name resolution). It is not used in many high-availability setups, when a leader-election service (like ZooKeeper) is used to elect and discover the JobManager leader from potentially multiple standby J [...]
+ </tr>
+ <tr>
+ <td><h5>jobmanager.rpc.port</h5></td>
+ <td style="word-wrap: break-word;">6123</td>
+ <td>Integer</td>
+ <td>The config parameter defining the network port to connect to for communication with the job manager. Like jobmanager.rpc.address, this value is only interpreted in setups where a single JobManager with static name/address and port exists (simple standalone setups, or container setups with dynamic service name resolution). This config option is not used in many high-availability setups, when a leader-election service (like ZooKeeper) is used to elect and discover the JobMa [...]
+ </tr>
+ <tr>
+ <td><h5>rest.address</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>The address that should be used by clients to connect to the server.</td>
+ </tr>
+ <tr>
+ <td><h5>rest.bind-address</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>The address that the server binds itself.</td>
+ </tr>
+ <tr>
+ <td><h5>rest.bind-port</h5></td>
+ <td style="word-wrap: break-word;">"8081"</td>
+ <td>String</td>
+ <td>The port that the server binds itself. Accepts a list of ports (“50100,50101”), ranges (“50100-50200”) or a combination of both. It is recommended to set a range of ports to avoid collisions when multiple Rest servers are running on the same machine.</td>
+ </tr>
+ <tr>
+ <td><h5>rest.port</h5></td>
+ <td style="word-wrap: break-word;">8081</td>
+ <td>Integer</td>
+ <td>The port that the client connects to. If rest.bind-port has not been specified, then the REST server will bind to this port.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.data.port</h5></td>
+ <td style="word-wrap: break-word;">0</td>
+ <td>Integer</td>
+ <td>The task manager’s port used for data exchange operations.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.host</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>The address of the network interface that the TaskManager binds to. This option can be used to define explicitly a binding address. Because different TaskManagers need different values for this option, usually it is specified in an additional non-shared TaskManager-specific config file.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.rpc.port</h5></td>
+ <td style="word-wrap: break-word;">"0"</td>
+ <td>String</td>
+ <td>The task manager’s IPC port. Accepts a list of ports (“50100,50101”), ranges (“50100-50200”) or a combination of both. It is recommended to set a range of ports to avoid collisions when multiple TaskManagers are running on the same machine.</td>
+ </tr>
+ </tbody>
+</table>
diff --git a/docs/_includes/generated/common_memory_section.html b/docs/_includes/generated/common_memory_section.html
new file mode 100644
index 0000000..713edff
--- /dev/null
+++ b/docs/_includes/generated/common_memory_section.html
@@ -0,0 +1,102 @@
+<table class="table table-bordered">
+ <thead>
+ <tr>
+ <th class="text-left" style="width: 20%">Key</th>
+ <th class="text-left" style="width: 15%">Default</th>
+ <th class="text-left" style="width: 10%">Type</th>
+ <th class="text-left" style="width: 55%">Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><h5>taskmanager.memory.flink.size</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>MemorySize</td>
+ <td>Total Flink Memory size for the TaskExecutors. This includes all the memory that a TaskExecutor consumes, except for JVM Metaspace and JVM Overhead. It consists of Framework Heap Memory, Task Heap Memory, Task Off-Heap Memory, Managed Memory, and Network Memory. See also 'taskmanager.memory.process.size' for total process memory size configuration.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.memory.framework.heap.size</h5></td>
+ <td style="word-wrap: break-word;">128 mb</td>
+ <td>MemorySize</td>
+ <td>Framework Heap Memory size for TaskExecutors. This is the size of JVM heap memory reserved for TaskExecutor framework, which will not be allocated to task slots.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.memory.framework.off-heap.size</h5></td>
+ <td style="word-wrap: break-word;">128 mb</td>
+ <td>MemorySize</td>
+ <td>Framework Off-Heap Memory size for TaskExecutors. This is the size of off-heap memory (JVM direct memory and native memory) reserved for TaskExecutor framework, which will not be allocated to task slots. The configured value will be fully counted when Flink calculates the JVM max direct memory size parameter.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.memory.jvm-metaspace.size</h5></td>
+ <td style="word-wrap: break-word;">96 mb</td>
+ <td>MemorySize</td>
+ <td>JVM Metaspace Size for the TaskExecutors.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.memory.jvm-overhead.fraction</h5></td>
+ <td style="word-wrap: break-word;">0.1</td>
+ <td>Float</td>
+ <td>Fraction of Total Process Memory to be reserved for JVM Overhead. This is off-heap memory reserved for JVM overhead, such as thread stack space, compile cache, etc. This includes native memory but not direct memory, and will not be counted when Flink calculates JVM max direct memory size parameter. The size of JVM Overhead is derived to make up the configured fraction of the Total Process Memory. If the derived size is less/greater than the configured min/max size, the mi [...]
+ </tr>
+ <tr>
+ <td><h5>taskmanager.memory.jvm-overhead.max</h5></td>
+ <td style="word-wrap: break-word;">1 gb</td>
+ <td>MemorySize</td>
+ <td>Max JVM Overhead size for the TaskExecutors. This is off-heap memory reserved for JVM overhead, such as thread stack space, compile cache, etc. This includes native memory but not direct memory, and will not be counted when Flink calculates JVM max direct memory size parameter. The size of JVM Overhead is derived to make up the configured fraction of the Total Process Memory. If the derived size is less/greater than the configured min/max size, the min/max size will be us [...]
+ </tr>
+ <tr>
+ <td><h5>taskmanager.memory.jvm-overhead.min</h5></td>
+ <td style="word-wrap: break-word;">192 mb</td>
+ <td>MemorySize</td>
+ <td>Min JVM Overhead size for the TaskExecutors. This is off-heap memory reserved for JVM overhead, such as thread stack space, compile cache, etc. This includes native memory but not direct memory, and will not be counted when Flink calculates JVM max direct memory size parameter. The size of JVM Overhead is derived to make up the configured fraction of the Total Process Memory. If the derived size is less/greater than the configured min/max size, the min/max size will be us [...]
+ </tr>
+ <tr>
+ <td><h5>taskmanager.memory.managed.fraction</h5></td>
+ <td style="word-wrap: break-word;">0.4</td>
+ <td>Float</td>
+ <td>Fraction of Total Flink Memory to be used as Managed Memory, if Managed Memory size is not explicitly specified.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.memory.managed.size</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>MemorySize</td>
+ <td>Managed Memory size for TaskExecutors. This is the size of off-heap memory managed by the memory manager, reserved for sorting, hash tables, caching of intermediate results and RocksDB state backend. Memory consumers can either allocate memory from the memory manager in the form of MemorySegments, or reserve bytes from the memory manager and keep their memory usage within that boundary. If unspecified, it will be derived to make up the configured fraction of the Total Fli [...]
+ </tr>
+ <tr>
+ <td><h5>taskmanager.memory.network.fraction</h5></td>
+ <td style="word-wrap: break-word;">0.1</td>
+ <td>Float</td>
+ <td>Fraction of Total Flink Memory to be used as Network Memory. Network Memory is off-heap memory reserved for ShuffleEnvironment (e.g., network buffers). Network Memory size is derived to make up the configured fraction of the Total Flink Memory. If the derived size is less/greater than the configured min/max size, the min/max size will be used. The exact size of Network Memory can be explicitly specified by setting the min/max size to the same value.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.memory.network.max</h5></td>
+ <td style="word-wrap: break-word;">1 gb</td>
+ <td>MemorySize</td>
+ <td>Max Network Memory size for TaskExecutors. Network Memory is off-heap memory reserved for ShuffleEnvironment (e.g., network buffers). Network Memory size is derived to make up the configured fraction of the Total Flink Memory. If the derived size is less/greater than the configured min/max size, the min/max size will be used. The exact size of Network Memory can be explicitly specified by setting the min/max to the same value.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.memory.network.min</h5></td>
+ <td style="word-wrap: break-word;">64 mb</td>
+ <td>MemorySize</td>
+ <td>Min Network Memory size for TaskExecutors. Network Memory is off-heap memory reserved for ShuffleEnvironment (e.g., network buffers). Network Memory size is derived to make up the configured fraction of the Total Flink Memory. If the derived size is less/greater than the configured min/max size, the min/max size will be used. The exact size of Network Memory can be explicitly specified by setting the min/max to the same value.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.memory.process.size</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>MemorySize</td>
+ <td>Total Process Memory size for the TaskExecutors. This includes all the memory that a TaskExecutor consumes, consisting of Total Flink Memory, JVM Metaspace, and JVM Overhead. On containerized setups, this should be set to the container memory. See also 'taskmanager.memory.flink.size' for total Flink memory size configuration.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.memory.task.heap.size</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>MemorySize</td>
+ <td>Task Heap Memory size for TaskExecutors. This is the size of JVM heap memory reserved for tasks. If not specified, it will be derived as Total Flink Memory minus Framework Heap Memory, Task Off-Heap Memory, Managed Memory and Network Memory.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.memory.task.off-heap.size</h5></td>
+ <td style="word-wrap: break-word;">0 bytes</td>
+ <td>MemorySize</td>
+ <td>Task Off-Heap Memory size for TaskExecutors. This is the size of off heap memory (JVM direct memory and native memory) reserved for tasks. The configured value will be fully counted when Flink calculates the JVM max direct memory size parameter.</td>
+ </tr>
+ </tbody>
+</table>
diff --git a/docs/_includes/generated/common_miscellaneous_section.html b/docs/_includes/generated/common_miscellaneous_section.html
new file mode 100644
index 0000000..0a4e15a
--- /dev/null
+++ b/docs/_includes/generated/common_miscellaneous_section.html
@@ -0,0 +1,24 @@
+<table class="table table-bordered">
+ <thead>
+ <tr>
+ <th class="text-left" style="width: 20%">Key</th>
+ <th class="text-left" style="width: 15%">Default</th>
+ <th class="text-left" style="width: 10%">Type</th>
+ <th class="text-left" style="width: 55%">Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><h5>fs.default-scheme</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>The default filesystem scheme, used for paths that do not declare a scheme explicitly. May contain an authority, e.g. host:port in case of an HDFS NameNode.</td>
+ </tr>
+ <tr>
+ <td><h5>io.tmp.dirs</h5></td>
+ <td style="word-wrap: break-word;">'LOCAL_DIRS' on Yarn. '_FLINK_TMP_DIR' on Mesos. System.getProperty("java.io.tmpdir") in standalone.</td>
+ <td>String</td>
+ <td>Directories for temporary files, separated by",", "|", or the system's java.io.File.pathSeparator.</td>
+ </tr>
+ </tbody>
+</table>
diff --git a/docs/_includes/generated/common_section.html b/docs/_includes/generated/common_section.html
deleted file mode 100644
index ac1d7d5..0000000
--- a/docs/_includes/generated/common_section.html
+++ /dev/null
@@ -1,84 +0,0 @@
-<table class="table table-bordered">
- <thead>
- <tr>
- <th class="text-left" style="width: 20%">Key</th>
- <th class="text-left" style="width: 15%">Default</th>
- <th class="text-left" style="width: 10%">Type</th>
- <th class="text-left" style="width: 55%">Description</th>
- </tr>
- </thead>
- <tbody>
- <tr>
- <td><h5>jobmanager.heap.size</h5></td>
- <td style="word-wrap: break-word;">"1024m"</td>
- <td>String</td>
- <td>JVM heap size for the JobManager.</td>
- </tr>
- <tr>
- <td><h5>taskmanager.memory.flink.size</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>MemorySize</td>
- <td>Total Flink Memory size for the TaskExecutors. This includes all the memory that a TaskExecutor consumes, except for JVM Metaspace and JVM Overhead. It consists of Framework Heap Memory, Task Heap Memory, Task Off-Heap Memory, Managed Memory, and Network Memory. See also 'taskmanager.memory.process.size' for total process memory size configuration.</td>
- </tr>
- <tr>
- <td><h5>taskmanager.memory.process.size</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>MemorySize</td>
- <td>Total Process Memory size for the TaskExecutors. This includes all the memory that a TaskExecutor consumes, consisting of Total Flink Memory, JVM Metaspace, and JVM Overhead. On containerized setups, this should be set to the container memory. See also 'taskmanager.memory.flink.size' for total Flink memory size configuration.</td>
- </tr>
- <tr>
- <td><h5>parallelism.default</h5></td>
- <td style="word-wrap: break-word;">1</td>
- <td>Integer</td>
- <td>Default parallelism for jobs.</td>
- </tr>
- <tr>
- <td><h5>taskmanager.numberOfTaskSlots</h5></td>
- <td style="word-wrap: break-word;">1</td>
- <td>Integer</td>
- <td>The number of parallel operator or user function instances that a single TaskManager can run. If this value is larger than 1, a single TaskManager takes multiple instances of a function or operator. That way, the TaskManager can utilize multiple CPU cores, but at the same time, the available memory is divided between the different operator or function instances. This value is typically proportional to the number of physical CPU cores that the TaskManager's machine has (e. [...]
- </tr>
- <tr>
- <td><h5>state.backend</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>The state backend to be used to store and checkpoint state.</td>
- </tr>
- <tr>
- <td><h5>state.checkpoints.dir</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>The default directory used for storing the data files and meta data of checkpoints in a Flink supported filesystem. The storage path must be accessible from all participating processes/nodes(i.e. all TaskManagers and JobManagers).</td>
- </tr>
- <tr>
- <td><h5>state.savepoints.dir</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>The default directory for savepoints. Used by the state backends that write savepoints to file systems (MemoryStateBackend, FsStateBackend, RocksDBStateBackend).</td>
- </tr>
- <tr>
- <td><h5>high-availability</h5></td>
- <td style="word-wrap: break-word;">"NONE"</td>
- <td>String</td>
- <td>Defines high-availability mode used for the cluster execution. To enable high-availability, set this mode to "ZOOKEEPER" or specify FQN of factory class.</td>
- </tr>
- <tr>
- <td><h5>high-availability.storageDir</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>File system path (URI) where Flink persists metadata in high-availability setups.</td>
- </tr>
- <tr>
- <td><h5>security.ssl.internal.enabled</h5></td>
- <td style="word-wrap: break-word;">false</td>
- <td>Boolean</td>
- <td>Turns on SSL for internal network communication. Optionally, specific components may override this through their own settings (rpc, data transport, REST, etc).</td>
- </tr>
- <tr>
- <td><h5>security.ssl.rest.enabled</h5></td>
- <td style="word-wrap: break-word;">false</td>
- <td>Boolean</td>
- <td>Turns on SSL for external communication via the REST endpoints.</td>
- </tr>
- </tbody>
-</table>
diff --git a/docs/_includes/generated/common_state_backends_section.html b/docs/_includes/generated/common_state_backends_section.html
new file mode 100644
index 0000000..0df38e9
--- /dev/null
+++ b/docs/_includes/generated/common_state_backends_section.html
@@ -0,0 +1,54 @@
+<table class="table table-bordered">
+ <thead>
+ <tr>
+ <th class="text-left" style="width: 20%">Key</th>
+ <th class="text-left" style="width: 15%">Default</th>
+ <th class="text-left" style="width: 10%">Type</th>
+ <th class="text-left" style="width: 55%">Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><h5>state.backend</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>The state backend to be used to store and checkpoint state.</td>
+ </tr>
+ <tr>
+ <td><h5>state.checkpoints.dir</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>The default directory used for storing the data files and meta data of checkpoints in a Flink supported filesystem. The storage path must be accessible from all participating processes/nodes(i.e. all TaskManagers and JobManagers).</td>
+ </tr>
+ <tr>
+ <td><h5>state.savepoints.dir</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>The default directory for savepoints. Used by the state backends that write savepoints to file systems (MemoryStateBackend, FsStateBackend, RocksDBStateBackend).</td>
+ </tr>
+ <tr>
+ <td><h5>state.backend.incremental</h5></td>
+ <td style="word-wrap: break-word;">false</td>
+ <td>Boolean</td>
+ <td>Option whether the state backend should create incremental checkpoints, if possible. For an incremental checkpoint, only a diff from the previous checkpoint is stored, rather than the complete checkpoint state. Some state backends may not support incremental checkpoints and ignore this option.</td>
+ </tr>
+ <tr>
+ <td><h5>state.backend.local-recovery</h5></td>
+ <td style="word-wrap: break-word;">false</td>
+ <td>Boolean</td>
+ <td>This option configures local recovery for this state backend. By default, local recovery is deactivated. Local recovery currently only covers keyed state backends. Currently, MemoryStateBackend does not support local recovery and ignore this option.</td>
+ </tr>
+ <tr>
+ <td><h5>state.checkpoints.num-retained</h5></td>
+ <td style="word-wrap: break-word;">1</td>
+ <td>Integer</td>
+ <td>The maximum number of completed checkpoints to retain.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.state.local.root-dirs</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>The config parameter defining the root directories for storing file-based state for local recovery. Local recovery currently only covers keyed state backends. Currently, MemoryStateBackend does not support local recovery and ignore this option</td>
+ </tr>
+ </tbody>
+</table>
diff --git a/docs/_includes/generated/core_configuration.html b/docs/_includes/generated/core_configuration.html
index 60f0520..cf68ddf 100644
--- a/docs/_includes/generated/core_configuration.html
+++ b/docs/_includes/generated/core_configuration.html
@@ -27,6 +27,24 @@
<td>Defines the class resolution strategy when loading classes from user code, meaning whether to first check the user code jar ("child-first") or the application classpath ("parent-first"). The default settings indicate to load classes first from the user code jar, which means that user code jars can include and load different dependencies than Flink uses (transitively).</td>
</tr>
<tr>
+ <td><h5>fs.default-scheme</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>The default filesystem scheme, used for paths that do not declare a scheme explicitly. May contain an authority, e.g. host:port in case of an HDFS NameNode.</td>
+ </tr>
+ <tr>
+ <td><h5>fs.output.always-create-directory</h5></td>
+ <td style="word-wrap: break-word;">false</td>
+ <td>Boolean</td>
+ <td>File writers running with a parallelism larger than one create a directory for the output file path and put the different result files (one per parallel writer task) into that directory. If this option is set to "true", writers with a parallelism of 1 will also create a directory and place a single result file into it. If the option is set to "false", the writer will directly create the file directly at the output path, without creating a containing directory.</td>
+ </tr>
+ <tr>
+ <td><h5>fs.overwrite-files</h5></td>
+ <td style="word-wrap: break-word;">false</td>
+ <td>Boolean</td>
+ <td>Specifies whether file output writers should overwrite existing files by default. Set to "true" to overwrite by default,"false" otherwise.</td>
+ </tr>
+ <tr>
<td><h5>io.tmp.dirs</h5></td>
<td style="word-wrap: break-word;">'LOCAL_DIRS' on Yarn. '_FLINK_TMP_DIR' on Mesos. System.getProperty("java.io.tmpdir") in standalone.</td>
<td>String</td>
diff --git a/docs/_includes/generated/deprecated_file_sinks_section.html b/docs/_includes/generated/deprecated_file_sinks_section.html
new file mode 100644
index 0000000..26e3d53
--- /dev/null
+++ b/docs/_includes/generated/deprecated_file_sinks_section.html
@@ -0,0 +1,24 @@
+<table class="table table-bordered">
+ <thead>
+ <tr>
+ <th class="text-left" style="width: 20%">Key</th>
+ <th class="text-left" style="width: 15%">Default</th>
+ <th class="text-left" style="width: 10%">Type</th>
+ <th class="text-left" style="width: 55%">Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><h5>fs.output.always-create-directory</h5></td>
+ <td style="word-wrap: break-word;">false</td>
+ <td>Boolean</td>
+ <td>File writers running with a parallelism larger than one create a directory for the output file path and put the different result files (one per parallel writer task) into that directory. If this option is set to "true", writers with a parallelism of 1 will also create a directory and place a single result file into it. If the option is set to "false", the writer will directly create the file directly at the output path, without creating a containing directory.</td>
+ </tr>
+ <tr>
+ <td><h5>fs.overwrite-files</h5></td>
+ <td style="word-wrap: break-word;">false</td>
+ <td>Boolean</td>
+ <td>Specifies whether file output writers should overwrite existing files by default. Set to "true" to overwrite by default,"false" otherwise.</td>
+ </tr>
+ </tbody>
+</table>
diff --git a/docs/_includes/generated/core_configuration.html b/docs/_includes/generated/expert_class_loading_section.html
similarity index 79%
copy from docs/_includes/generated/core_configuration.html
copy to docs/_includes/generated/expert_class_loading_section.html
index 60f0520..1732e84 100644
--- a/docs/_includes/generated/core_configuration.html
+++ b/docs/_includes/generated/expert_class_loading_section.html
@@ -26,17 +26,5 @@
<td>String</td>
<td>Defines the class resolution strategy when loading classes from user code, meaning whether to first check the user code jar ("child-first") or the application classpath ("parent-first"). The default settings indicate to load classes first from the user code jar, which means that user code jars can include and load different dependencies than Flink uses (transitively).</td>
</tr>
- <tr>
- <td><h5>io.tmp.dirs</h5></td>
- <td style="word-wrap: break-word;">'LOCAL_DIRS' on Yarn. '_FLINK_TMP_DIR' on Mesos. System.getProperty("java.io.tmpdir") in standalone.</td>
- <td>String</td>
- <td>Directories for temporary files, separated by",", "|", or the system's java.io.File.pathSeparator.</td>
- </tr>
- <tr>
- <td><h5>parallelism.default</h5></td>
- <td style="word-wrap: break-word;">1</td>
- <td>Integer</td>
- <td>Default parallelism for jobs.</td>
- </tr>
</tbody>
</table>
diff --git a/docs/_includes/generated/expert_fault_tolerance_section.html b/docs/_includes/generated/expert_fault_tolerance_section.html
new file mode 100644
index 0000000..520717b
--- /dev/null
+++ b/docs/_includes/generated/expert_fault_tolerance_section.html
@@ -0,0 +1,60 @@
+<table class="table table-bordered">
+ <thead>
+ <tr>
+ <th class="text-left" style="width: 20%">Key</th>
+ <th class="text-left" style="width: 15%">Default</th>
+ <th class="text-left" style="width: 10%">Type</th>
+ <th class="text-left" style="width: 55%">Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><h5>cluster.registration.error-delay</h5></td>
+ <td style="word-wrap: break-word;">10000</td>
+ <td>Long</td>
+ <td>The pause made after an registration attempt caused an exception (other than timeout) in milliseconds.</td>
+ </tr>
+ <tr>
+ <td><h5>cluster.registration.initial-timeout</h5></td>
+ <td style="word-wrap: break-word;">100</td>
+ <td>Long</td>
+ <td>Initial registration timeout between cluster components in milliseconds.</td>
+ </tr>
+ <tr>
+ <td><h5>cluster.registration.max-timeout</h5></td>
+ <td style="word-wrap: break-word;">30000</td>
+ <td>Long</td>
+ <td>Maximum registration timeout between cluster components in milliseconds.</td>
+ </tr>
+ <tr>
+ <td><h5>cluster.registration.refused-registration-delay</h5></td>
+ <td style="word-wrap: break-word;">30000</td>
+ <td>Long</td>
+ <td>The pause made after the registration attempt was refused in milliseconds.</td>
+ </tr>
+ <tr>
+ <td><h5>cluster.services.shutdown-timeout</h5></td>
+ <td style="word-wrap: break-word;">30000</td>
+ <td>Long</td>
+ <td>The shutdown timeout for cluster services like executors in milliseconds.</td>
+ </tr>
+ <tr>
+ <td><h5>heartbeat.interval</h5></td>
+ <td style="word-wrap: break-word;">10000</td>
+ <td>Long</td>
+ <td>Time interval for requesting heartbeat from sender side.</td>
+ </tr>
+ <tr>
+ <td><h5>heartbeat.timeout</h5></td>
+ <td style="word-wrap: break-word;">50000</td>
+ <td>Long</td>
+ <td>Timeout for requesting and receiving heartbeat for both sender and receiver sides.</td>
+ </tr>
+ <tr>
+ <td><h5>jobmanager.execution.failover-strategy</h5></td>
+ <td style="word-wrap: break-word;">region</td>
+ <td>String</td>
+ <td>This option specifies how the job computation recovers from task failures. Accepted values are:<ul><li>'full': Restarts all tasks to recover the job.</li><li>'region': Restarts all tasks that could be affected by the task failure. More details can be found <a href="../dev/task_failure_recovery.html#restart-pipelined-region-failover-strategy">here</a>.</li></ul></td>
+ </tr>
+ </tbody>
+</table>
diff --git a/docs/_includes/generated/expert_high_availability_section.html b/docs/_includes/generated/expert_high_availability_section.html
new file mode 100644
index 0000000..6ddbeb5
--- /dev/null
+++ b/docs/_includes/generated/expert_high_availability_section.html
@@ -0,0 +1,18 @@
+<table class="table table-bordered">
+ <thead>
+ <tr>
+ <th class="text-left" style="width: 20%">Key</th>
+ <th class="text-left" style="width: 15%">Default</th>
+ <th class="text-left" style="width: 10%">Type</th>
+ <th class="text-left" style="width: 55%">Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><h5>high-availability.jobmanager.port</h5></td>
+ <td style="word-wrap: break-word;">"0"</td>
+ <td>String</td>
+ <td>Optional port (range) used by the job manager in high-availability mode.</td>
+ </tr>
+ </tbody>
+</table>
diff --git a/docs/_includes/generated/expert_high_availability_zk_section.html b/docs/_includes/generated/expert_high_availability_zk_section.html
new file mode 100644
index 0000000..d7774e2
--- /dev/null
+++ b/docs/_includes/generated/expert_high_availability_zk_section.html
@@ -0,0 +1,84 @@
+<table class="table table-bordered">
+ <thead>
+ <tr>
+ <th class="text-left" style="width: 20%">Key</th>
+ <th class="text-left" style="width: 15%">Default</th>
+ <th class="text-left" style="width: 10%">Type</th>
+ <th class="text-left" style="width: 55%">Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><h5>high-availability.zookeeper.client.acl</h5></td>
+ <td style="word-wrap: break-word;">"open"</td>
+ <td>String</td>
+ <td>Defines the ACL (open|creator) to be configured on ZK node. The configuration value can be set to “creator” if the ZooKeeper server configuration has the “authProvider” property mapped to use SASLAuthenticationProvider and the cluster is configured to run in secure mode (Kerberos).</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.client.connection-timeout</h5></td>
+ <td style="word-wrap: break-word;">15000</td>
+ <td>Integer</td>
+ <td>Defines the connection timeout for ZooKeeper in ms.</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.client.max-retry-attempts</h5></td>
+ <td style="word-wrap: break-word;">3</td>
+ <td>Integer</td>
+ <td>Defines the number of connection retries before the client gives up.</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.client.retry-wait</h5></td>
+ <td style="word-wrap: break-word;">5000</td>
+ <td>Integer</td>
+ <td>Defines the pause between consecutive retries in ms.</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.client.session-timeout</h5></td>
+ <td style="word-wrap: break-word;">60000</td>
+ <td>Integer</td>
+ <td>Defines the session timeout for the ZooKeeper session in ms.</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.path.checkpoint-counter</h5></td>
+ <td style="word-wrap: break-word;">"/checkpoint-counter"</td>
+ <td>String</td>
+ <td>ZooKeeper root path (ZNode) for checkpoint counters.</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.path.checkpoints</h5></td>
+ <td style="word-wrap: break-word;">"/checkpoints"</td>
+ <td>String</td>
+ <td>ZooKeeper root path (ZNode) for completed checkpoints.</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.path.jobgraphs</h5></td>
+ <td style="word-wrap: break-word;">"/jobgraphs"</td>
+ <td>String</td>
+ <td>ZooKeeper root path (ZNode) for job graphs</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.path.latch</h5></td>
+ <td style="word-wrap: break-word;">"/leaderlatch"</td>
+ <td>String</td>
+ <td>Defines the znode of the leader latch which is used to elect the leader.</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.path.leader</h5></td>
+ <td style="word-wrap: break-word;">"/leader"</td>
+ <td>String</td>
+ <td>Defines the znode of the leader which contains the URL to the leader and the current leader session ID.</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.path.mesos-workers</h5></td>
+ <td style="word-wrap: break-word;">"/mesos-workers"</td>
+ <td>String</td>
+ <td>The ZooKeeper root path for persisting the Mesos worker information.</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.path.running-registry</h5></td>
+ <td style="word-wrap: break-word;">"/running_job_registry/"</td>
+ <td>String</td>
+ <td></td>
+ </tr>
+ </tbody>
+</table>
diff --git a/docs/_includes/generated/expert_rest_section.html b/docs/_includes/generated/expert_rest_section.html
new file mode 100644
index 0000000..ab1a70c
--- /dev/null
+++ b/docs/_includes/generated/expert_rest_section.html
@@ -0,0 +1,66 @@
+<table class="table table-bordered">
+ <thead>
+ <tr>
+ <th class="text-left" style="width: 20%">Key</th>
+ <th class="text-left" style="width: 15%">Default</th>
+ <th class="text-left" style="width: 10%">Type</th>
+ <th class="text-left" style="width: 55%">Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><h5>rest.await-leader-timeout</h5></td>
+ <td style="word-wrap: break-word;">30000</td>
+ <td>Long</td>
+ <td>The time in ms that the client waits for the leader address, e.g., Dispatcher or WebMonitorEndpoint</td>
+ </tr>
+ <tr>
+ <td><h5>rest.client.max-content-length</h5></td>
+ <td style="word-wrap: break-word;">104857600</td>
+ <td>Integer</td>
+ <td>The maximum content length in bytes that the client will handle.</td>
+ </tr>
+ <tr>
+ <td><h5>rest.connection-timeout</h5></td>
+ <td style="word-wrap: break-word;">15000</td>
+ <td>Long</td>
+ <td>The maximum time in ms for the client to establish a TCP connection.</td>
+ </tr>
+ <tr>
+ <td><h5>rest.idleness-timeout</h5></td>
+ <td style="word-wrap: break-word;">300000</td>
+ <td>Long</td>
+ <td>The maximum time in ms for a connection to stay idle before failing.</td>
+ </tr>
+ <tr>
+ <td><h5>rest.retry.delay</h5></td>
+ <td style="word-wrap: break-word;">3000</td>
+ <td>Long</td>
+ <td>The time in ms that the client waits between retries (See also `rest.retry.max-attempts`).</td>
+ </tr>
+ <tr>
+ <td><h5>rest.retry.max-attempts</h5></td>
+ <td style="word-wrap: break-word;">20</td>
+ <td>Integer</td>
+ <td>The number of retries the client will attempt if a retryable operations fails.</td>
+ </tr>
+ <tr>
+ <td><h5>rest.server.max-content-length</h5></td>
+ <td style="word-wrap: break-word;">104857600</td>
+ <td>Integer</td>
+ <td>The maximum content length in bytes that the server will handle.</td>
+ </tr>
+ <tr>
+ <td><h5>rest.server.numThreads</h5></td>
+ <td style="word-wrap: break-word;">4</td>
+ <td>Integer</td>
+ <td>The number of threads for the asynchronous processing of requests.</td>
+ </tr>
+ <tr>
+ <td><h5>rest.server.thread-priority</h5></td>
+ <td style="word-wrap: break-word;">5</td>
+ <td>Integer</td>
+ <td>Thread priority of the REST server's executor for processing asynchronous requests. Lowering the thread priority will give Flink's main components more CPU time whereas increasing will allocate more time for the REST server's processing.</td>
+ </tr>
+ </tbody>
+</table>
diff --git a/docs/_includes/generated/expert_rocksdb_section.html b/docs/_includes/generated/expert_rocksdb_section.html
new file mode 100644
index 0000000..d3cbfe1
--- /dev/null
+++ b/docs/_includes/generated/expert_rocksdb_section.html
@@ -0,0 +1,36 @@
+<table class="table table-bordered">
+ <thead>
+ <tr>
+ <th class="text-left" style="width: 20%">Key</th>
+ <th class="text-left" style="width: 15%">Default</th>
+ <th class="text-left" style="width: 10%">Type</th>
+ <th class="text-left" style="width: 55%">Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><h5>state.backend.rocksdb.checkpoint.transfer.thread.num</h5></td>
+ <td style="word-wrap: break-word;">1</td>
+ <td>Integer</td>
+ <td>The number of threads (per stateful operator) used to transfer (download and upload) files in RocksDBStateBackend.</td>
+ </tr>
+ <tr>
+ <td><h5>state.backend.rocksdb.localdir</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>The local directory (on the TaskManager) where RocksDB puts its files.</td>
+ </tr>
+ <tr>
+ <td><h5>state.backend.rocksdb.options-factory</h5></td>
+ <td style="word-wrap: break-word;">"org.apache.flink.contrib.streaming.state.DefaultConfigurableOptionsFactory"</td>
+ <td>String</td>
+ <td>The options factory class for RocksDB to create DBOptions and ColumnFamilyOptions. The default options factory is org.apache.flink.contrib.streaming.state.DefaultConfigurableOptionsFactory, and it would read the configured options which provided in 'RocksDBConfigurableOptions'.</td>
+ </tr>
+ <tr>
+ <td><h5>state.backend.rocksdb.predefined-options</h5></td>
+ <td style="word-wrap: break-word;">"DEFAULT"</td>
+ <td>String</td>
+ <td>The predefined settings for RocksDB DBOptions and ColumnFamilyOptions by Flink community. Current supported candidate predefined-options are DEFAULT, SPINNING_DISK_OPTIMIZED, SPINNING_DISK_OPTIMIZED_HIGH_MEM or FLASH_SSD_OPTIMIZED. Note that user customized options and options from the OptionsFactory are applied on top of these predefined ones.</td>
+ </tr>
+ </tbody>
+</table>
diff --git a/docs/_includes/generated/expert_scheduling_section.html b/docs/_includes/generated/expert_scheduling_section.html
new file mode 100644
index 0000000..a29ad69
--- /dev/null
+++ b/docs/_includes/generated/expert_scheduling_section.html
@@ -0,0 +1,30 @@
+<table class="table table-bordered">
+ <thead>
+ <tr>
+ <th class="text-left" style="width: 20%">Key</th>
+ <th class="text-left" style="width: 15%">Default</th>
+ <th class="text-left" style="width: 10%">Type</th>
+ <th class="text-left" style="width: 55%">Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><h5>cluster.evenly-spread-out-slots</h5></td>
+ <td style="word-wrap: break-word;">false</td>
+ <td>Boolean</td>
+ <td>Enable the slot spread out allocation strategy. This strategy tries to spread out the slots evenly across all available <span markdown="span">`TaskExecutors`</span>.</td>
+ </tr>
+ <tr>
+ <td><h5>slot.idle.timeout</h5></td>
+ <td style="word-wrap: break-word;">50000</td>
+ <td>Long</td>
+ <td>The timeout in milliseconds for a idle slot in Slot Pool.</td>
+ </tr>
+ <tr>
+ <td><h5>slot.request.timeout</h5></td>
+ <td style="word-wrap: break-word;">300000</td>
+ <td>Long</td>
+ <td>The timeout in milliseconds for requesting a slot from Slot Pool.</td>
+ </tr>
+ </tbody>
+</table>
diff --git a/docs/_includes/generated/expert_security_ssl_section.html b/docs/_includes/generated/expert_security_ssl_section.html
new file mode 100644
index 0000000..9e5af5f
--- /dev/null
+++ b/docs/_includes/generated/expert_security_ssl_section.html
@@ -0,0 +1,42 @@
+<table class="table table-bordered">
+ <thead>
+ <tr>
+ <th class="text-left" style="width: 20%">Key</th>
+ <th class="text-left" style="width: 15%">Default</th>
+ <th class="text-left" style="width: 10%">Type</th>
+ <th class="text-left" style="width: 55%">Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><h5>security.ssl.internal.close-notify-flush-timeout</h5></td>
+ <td style="word-wrap: break-word;">-1</td>
+ <td>Integer</td>
+ <td>The timeout (in ms) for flushing the `close_notify` that was triggered by closing a channel. If the `close_notify` was not flushed in the given timeout the channel will be closed forcibly. (-1 = use system default)</td>
+ </tr>
+ <tr>
+ <td><h5>security.ssl.internal.handshake-timeout</h5></td>
+ <td style="word-wrap: break-word;">-1</td>
+ <td>Integer</td>
+ <td>The timeout (in ms) during SSL handshake. (-1 = use system default)</td>
+ </tr>
+ <tr>
+ <td><h5>security.ssl.internal.session-cache-size</h5></td>
+ <td style="word-wrap: break-word;">-1</td>
+ <td>Integer</td>
+ <td>The size of the cache used for storing SSL session objects. According to https://github.com/netty/netty/issues/832, you should always set this to an appropriate number to not run into a bug with stalling IO threads during garbage collection. (-1 = use system default).</td>
+ </tr>
+ <tr>
+ <td><h5>security.ssl.internal.session-timeout</h5></td>
+ <td style="word-wrap: break-word;">-1</td>
+ <td>Integer</td>
+ <td>The timeout (in ms) for the cached SSL session objects. (-1 = use system default)</td>
+ </tr>
+ <tr>
+ <td><h5>security.ssl.provider</h5></td>
+ <td style="word-wrap: break-word;">"JDK"</td>
+ <td>String</td>
+ <td>The SSL engine provider to use for the ssl transport:<ul><li><span markdown="span">`JDK`</span>: default Java-based SSL engine</li><li><span markdown="span">`OPENSSL`</span>: openSSL-based SSL engine using system libraries</li></ul><span markdown="span">`OPENSSL`</span> is based on <a href="http://netty.io/wiki/forked-tomcat-native.html#wiki-h2-4">netty-tcnative</a> and comes in two flavours:<ul><li>dynamically linked: This will use your system's openSSL libraries (if com [...]
+ </tr>
+ </tbody>
+</table>
diff --git a/docs/_includes/generated/expert_state_backends_section.html b/docs/_includes/generated/expert_state_backends_section.html
new file mode 100644
index 0000000..9d50be1
--- /dev/null
+++ b/docs/_includes/generated/expert_state_backends_section.html
@@ -0,0 +1,30 @@
+<table class="table table-bordered">
+ <thead>
+ <tr>
+ <th class="text-left" style="width: 20%">Key</th>
+ <th class="text-left" style="width: 15%">Default</th>
+ <th class="text-left" style="width: 10%">Type</th>
+ <th class="text-left" style="width: 55%">Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><h5>state.backend.async</h5></td>
+ <td style="word-wrap: break-word;">true</td>
+ <td>Boolean</td>
+ <td>Option whether the state backend should use an asynchronous snapshot method where possible and configurable. Some state backends may not support asynchronous snapshots, or only support asynchronous snapshots, and ignore this option.</td>
+ </tr>
+ <tr>
+ <td><h5>state.backend.fs.memory-threshold</h5></td>
+ <td style="word-wrap: break-word;">1024</td>
+ <td>Integer</td>
+ <td>The minimum size of state data files. All state chunks smaller than that are stored inline in the root checkpoint metadata file.</td>
+ </tr>
+ <tr>
+ <td><h5>state.backend.fs.write-buffer-size</h5></td>
+ <td style="word-wrap: break-word;">4096</td>
+ <td>Integer</td>
+ <td>The default size of the write buffer for the checkpoint streams that write to file systems. The actual write buffer size is determined to be the maximum of the value of this option and option 'state.backend.fs.memory-threshold'.</td>
+ </tr>
+ </tbody>
+</table>
diff --git a/docs/_includes/generated/high_availability_configuration.html b/docs/_includes/generated/high_availability_configuration.html
index bf30e91..86573d9 100644
--- a/docs/_includes/generated/high_availability_configuration.html
+++ b/docs/_includes/generated/high_availability_configuration.html
@@ -32,5 +32,89 @@
<td>String</td>
<td>File system path (URI) where Flink persists metadata in high-availability setups.</td>
</tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.client.acl</h5></td>
+ <td style="word-wrap: break-word;">"open"</td>
+ <td>String</td>
+ <td>Defines the ACL (open|creator) to be configured on ZK node. The configuration value can be set to “creator” if the ZooKeeper server configuration has the “authProvider” property mapped to use SASLAuthenticationProvider and the cluster is configured to run in secure mode (Kerberos).</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.client.connection-timeout</h5></td>
+ <td style="word-wrap: break-word;">15000</td>
+ <td>Integer</td>
+ <td>Defines the connection timeout for ZooKeeper in ms.</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.client.max-retry-attempts</h5></td>
+ <td style="word-wrap: break-word;">3</td>
+ <td>Integer</td>
+ <td>Defines the number of connection retries before the client gives up.</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.client.retry-wait</h5></td>
+ <td style="word-wrap: break-word;">5000</td>
+ <td>Integer</td>
+ <td>Defines the pause between consecutive retries in ms.</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.client.session-timeout</h5></td>
+ <td style="word-wrap: break-word;">60000</td>
+ <td>Integer</td>
+ <td>Defines the session timeout for the ZooKeeper session in ms.</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.path.checkpoint-counter</h5></td>
+ <td style="word-wrap: break-word;">"/checkpoint-counter"</td>
+ <td>String</td>
+ <td>ZooKeeper root path (ZNode) for checkpoint counters.</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.path.checkpoints</h5></td>
+ <td style="word-wrap: break-word;">"/checkpoints"</td>
+ <td>String</td>
+ <td>ZooKeeper root path (ZNode) for completed checkpoints.</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.path.jobgraphs</h5></td>
+ <td style="word-wrap: break-word;">"/jobgraphs"</td>
+ <td>String</td>
+ <td>ZooKeeper root path (ZNode) for job graphs</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.path.latch</h5></td>
+ <td style="word-wrap: break-word;">"/leaderlatch"</td>
+ <td>String</td>
+ <td>Defines the znode of the leader latch which is used to elect the leader.</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.path.leader</h5></td>
+ <td style="word-wrap: break-word;">"/leader"</td>
+ <td>String</td>
+ <td>Defines the znode of the leader which contains the URL to the leader and the current leader session ID.</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.path.mesos-workers</h5></td>
+ <td style="word-wrap: break-word;">"/mesos-workers"</td>
+ <td>String</td>
+ <td>The ZooKeeper root path for persisting the Mesos worker information.</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.path.root</h5></td>
+ <td style="word-wrap: break-word;">"/flink"</td>
+ <td>String</td>
+ <td>The root path under which Flink stores its entries in ZooKeeper.</td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.path.running-registry</h5></td>
+ <td style="word-wrap: break-word;">"/running_job_registry/"</td>
+ <td>String</td>
+ <td></td>
+ </tr>
+ <tr>
+ <td><h5>high-availability.zookeeper.quorum</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>The ZooKeeper quorum to use, when running Flink in a high-availability mode with ZooKeeper.</td>
+ </tr>
</tbody>
</table>
diff --git a/docs/_includes/generated/job_manager_configuration.html b/docs/_includes/generated/job_manager_configuration.html
index 76671e5..ef9c565 100644
--- a/docs/_includes/generated/job_manager_configuration.html
+++ b/docs/_includes/generated/job_manager_configuration.html
@@ -22,7 +22,7 @@
</tr>
<tr>
<td><h5>jobmanager.execution.failover-strategy</h5></td>
- <td style="word-wrap: break-word;">"full"</td>
+ <td style="word-wrap: break-word;">region</td>
<td>String</td>
<td>This option specifies how the job computation recovers from task failures. Accepted values are:<ul><li>'full': Restarts all tasks to recover the job.</li><li>'region': Restarts all tasks that could be affected by the task failure. More details can be found <a href="../dev/task_failure_recovery.html#restart-pipelined-region-failover-strategy">here</a>.</li></ul></td>
</tr>
diff --git a/docs/_includes/generated/netty_shuffle_environment_configuration.html b/docs/_includes/generated/netty_shuffle_environment_configuration.html
index 8a5b3ba..901a40b 100644
--- a/docs/_includes/generated/netty_shuffle_environment_configuration.html
+++ b/docs/_includes/generated/netty_shuffle_environment_configuration.html
@@ -51,6 +51,48 @@
<td>Number of extra network buffers to use for each outgoing/incoming gate (result partition/input gate). In credit-based flow control mode, this indicates how many floating credits are shared among all the input channels. The floating buffers are distributed based on backlog (real-time output buffers in the subpartition) feedback, and can help relieve back-pressure caused by unbalanced data distribution among the subpartitions. This value should be increased in case of highe [...]
</tr>
<tr>
+ <td><h5>taskmanager.network.netty.client.connectTimeoutSec</h5></td>
+ <td style="word-wrap: break-word;">120</td>
+ <td>Integer</td>
+ <td>The Netty client connection timeout.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.network.netty.client.numThreads</h5></td>
+ <td style="word-wrap: break-word;">-1</td>
+ <td>Integer</td>
+ <td>The number of Netty client threads.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.network.netty.num-arenas</h5></td>
+ <td style="word-wrap: break-word;">-1</td>
+ <td>Integer</td>
+ <td>The number of Netty arenas.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.network.netty.sendReceiveBufferSize</h5></td>
+ <td style="word-wrap: break-word;">0</td>
+ <td>Integer</td>
+ <td>The Netty send and receive buffer size. This defaults to the system buffer size (cat /proc/sys/net/ipv4/tcp_[rw]mem) and is 4 MiB in modern Linux.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.network.netty.server.backlog</h5></td>
+ <td style="word-wrap: break-word;">0</td>
+ <td>Integer</td>
+ <td>The netty server connection backlog.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.network.netty.server.numThreads</h5></td>
+ <td style="word-wrap: break-word;">-1</td>
+ <td>Integer</td>
+ <td>The number of Netty server threads.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.network.netty.transport</h5></td>
+ <td style="word-wrap: break-word;">"auto"</td>
+ <td>String</td>
+ <td>The Netty transport type, either "nio" or "epoll". The "auto" means selecting the property mode automatically based on the platform. Note that the "epoll" mode can get better performance, less GC and have more advanced features which are only available on modern Linux.</td>
+ </tr>
+ <tr>
<td><h5>taskmanager.network.request-backoff.initial</h5></td>
<td style="word-wrap: break-word;">100</td>
<td>Integer</td>
diff --git a/docs/_includes/generated/resource_manager_configuration.html b/docs/_includes/generated/resource_manager_configuration.html
index 0e7192f..c1955f8 100644
--- a/docs/_includes/generated/resource_manager_configuration.html
+++ b/docs/_includes/generated/resource_manager_configuration.html
@@ -21,12 +21,6 @@
<td>Percentage of heap space to remove from Job Master containers (YARN / Mesos / Kubernetes), to compensate for other JVM memory usage.</td>
</tr>
<tr>
- <td><h5>local.number-resourcemanager</h5></td>
- <td style="word-wrap: break-word;">1</td>
- <td>Integer</td>
- <td>The number of resource managers start.</td>
- </tr>
- <tr>
<td><h5>resourcemanager.job.timeout</h5></td>
<td style="word-wrap: break-word;">"5 minutes"</td>
<td>String</td>
diff --git a/docs/_includes/generated/security_auth_kerberos_section.html b/docs/_includes/generated/security_auth_kerberos_section.html
new file mode 100644
index 0000000..cf27917
--- /dev/null
+++ b/docs/_includes/generated/security_auth_kerberos_section.html
@@ -0,0 +1,36 @@
+<table class="table table-bordered">
+ <thead>
+ <tr>
+ <th class="text-left" style="width: 20%">Key</th>
+ <th class="text-left" style="width: 15%">Default</th>
+ <th class="text-left" style="width: 10%">Type</th>
+ <th class="text-left" style="width: 55%">Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><h5>security.kerberos.login.contexts</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>A comma-separated list of login contexts to provide the Kerberos credentials to (for example, `Client,KafkaClient` to use the credentials for ZooKeeper authentication and for Kafka authentication)</td>
+ </tr>
+ <tr>
+ <td><h5>security.kerberos.login.keytab</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>Absolute path to a Kerberos keytab file that contains the user credentials.</td>
+ </tr>
+ <tr>
+ <td><h5>security.kerberos.login.principal</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>Kerberos principal name associated with the keytab.</td>
+ </tr>
+ <tr>
+ <td><h5>security.kerberos.login.use-ticket-cache</h5></td>
+ <td style="word-wrap: break-word;">true</td>
+ <td>Boolean</td>
+ <td>Indicates whether to read from your Kerberos ticket cache.</td>
+ </tr>
+ </tbody>
+</table>
diff --git a/docs/_includes/generated/security_auth_zk_section.html b/docs/_includes/generated/security_auth_zk_section.html
new file mode 100644
index 0000000..7aaaaff
--- /dev/null
+++ b/docs/_includes/generated/security_auth_zk_section.html
@@ -0,0 +1,30 @@
+<table class="table table-bordered">
+ <thead>
+ <tr>
+ <th class="text-left" style="width: 20%">Key</th>
+ <th class="text-left" style="width: 15%">Default</th>
+ <th class="text-left" style="width: 10%">Type</th>
+ <th class="text-left" style="width: 55%">Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><h5>zookeeper.sasl.disable</h5></td>
+ <td style="word-wrap: break-word;">false</td>
+ <td>Boolean</td>
+ <td></td>
+ </tr>
+ <tr>
+ <td><h5>zookeeper.sasl.login-context-name</h5></td>
+ <td style="word-wrap: break-word;">"Client"</td>
+ <td>String</td>
+ <td></td>
+ </tr>
+ <tr>
+ <td><h5>zookeeper.sasl.service-name</h5></td>
+ <td style="word-wrap: break-word;">"zookeeper"</td>
+ <td>String</td>
+ <td></td>
+ </tr>
+ </tbody>
+</table>
diff --git a/docs/_includes/generated/security_configuration.html b/docs/_includes/generated/security_configuration.html
index bdf49f2..caeaa33 100644
--- a/docs/_includes/generated/security_configuration.html
+++ b/docs/_includes/generated/security_configuration.html
@@ -9,6 +9,30 @@
</thead>
<tbody>
<tr>
+ <td><h5>security.kerberos.login.contexts</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>A comma-separated list of login contexts to provide the Kerberos credentials to (for example, `Client,KafkaClient` to use the credentials for ZooKeeper authentication and for Kafka authentication)</td>
+ </tr>
+ <tr>
+ <td><h5>security.kerberos.login.keytab</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>Absolute path to a Kerberos keytab file that contains the user credentials.</td>
+ </tr>
+ <tr>
+ <td><h5>security.kerberos.login.principal</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>String</td>
+ <td>Kerberos principal name associated with the keytab.</td>
+ </tr>
+ <tr>
+ <td><h5>security.kerberos.login.use-ticket-cache</h5></td>
+ <td style="word-wrap: break-word;">true</td>
+ <td>Boolean</td>
+ <td>Indicates whether to read from your Kerberos ticket cache.</td>
+ </tr>
+ <tr>
<td><h5>security.ssl.algorithms</h5></td>
<td style="word-wrap: break-word;">"TLS_RSA_WITH_AES_128_CBC_SHA"</td>
<td>String</td>
@@ -81,24 +105,6 @@
<td>The password to decrypt the truststore for Flink's internal endpoints (rpc, data transport, blob server).</td>
</tr>
<tr>
- <td><h5>security.ssl.key-password</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>The secret to decrypt the server key in the keystore.</td>
- </tr>
- <tr>
- <td><h5>security.ssl.keystore</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>The Java keystore file to be used by the flink endpoint for its SSL Key and Certificate.</td>
- </tr>
- <tr>
- <td><h5>security.ssl.keystore-password</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>The secret to decrypt the keystore file.</td>
- </tr>
- <tr>
<td><h5>security.ssl.protocol</h5></td>
<td style="word-wrap: break-word;">"TLSv1.2"</td>
<td>String</td>
@@ -159,22 +165,28 @@
<td>The password to decrypt the truststore for Flink's external REST endpoints.</td>
</tr>
<tr>
- <td><h5>security.ssl.truststore</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>The truststore file containing the public CA certificates to be used by flink endpoints to verify the peer’s certificate.</td>
+ <td><h5>security.ssl.verify-hostname</h5></td>
+ <td style="word-wrap: break-word;">true</td>
+ <td>Boolean</td>
+ <td>Flag to enable peer’s hostname verification during ssl handshake.</td>
</tr>
<tr>
- <td><h5>security.ssl.truststore-password</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
+ <td><h5>zookeeper.sasl.disable</h5></td>
+ <td style="word-wrap: break-word;">false</td>
+ <td>Boolean</td>
+ <td></td>
+ </tr>
+ <tr>
+ <td><h5>zookeeper.sasl.login-context-name</h5></td>
+ <td style="word-wrap: break-word;">"Client"</td>
<td>String</td>
- <td>The secret to decrypt the truststore.</td>
+ <td></td>
</tr>
<tr>
- <td><h5>security.ssl.verify-hostname</h5></td>
- <td style="word-wrap: break-word;">true</td>
- <td>Boolean</td>
- <td>Flag to enable peer’s hostname verification during ssl handshake.</td>
+ <td><h5>zookeeper.sasl.service-name</h5></td>
+ <td style="word-wrap: break-word;">"zookeeper"</td>
+ <td>String</td>
+ <td></td>
</tr>
</tbody>
</table>
diff --git a/docs/_includes/generated/security_configuration.html b/docs/_includes/generated/security_ssl_section.html
similarity index 60%
copy from docs/_includes/generated/security_configuration.html
copy to docs/_includes/generated/security_ssl_section.html
index bdf49f2..ae345d8 100644
--- a/docs/_includes/generated/security_configuration.html
+++ b/docs/_includes/generated/security_ssl_section.html
@@ -21,24 +21,12 @@
<td>The sha1 fingerprint of the internal certificate. This further protects the internal communication to present the exact certificate used by Flink.This is necessary where one cannot use private CA(self signed) or there is internal firm wide CA is required</td>
</tr>
<tr>
- <td><h5>security.ssl.internal.close-notify-flush-timeout</h5></td>
- <td style="word-wrap: break-word;">-1</td>
- <td>Integer</td>
- <td>The timeout (in ms) for flushing the `close_notify` that was triggered by closing a channel. If the `close_notify` was not flushed in the given timeout the channel will be closed forcibly. (-1 = use system default)</td>
- </tr>
- <tr>
<td><h5>security.ssl.internal.enabled</h5></td>
<td style="word-wrap: break-word;">false</td>
<td>Boolean</td>
<td>Turns on SSL for internal network communication. Optionally, specific components may override this through their own settings (rpc, data transport, REST, etc).</td>
</tr>
<tr>
- <td><h5>security.ssl.internal.handshake-timeout</h5></td>
- <td style="word-wrap: break-word;">-1</td>
- <td>Integer</td>
- <td>The timeout (in ms) during SSL handshake. (-1 = use system default)</td>
- </tr>
- <tr>
<td><h5>security.ssl.internal.key-password</h5></td>
<td style="word-wrap: break-word;">(none)</td>
<td>String</td>
@@ -57,18 +45,6 @@
<td>The secret to decrypt the keystore file for Flink's for Flink's internal endpoints (rpc, data transport, blob server).</td>
</tr>
<tr>
- <td><h5>security.ssl.internal.session-cache-size</h5></td>
- <td style="word-wrap: break-word;">-1</td>
- <td>Integer</td>
- <td>The size of the cache used for storing SSL session objects. According to https://github.com/netty/netty/issues/832, you should always set this to an appropriate number to not run into a bug with stalling IO threads during garbage collection. (-1 = use system default).</td>
- </tr>
- <tr>
- <td><h5>security.ssl.internal.session-timeout</h5></td>
- <td style="word-wrap: break-word;">-1</td>
- <td>Integer</td>
- <td>The timeout (in ms) for the cached SSL session objects. (-1 = use system default)</td>
- </tr>
- <tr>
<td><h5>security.ssl.internal.truststore</h5></td>
<td style="word-wrap: break-word;">(none)</td>
<td>String</td>
@@ -81,36 +57,12 @@
<td>The password to decrypt the truststore for Flink's internal endpoints (rpc, data transport, blob server).</td>
</tr>
<tr>
- <td><h5>security.ssl.key-password</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>The secret to decrypt the server key in the keystore.</td>
- </tr>
- <tr>
- <td><h5>security.ssl.keystore</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>The Java keystore file to be used by the flink endpoint for its SSL Key and Certificate.</td>
- </tr>
- <tr>
- <td><h5>security.ssl.keystore-password</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>The secret to decrypt the keystore file.</td>
- </tr>
- <tr>
<td><h5>security.ssl.protocol</h5></td>
<td style="word-wrap: break-word;">"TLSv1.2"</td>
<td>String</td>
<td>The SSL protocol version to be supported for the ssl transport. Note that it doesn’t support comma separated list.</td>
</tr>
<tr>
- <td><h5>security.ssl.provider</h5></td>
- <td style="word-wrap: break-word;">"JDK"</td>
- <td>String</td>
- <td>The SSL engine provider to use for the ssl transport:<ul><li><span markdown="span">`JDK`</span>: default Java-based SSL engine</li><li><span markdown="span">`OPENSSL`</span>: openSSL-based SSL engine using system libraries</li></ul><span markdown="span">`OPENSSL`</span> is based on <a href="http://netty.io/wiki/forked-tomcat-native.html#wiki-h2-4">netty-tcnative</a> and comes in two flavours:<ul><li>dynamically linked: This will use your system's openSSL libraries (if com [...]
- </tr>
- <tr>
<td><h5>security.ssl.rest.authentication-enabled</h5></td>
<td style="word-wrap: break-word;">false</td>
<td>Boolean</td>
@@ -159,18 +111,6 @@
<td>The password to decrypt the truststore for Flink's external REST endpoints.</td>
</tr>
<tr>
- <td><h5>security.ssl.truststore</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>The truststore file containing the public CA certificates to be used by flink endpoints to verify the peer’s certificate.</td>
- </tr>
- <tr>
- <td><h5>security.ssl.truststore-password</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>The secret to decrypt the truststore.</td>
- </tr>
- <tr>
<td><h5>security.ssl.verify-hostname</h5></td>
<td style="word-wrap: break-word;">true</td>
<td>Boolean</td>
diff --git a/docs/_includes/generated/state_backend_rocksdb_section.html b/docs/_includes/generated/state_backend_rocksdb_section.html
new file mode 100644
index 0000000..974c5c1
--- /dev/null
+++ b/docs/_includes/generated/state_backend_rocksdb_section.html
@@ -0,0 +1,42 @@
+<table class="table table-bordered">
+ <thead>
+ <tr>
+ <th class="text-left" style="width: 20%">Key</th>
+ <th class="text-left" style="width: 15%">Default</th>
+ <th class="text-left" style="width: 10%">Type</th>
+ <th class="text-left" style="width: 55%">Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><h5>state.backend.rocksdb.memory.fixed-per-slot</h5></td>
+ <td style="word-wrap: break-word;">(none)</td>
+ <td>MemorySize</td>
+ <td>The fixed total amount of memory, shared among all RocksDB instances per slot. This option overrides the 'state.backend.rocksdb.memory.managed' option when configured. If neither this option, nor the 'state.backend.rocksdb.memory.managed' optionare set, then each RocksDB column family state has its own memory caches (as controlled by the column family options).</td>
+ </tr>
+ <tr>
+ <td><h5>state.backend.rocksdb.memory.high-prio-pool-ratio</h5></td>
+ <td style="word-wrap: break-word;">0.1</td>
+ <td>Double</td>
+ <td>The fraction of cache memory that is reserved for high-priority data like index, filter, and compression dictionary blocks. This option only has an effect when 'state.backend.rocksdb.memory.managed' or 'state.backend.rocksdb.memory.fixed-per-slot' are configured.</td>
+ </tr>
+ <tr>
+ <td><h5>state.backend.rocksdb.memory.managed</h5></td>
+ <td style="word-wrap: break-word;">true</td>
+ <td>Boolean</td>
+ <td>If set, the RocksDB state backend will automatically configure itself to use the managed memory budget of the task slot, and divide the memory over write buffers, indexes, block caches, etc. That way, the three major uses of memory of RocksDB will be capped.</td>
+ </tr>
+ <tr>
+ <td><h5>state.backend.rocksdb.memory.write-buffer-ratio</h5></td>
+ <td style="word-wrap: break-word;">0.5</td>
+ <td>Double</td>
+ <td>The maximum amount of memory that write buffers may take, as a fraction of the total shared memory. This option only has an effect when 'state.backend.rocksdb.memory.managed' or 'state.backend.rocksdb.memory.fixed-per-slot' are configured.</td>
+ </tr>
+ <tr>
+ <td><h5>state.backend.rocksdb.timer-service.factory</h5></td>
+ <td style="word-wrap: break-word;">"ROCKSDB"</td>
+ <td>String</td>
+ <td>This determines the factory for timer service state implementation. Options are either HEAP (heap-based, default) or ROCKSDB for an implementation based on RocksDB .</td>
+ </tr>
+ </tbody>
+</table>
diff --git a/docs/_includes/generated/task_manager_configuration.html b/docs/_includes/generated/task_manager_configuration.html
index fed0f21..db02dee 100644
--- a/docs/_includes/generated/task_manager_configuration.html
+++ b/docs/_includes/generated/task_manager_configuration.html
@@ -27,12 +27,6 @@
<td>Time we wait for the timers in milliseconds to finish all pending timer threads when the stream task is cancelled.</td>
</tr>
<tr>
- <td><h5>task.checkpoint.alignment.max-size</h5></td>
- <td style="word-wrap: break-word;">-1</td>
- <td>Long</td>
- <td>The maximum number of bytes that a checkpoint alignment may buffer. If the checkpoint alignment buffers more than the configured amount of data, the checkpoint is aborted (skipped). A value of -1 indicates that there is no limit.</td>
- </tr>
- <tr>
<td><h5>taskmanager.debug.memory.log</h5></td>
<td style="word-wrap: break-word;">false</td>
<td>Boolean</td>
diff --git a/docs/_includes/generated/web_configuration.html b/docs/_includes/generated/web_configuration.html
index 11a6968..f377f51 100644
--- a/docs/_includes/generated/web_configuration.html
+++ b/docs/_includes/generated/web_configuration.html
@@ -15,12 +15,6 @@
<td>Access-Control-Allow-Origin header for all responses from the web-frontend.</td>
</tr>
<tr>
- <td><h5>web.address</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>Address for runtime monitor web-frontend server.</td>
- </tr>
- <tr>
<td><h5>web.backpressure.cleanup-interval</h5></td>
<td style="word-wrap: break-word;">600000</td>
<td>Integer</td>
@@ -69,12 +63,6 @@
<td>Refresh interval for the web-frontend in milliseconds.</td>
</tr>
<tr>
- <td><h5>web.ssl.enabled</h5></td>
- <td style="word-wrap: break-word;">true</td>
- <td>Boolean</td>
- <td>Flag indicating whether to override SSL support for the JobManager Web UI.</td>
- </tr>
- <tr>
<td><h5>web.submit.enable</h5></td>
<td style="word-wrap: break-word;">true</td>
<td>Boolean</td>
diff --git a/docs/ops/config.md b/docs/ops/config.md
index 0a42ec7..032faa5 100644
--- a/docs/ops/config.md
+++ b/docs/ops/config.md
@@ -23,213 +23,411 @@ specific language governing permissions and limitations
under the License.
-->
-**For single-node setups Flink is ready to go out of the box and you don't need to change the default configuration to get started.**
+All configuration is done in `conf/flink-conf.yaml`, which is expected to be a flat collection of [YAML key value pairs](http://www.yaml.org/spec/1.2/spec.html) with format `key: value`.
+
+The configuration is parsed and evaluated when the Flink processes are started. Changes to the configuration file require restarting the relevant processes.
The out of the box configuration will use your default Java installation. You can manually set the environment variable `JAVA_HOME` or the configuration key `env.java.home` in `conf/flink-conf.yaml` if you want to manually override the Java runtime to use.
-This page lists the most common options that are typically needed to set up a well performing (distributed) installation. In addition a full list of all available configuration parameters is listed here.
+* This will be replaced by the TOC
+{:toc}
-All configuration is done in `conf/flink-conf.yaml`, which is expected to be a flat collection of [YAML key value pairs](http://www.yaml.org/spec/1.2/spec.html) with format `key: value`.
+# Basic Setup
-The system and run scripts parse the config at startup time. Changes to the configuration file require restarting the Flink JobManager and TaskManagers.
+The default configuration supports starting a single-node Flink session cluster without any changes.
+The options in this section are the ones most commonly needed for a basic distributed Flink setup.
-The configuration files for the TaskManagers can be different, Flink does not assume uniform machines in the cluster.
+**Hostnames / Ports**
-* This will be replaced by the TOC
-{:toc}
+These options are only necessary for *standalone* application- or session deployments ([simple standalone]({{site.baseurl}}/ops/deployment/cluster_setup.html) or [Kubernetes]({{site.baseurl}}/ops/deployment/kubernetes.html)).
-## Common Options
+If you use Flink with [Yarn]({{site.baseurl}}/ops/deployment/yarn_setup.html), [Mesos]({{site.baseurl}}/ops/deployment/mesos.html), or the [*active* Kubernetes integration]({{site.baseurl}}/ops/deployment/native_kubernetes.html), the hostnames and ports are automatically discovered.
-{% include generated/common_section.html %}
+ - `rest.address`, `rest.port`: These are used by the client to connect to Flink. Set this to the hostname where the master (JobManager) runs, or to the hostname of the (Kubernetes) service in front of the Flink Master's REST interface.
-## Full Reference
+ - The `jobmanager.rpc.address` (defaults to *"localhost"*) and `jobmanager.rpc.port` (defaults to *6123*) config entries are used by the TaskManager to connect to the JobManager/ResourceManager. Set this to the hostname where the master (JobManager) runs, or to the hostname of the (Kubernetes internal) service for the Flink master (JobManager). This option is ignored on [setups with high-availability]({{site.baseurl}}/ops/jobmanager_high_availability.html) where the leader election mec [...]
-### Core
+**Memory Sizes**
-{% include generated/core_configuration.html %}
+The default memory sizes support simple streaming/batch applications, but are too low to yield good performance for more complex applications.
-### Execution
+ - `jobmanager.heap.size`: Sets the size of the *Flink Master* (JobManager / ResourceManager / Dispatcher) JVM heap.
+ - `taskmanager.memory.process.size`: Total size of the TaskManager process, including everything. Flink will subtract some memory for the JVM's own memory requirements (metaspace and others), and divide and configure the rest automatically between its components (network, managed memory, JVM Heap, etc.).
-{% include generated/deployment_configuration.html %}
-{% include generated/savepoint_config_configuration.html %}
-{% include generated/execution_configuration.html %}
+These value are configured as memory sizes, for example *1536m* or *2g*.
+
+**Parallelism**
+
+ - `taskmanager.numberOfTaskSlots`: The number of slots that a TaskManager offers *(default: 1)*. Each slot can take one task or pipeline.
+ Having multiple slots in a TaskManager can help amortize certain constant overheads (of the JVM, application libraries, or network connections) across parallel tasks or pipelines.
+
+ Running more smaller TaskManagers with one slot each is a good starting point and leads to the best isolation between tasks. Dedicating the same resources to fewer larger TaskManagers with more slots can help to increase resource utilization, at the cost of weaker isolation between the tasks (more tasks share the same JVM).
+
+ - `parallelism.default`: The default parallelism used when no parallelism is specified anywhere *(default: 1)*.
+
+**Checkpointing**
+
+You can configure checkpointing directly in code within your Flink job or application. Putting these values here in the configuration defines them as defaults in case the application does not configure anything.
+
+ - `state.backend`: The state backend to use. This defines the data structure mechanism for taking snapshots. Common values are `filesystem` or `rocksdb`.
+ - `state.checkpoints.dir`: The directory to write checkpoints to. This takes a path URI like *s3://mybucket/flink-app/checkpoints* or *hdfs://namenode:port/flink/checkpoints*.
+ - `state.savepoints.dir`: The default directory for savepoints. Takes a path URI, similar to `state.checkpoints.dir`.
-### JobManager
+**Web UI**
-{% include generated/job_manager_configuration.html %}
+ - `web.submit.enable`: Enables uploading and starting jobs through the Flink UI *(true by default)*. Please note that even when this is disabled, session clusters still accept jobs through REST requests (HTTP calls). This flag only guards the feature to upload jobs in the UI.
+ - `web.upload.dir`: The directory where to store uploaded jobs. Only used when `web.submit.enable` is true.
-### Restart Strategies
+**Other**
-Configuration options to control Flink's restart behaviour in case of job failures.
+ - `io.tmp.dirs`: The directories where Flink puts local data, defaults to the system temp directory (`java.io.tmpdir` property). If a list of directories is configured, Flink will rotate files across the directories.
+
+ The data put in these directories include by default the files created by RocksDB, spilled intermediate results (batch algorithms), and cached jar files.
+
+ This data is NOT relied upon for persistence/recovery, but if this data gets deleted, it typically causes a heavyweight recovery operation. It is hence recommended to set this to a directory that is not automatically periodically purged.
+
+ Yarn, Mesos, and Kubernetes setups automatically configure this value to the local working directories by default.
+
+----
+----
+
+# Common Setup Options
+
+*Common options to configure your Flink application or cluster.*
+
+### Hosts and Ports
+
+Options to configure hostnames and ports for the different Flink components.
+
+The JobManager hostname and port are only relevant for standalone setups without high-availability.
+In that setup, the config values are used by the TaskManagers to find (and connect to) the JobManager.
+In all highly-available setups, the TaskManagers discover the JobManager via the High-Availability-Service (for example ZooKeeper).
+
+Setups using resource orchestration frameworks (K8s, Yarn, Mesos) typically use the framework's service discovery facilities.
+
+You do not need to configure any TaskManager hosts and ports, unless the setup requires the use of specific port ranges or specific network interfaces to bind to.
+
+{% include generated/common_host_port_section.html %}
+
+### Fault Tolerance
+
+These configuration options control Flink's restart behaviour in case of failures during the execution.
+By configuring these options in your `flink-conf.yaml`, you define the cluster's default restart strategy.
+
+The default restart strategy will only take effect if no job specific restart strategy has been configured via the `ExecutionConfig`.
{% include generated/restart_strategy_configuration.html %}
-#### Fixed Delay Restart Strategy
+**Fixed Delay Restart Strategy**
{% include generated/fixed_delay_restart_strategy_configuration.html %}
-#### Failure Rate Restart Strategy
+**Failure Rate Restart Strategy**
{% include generated/failure_rate_restart_strategy_configuration.html %}
-### TaskManager
+### Checkpoints and State Backends
-{% include generated/task_manager_configuration.html %}
+These options control the basic setup of state backends and checkpointing behavior.
-For *batch* jobs Flink allocates a fraction of 0.4 of the total flink memory (configured via taskmanager.memory.flink.size) for its managed memory. Managed memory helps Flink to run the batch operators efficiently. It prevents OutOfMemoryExceptions because Flink knows how much memory it can use to execute operations. If Flink runs out of managed memory, it utilizes disk space. Using managed memory, some operations can be performed directly on the raw data without having to deserialize th [...]
+The options are only relevant for jobs/applications executing in a continuous streaming fashion.
+Jobs/applications executing in a batch fashion do not use state backends and checkpoints, but different internal data structures that are optimized for batch processing.
-The default fraction for managed memory can be adjusted using the taskmanager.memory.managed.fraction parameter. An absolute value may be set using taskmanager.memory.managed.size (overrides the fraction parameter). If desired, the managed memory may be allocated outside the JVM heap. This may improve performance in setups with large memory sizes.
+{% include generated/common_state_backends_section.html %}
-{% include generated/task_manager_memory_configuration.html %}
+### High Availability
-### Distributed Coordination
+High-availability here refers to the ability of the master (JobManager) process to recover from failures.
-{% include generated/cluster_configuration.html %}
+The JobManager ensures consistency during recovery across TaskManagers. For the JobManager itself to recover consistently, an external service must store a minimal amount of recovery metadata (like "ID of last committed checkpoint"), as well as help to elect and lock which JobManager is the leader (to avoid split-brain situations).
-### Distributed Coordination (via Akka)
+{% include generated/common_high_availability_section.html %}
-{% include generated/akka_configuration.html %}
+**Options for high-availability setups with ZooKeeper**
-### REST
+{% include generated/common_high_availability_zk_section.html %}
-{% include generated/rest_configuration.html %}
+### Memory Configuration
-### Blob Server
+These configuration values control the way that TaskManagers and JobManagers use memory.
-{% include generated/blob_server_configuration.html %}
+Flink tries to shield users as much as possible from the complexity of configuring the JVM for data-intensive processing.
+In most cases, users should only need to set the values `taskmanager.memory.process.size` or `taskmanager.memory.flink.size` (depending on how the setup), and possibly adjusting the ratio of JVM heap and Managed Memory via `taskmanager.memory.managed.fraction`. The other options below can be used for performane tuning and fixing memory related errors.
-### Heartbeat Manager
+For a detailed explanation of how these options interact, see the [documentation on TaskManager memory configuration]({{site.baseurl}}/ops/memory_configuration.html).
-{% include generated/heartbeat_manager_configuration.html %}
+{% include generated/common_memory_section.html %}
-### SSL Settings
+### Miscellaneous Options
-{% include generated/security_configuration.html %}
+{% include generated/common_miscellaneous_section.html %}
-### Netty Shuffle Environment
+----
+----
-{% include generated/netty_shuffle_environment_configuration.html %}
+# Security
-### Network Communication (via Netty)
+Options for configuring Flink's security and secure interaction with external systems.
-These parameters allow for advanced tuning. The default values are sufficient when running concurrent high-throughput jobs on a large cluster.
+### SSL
-{% include generated/network_netty_configuration.html %}
+Flink's network connections can be secured via SSL. Please refer to the [SSL Setup Docs]({{site.baseurl}}/ops/security-ssl.html) for detailed setup guide and background.
-### Web Frontend
+{% include generated/security_ssl_section.html %}
-{% include generated/web_configuration.html %}
-### File Systems
+### Auth with External Systems
-{% include generated/file_system_configuration.html %}
+**ZooKeeper Authentication / Authorization**
-### Compiler/Optimizer
+These options are necessary when connecting to a secured ZooKeeper quorum.
-{% include generated/optimizer_configuration.html %}
+{% include generated/security_auth_zk_section.html %}
-### Runtime Algorithms
+**Kerberos-based Authentication / Authorization**
-{% include generated/algorithm_configuration.html %}
+Please refer to the [Flink and Kerberos Docs]({{site.baseurl}}/ops/security-kerberos.html) for a setup guide and a list of external system to which Flink can authenticate itself via Kerberos.
-### Resource Manager
+{% include generated/security_auth_kerberos_section.html %}
-The configuration keys in this section are independent of the used resource management framework (YARN, Mesos, Standalone, ...)
+----
+----
-{% include generated/resource_manager_configuration.html %}
+# Resource Orchestration Frameworks
+
+This section contains options related to integrating Flink with resource orchestration frameworks, like Kubernetes, Yarn, Mesos, etc.
-### Shuffle Service
+Note that is not always necessary to integrate Flink with the resource orchestration framework.
+For example, you can easily deploy Flink applications on Kubernetes without Flink knowing that it runs on Kubernetes (and without specifying any of the Kubernetes config options here.) See [this setup guide]({{site.baseurl}}/ops/deployment/kubernetes.html) for an example.
-{% include generated/shuffle_service_configuration.html %}
+The options in this section are necessary for setups where Flink itself actively requests and releases resources from the orchestrators.
### YARN
{% include generated/yarn_config_configuration.html %}
+### Kubernetes
+
+{% include generated/kubernetes_config_configuration.html %}
+
### Mesos
{% include generated/mesos_configuration.html %}
-#### Mesos TaskManager
+**Mesos TaskManager**
{% include generated/mesos_task_manager_configuration.html %}
-### Kubernetes
+----
+----
-{% include generated/kubernetes_config_configuration.html %}
+# State Backends
-### High Availability (HA)
+Please refer to the [State Backend Documentation]({{site.baseurl}}/ops/state/state_backends.html) for background on State Backends.
-{% include generated/high_availability_configuration.html %}
+### RocksDB State Backend
-#### ZooKeeper-based HA Mode
+These are the options commonly needed to configure the RocksDB state backend. See the [Advanced RocksDB Backend Section](#advanced-rocksdb-state-backends-options) for options necessary for advanced low level configurations and trouble-shooting.
-{% include generated/high_availability_zookeeper_configuration.html %}
+{% include generated/state_backend_rocksdb_section.html %}
-### ZooKeeper Security
+----
+----
-{% include generated/zoo_keeper_configuration.html %}
+# Metrics
-### Kerberos-based Security
+Please refer to the [metrics system documentation]({{site.baseurl}}/monitoring/metrics.html) for background on Flink's metrics infrastructure.
-{% include generated/kerberos_configuration.html %}
+{% include generated/metric_configuration.html %}
-### Environment
+### RocksDB Native Metrics
-{% include generated/environment_configuration.html %}
+Flink can report metrics from RocksDB's native code, for applications using the RocksDB state backend.
+The metrics here are scoped to the operators and then further broken down by column family; values are reported as unsigned longs.
-### Pipeline
+<div class="alert alert-warning">
+ <strong>Note:</strong> Enabling RocksDB's native metrics may cause degraded performance and should be set carefully.
+</div>
-{% include generated/pipeline_configuration.html %}
-{% include generated/stream_pipeline_configuration.html %}
+{% include generated/rocks_db_native_metric_configuration.html %}
-### Checkpointing
+----
+----
-{% include generated/checkpointing_configuration.html %}
-{% include generated/execution_checkpointing_configuration.html %}
+# History Server
-### RocksDB State Backend
+The history server keeps the information of completed jobs (graphs, runtimes, statistics). To enable it, you have to enable "job archiving" in the JobManager (`jobmanager.archive.fs.dir`).
-{% include generated/rocks_db_configuration.html %}
+See the [History Server Docs]({{site.baseurl}}/monitoring/historyserver.html) for details.
-### RocksDB Configurable Options
-Specific RocksDB configurable options, provided by Flink, to create a corresponding `ConfigurableOptionsFactory`.
-And the created one would be used as default `OptionsFactory` in `RocksDBStateBackend`
-unless user define a `OptionsFactory` and set via `RocksDBStateBackend.setOptions(optionsFactory)`
+{% include generated/history_server_configuration.html %}
-{% include generated/rocks_db_configurable_configuration.html %}
+----
+----
+
+# Experimental
+
+*Options for experimental features in Flink.*
### Queryable State
+*Queryable State* is an experimental features that gives lets you access Flink's internal state like a key/value store.
+See the [Queryable State Docs]({{site.baseurl}}/dev/stream/state/queryable_state.html) for details.
+
{% include generated/queryable_state_configuration.html %}
-### Metrics
+----
+----
-{% include generated/metric_configuration.html %}
-
-### RocksDB Native Metrics
-Certain RocksDB native metrics may be forwarded to Flink's metrics reporter.
-All native metrics are scoped to operators and then further broken down by column family; values are reported as unsigned longs.
+# Debugging & Expert Tuning
<div class="alert alert-warning">
- <strong>Note:</strong> Enabling native metrics may cause degraded performance and should be set carefully.
+ The options below here are meant for expert users and for fixing/debugging problems. Most setups should not need to configure these options.
</div>
-{% include generated/rocks_db_native_metric_configuration.html %}
+### Class Loading
-### History Server
+Flink dynamically loads the code for jobs submitted to a session cluster. In addition, Flink tries to hide many dependencies in the classpath from the application. This helps to reduce dependency conflicts between the application code and the dependencies in the classpath.
-You have to configure `jobmanager.archive.fs.dir` in order to archive terminated jobs and add it to the list of monitored directories via `historyserver.archive.fs.dir` if you want to display them via the HistoryServer's web frontend.
+Please refer to the [Debugging Classloading Docs]({{site.baseurl}}/monitoring/debugging_classloading.html) for details.
-- `jobmanager.archive.fs.dir`: Directory to upload information about terminated jobs to. You have to add this directory to the list of monitored directories of the history server via `historyserver.archive.fs.dir`.
+{% include generated/expert_class_loading_section.html %}
-{% include generated/history_server_configuration.html %}
+### Advanced State Backends Options
+
+{% include generated/expert_state_backends_section.html %}
+
+### Advanced RocksDB State Backends Options
+
+Advanced options to tune RocksDB and RocksDB checkpoints.
+
+{% include generated/expert_rocksdb_section.html %}
+
+**RocksDB Configurable Options**
-## Legacy
+These options give fine-grained control over the behavior and resoures of ColumnFamilies.
+With the introduction of `state.backend.rocksdb.memory.managed` and `state.backend.rocksdb.memory.fixed-per-slot` (Apache Flink 1.10), it should be only necessary to use the options here for advanced performance tuning. These options here can also be specified in the application program via `RocksDBStateBackend.setOptions(PptionsFactory)`.
+
+{% include generated/rocks_db_configurable_configuration.html %}
+
+### Advanced Fault Tolerance Options
+
+*These parameters can help with problems related to failover and to components erroneously considering each other as failed.*
+
+{% include generated/expert_fault_tolerance_section.html %}
+
+### Advanced Scheduling Options
+
+*These parameters can help with fine-tuning scheduling for specific situations.*
+
+{% include generated/expert_scheduling_section.html %}
+
+### Advanced High-availability Options
+
+{% include generated/expert_high_availability_section.html %}
+
+### Advanced High-availability ZooKeeper Options
+
+{% include generated/expert_high_availability_zk_section.html %}
+
+### Advanced SSL Security Options
+
+{% include generated/expert_security_ssl_section.html %}
+
+### Advanced Options for the REST endpoint and Client
+
+{% include generated/expert_rest_section.html %}
+
+### Advanced Options for Flink Web UI
+
+{% include generated/web_configuration.html %}
+
+### Full Flink Master Options
+
+**Master / JobManager**
+
+{% include generated/all_jobmanager_section.html %}
+
+**Blob Server**
+
+The Blob Server is a component in the Flink Master / JobManager. It is used for distribution of objects that are too large to be attached to a RPC message and that benefit from caching (like Jar files or large serialized code objects).
+
+{% include generated/blob_server_configuration.html %}
+
+**ResourceManager**
+
+These configuration keys control basic Resource Manager behavior, independent of the used resource orchestration management framework (YARN, Mesos, etc.)
+
+{% include generated/resource_manager_configuration.html %}
+
+### Full TaskManagerOptions
+
+{% include generated/all_taskmanager_section.html %}
+
+**Data Transport Network Stack**
+
+These options are for the network stack that handles the streaming and batch data exchanges between TaskManagers.
+
+{% include generated/all_taskmanager_network_section.html %}
+
+### RPC / Akka
+
+Flink uses Akka for RPC between components (JobManager/TaskManager/ResourceManager).
+Flink does not use Akka for data transport.
+
+{% include generated/akka_configuration.html %}
+
+----
+----
+
+# Startup Script Environment
+
+{% include generated/environment_configuration.html %}
+
+----
+----
+
+# Deprecated Options
+
+These options relate to parts of Flink that are not actively developed any more.
+These options may be removed in a future release.
+
+**DataSet API Optimizer**
+
+{% include generated/optimizer_configuration.html %}
+
+**DataSet API Runtime Algorithms**
+
+{% include generated/algorithm_configuration.html %}
+
+**DataSet File Sinks**
+
+{% include generated/deprecated_file_sinks_section.html %}
+
+----
+----
+
+# Backup
+
+#### Execution
+
+{% include generated/deployment_configuration.html %}
+{% include generated/savepoint_config_configuration.html %}
+{% include generated/execution_configuration.html %}
+
+#### Pipeline
+
+{% include generated/pipeline_configuration.html %}
+{% include generated/stream_pipeline_configuration.html %}
+
+#### Checkpointing
+
+{% include generated/execution_checkpointing_configuration.html %}
-- `mode`: Execution mode of Flink. Possible values are `legacy` and `new`. In order to start the legacy components, you have to specify `legacy` (DEFAULT: `new`).
+----
+----
-## Background
+# Background
### Configuring the Network Buffers
diff --git a/docs/ops/config.zh.md b/docs/ops/config.zh.md
index 8b7febc..6b70ec5 100644
--- a/docs/ops/config.zh.md
+++ b/docs/ops/config.zh.md
@@ -23,217 +23,411 @@ specific language governing permissions and limitations
under the License.
-->
-**For single-node setups Flink is ready to go out of the box and you don't need to change the default configuration to get started.**
+All configuration is done in `conf/flink-conf.yaml`, which is expected to be a flat collection of [YAML key value pairs](http://www.yaml.org/spec/1.2/spec.html) with format `key: value`.
+
+The configuration is parsed and evaluated when the Flink processes are started. Changes to the configuration file require restarting the relevant processes.
The out of the box configuration will use your default Java installation. You can manually set the environment variable `JAVA_HOME` or the configuration key `env.java.home` in `conf/flink-conf.yaml` if you want to manually override the Java runtime to use.
-This page lists the most common options that are typically needed to set up a well performing (distributed) installation. In addition a full list of all available configuration parameters is listed here.
+* This will be replaced by the TOC
+{:toc}
-All configuration is done in `conf/flink-conf.yaml`, which is expected to be a flat collection of [YAML key value pairs](http://www.yaml.org/spec/1.2/spec.html) with format `key: value`.
+# Basic Setup
-The system and run scripts parse the config at startup time. Changes to the configuration file require restarting the Flink JobManager and TaskManagers.
+The default configuration supports starting a single-node Flink session cluster without any changes.
+The options in this section are the ones most commonly needed for a basic distributed Flink setup.
-The configuration files for the TaskManagers can be different, Flink does not assume uniform machines in the cluster.
+**Hostnames / Ports**
-* This will be replaced by the TOC
-{:toc}
+These options are only necessary for *standalone* application- or session deployments ([simple standalone]({{site.baseurl}}/ops/deployment/cluster_setup.html) or [Kubernetes]({{site.baseurl}}/ops/deployment/kubernetes.html)).
-## Common Options
+If you use Flink with [Yarn]({{site.baseurl}}/ops/deployment/yarn_setup.html), [Mesos]({{site.baseurl}}/ops/deployment/mesos.html), or the [*active* Kubernetes integration]({{site.baseurl}}/ops/deployment/native_kubernetes.html), the hostnames and ports are automatically discovered.
-{% include generated/common_section.html %}
+ - `rest.address`, `rest.port`: These are used by the client to connect to Flink. Set this to the hostname where the master (JobManager) runs, or to the hostname of the (Kubernetes) service in front of the Flink Master's REST interface.
-## Full Reference
+ - The `jobmanager.rpc.address` (defaults to *"localhost"*) and `jobmanager.rpc.port` (defaults to *6123*) config entries are used by the TaskManager to connect to the JobManager/ResourceManager. Set this to the hostname where the master (JobManager) runs, or to the hostname of the (Kubernetes internal) service for the Flink master (JobManager). This option is ignored on [setups with high-availability]({{site.baseurl}}/ops/jobmanager_high_availability.html) where the leader election mec [...]
-### Core
+**Memory Sizes**
-{% include generated/core_configuration.html %}
+The default memory sizes support simple streaming/batch applications, but are too low to yield good performance for more complex applications.
-### Execution
+ - `jobmanager.heap.size`: Sets the size of the *Flink Master* (JobManager / ResourceManager / Dispatcher) JVM heap.
+ - `taskmanager.memory.process.size`: Total size of the TaskManager process, including everything. Flink will subtract some memory for the JVM's own memory requirements (metaspace and others), and divide and configure the rest automatically between its components (network, managed memory, JVM Heap, etc.).
-{% include generated/deployment_configuration.html %}
-{% include generated/savepoint_config_configuration.html %}
-{% include generated/execution_configuration.html %}
+These value are configured as memory sizes, for example *1536m* or *2g*.
+
+**Parallelism**
+
+ - `taskmanager.numberOfTaskSlots`: The number of slots that a TaskManager offers *(default: 1)*. Each slot can take one task or pipeline.
+ Having multiple slots in a TaskManager can help amortize certain constant overheads (of the JVM, application libraries, or network connections) across parallel tasks or pipelines.
+
+ Running more smaller TaskManagers with one slot each is a good starting point and leads to the best isolation between tasks. Dedicating the same resources to fewer larger TaskManagers with more slots can help to increase resource utilization, at the cost of weaker isolation between the tasks (more tasks share the same JVM).
+
+ - `parallelism.default`: The default parallelism used when no parallelism is specified anywhere *(default: 1)*.
+
+**Checkpointing**
+
+You can configure checkpointing directly in code within your Flink job or application. Putting these values here in the configuration defines them as defaults in case the application does not configure anything.
+
+ - `state.backend`: The state backend to use. This defines the data structure mechanism for taking snapshots. Common values are `filesystem` or `rocksdb`.
+ - `state.checkpoints.dir`: The directory to write checkpoints to. This takes a path URI like *s3://mybucket/flink-app/checkpoints* or *hdfs://namenode:port/flink/checkpoints*.
+ - `state.savepoints.dir`: The default directory for savepoints. Takes a path URI, similar to `state.checkpoints.dir`.
-### JobManager
+**Web UI**
-{% include generated/job_manager_configuration.html %}
+ - `web.submit.enable`: Enables uploading and starting jobs through the Flink UI *(true by default)*. Please note that even when this is disabled, session clusters still accept jobs through REST requests (HTTP calls). This flag only guards the feature to upload jobs in the UI.
+ - `web.upload.dir`: The directory where to store uploaded jobs. Only used when `web.submit.enable` is true.
-### Restart Strategies
+**Other**
-Configuration options to control Flink's restart behaviour in case of job failures.
+ - `io.tmp.dirs`: The directories where Flink puts local data, defaults to the system temp directory (`java.io.tmpdir` property). If a list of directories is configured, Flink will rotate files across the directories.
+
+ The data put in these directories include by default the files created by RocksDB, spilled intermediate results (batch algorithms), and cached jar files.
+
+ This data is NOT relied upon for persistence/recovery, but if this data gets deleted, it typically causes a heavyweight recovery operation. It is hence recommended to set this to a directory that is not automatically periodically purged.
+
+ Yarn, Mesos, and Kubernetes setups automatically configure this value to the local working directories by default.
+
+----
+----
+
+# Common Setup Options
+
+*Common options to configure your Flink application or cluster.*
+
+### Hosts and Ports
+
+Options to configure hostnames and ports for the different Flink components.
+
+The JobManager hostname and port are only relevant for standalone setups without high-availability.
+In that setup, the config values are used by the TaskManagers to find (and connect to) the JobManager.
+In all highly-available setups, the TaskManagers discover the JobManager via the High-Availability-Service (for example ZooKeeper).
+
+Setups using resource orchestration frameworks (K8s, Yarn, Mesos) typically use the framework's service discovery facilities.
+
+You do not need to configure any TaskManager hosts and ports, unless the setup requires the use of specific port ranges or specific network interfaces to bind to.
+
+{% include generated/common_host_port_section.html %}
+
+### Fault Tolerance
+
+These configuration options control Flink's restart behaviour in case of failures during the execution.
+By configuring these options in your `flink-conf.yaml`, you define the cluster's default restart strategy.
+
+The default restart strategy will only take effect if no job specific restart strategy has been configured via the `ExecutionConfig`.
{% include generated/restart_strategy_configuration.html %}
-#### Fixed Delay Restart Strategy
+**Fixed Delay Restart Strategy**
{% include generated/fixed_delay_restart_strategy_configuration.html %}
-#### Failure Rate Restart Strategy
+**Failure Rate Restart Strategy**
{% include generated/failure_rate_restart_strategy_configuration.html %}
-### TaskManager
+### Checkpoints and State Backends
-{% include generated/task_manager_configuration.html %}
+These options control the basic setup of state backends and checkpointing behavior.
-For *batch* jobs Flink allocates a fraction of 0.4 of the total flink memory (configured via taskmanager.memory.flink.size) for its managed memory. Managed memory helps Flink to run the batch operators efficiently. It prevents OutOfMemoryExceptions because Flink knows how much memory it can use to execute operations. If Flink runs out of managed memory, it utilizes disk space. Using managed memory, some operations can be performed directly on the raw data without having to deserialize th [...]
+The options are only relevant for jobs/applications executing in a continuous streaming fashion.
+Jobs/applications executing in a batch fashion do not use state backends and checkpoints, but different internal data structures that are optimized for batch processing.
-The default fraction for managed memory can be adjusted using the taskmanager.memory.managed.fraction parameter. An absolute value may be set using taskmanager.memory.managed.size (overrides the fraction parameter). If desired, the managed memory may be allocated outside the JVM heap. This may improve performance in setups with large memory sizes.
+{% include generated/common_state_backends_section.html %}
-{% include generated/task_manager_memory_configuration.html %}
+### High Availability
-### Distributed Coordination
+High-availability here refers to the ability of the master (JobManager) process to recover from failures.
-{% include generated/cluster_configuration.html %}
+The JobManager ensures consistency during recovery across TaskManagers. For the JobManager itself to recover consistently, an external service must store a minimal amount of recovery metadata (like "ID of last committed checkpoint"), as well as help to elect and lock which JobManager is the leader (to avoid split-brain situations).
-### Distributed Coordination (via Akka)
+{% include generated/common_high_availability_section.html %}
-{% include generated/akka_configuration.html %}
+**Options for high-availability setups with ZooKeeper**
-### REST
+{% include generated/common_high_availability_zk_section.html %}
-{% include generated/rest_configuration.html %}
+### Memory Configuration
-### Blob Server
+These configuration values control the way that TaskManagers and JobManagers use memory.
-{% include generated/blob_server_configuration.html %}
+Flink tries to shield users as much as possible from the complexity of configuring the JVM for data-intensive processing.
+In most cases, users should only need to set the values `taskmanager.memory.process.size` or `taskmanager.memory.flink.size` (depending on how the setup), and possibly adjusting the ratio of JVM heap and Managed Memory via `taskmanager.memory.managed.fraction`. The other options below can be used for performane tuning and fixing memory related errors.
-### Heartbeat Manager
+For a detailed explanation of how these options interact, see the [documentation on TaskManager memory configuration]({{site.baseurl}}/ops/memory_configuration.html).
-{% include generated/heartbeat_manager_configuration.html %}
+{% include generated/common_memory_section.html %}
-### SSL Settings
+### Miscellaneous Options
-{% include generated/security_configuration.html %}
+{% include generated/common_miscellaneous_section.html %}
-### Netty Shuffle Environment
+----
+----
-{% include generated/netty_shuffle_environment_configuration.html %}
+# Security
-### Network Communication (via Netty)
+Options for configuring Flink's security and secure interaction with external systems.
-These parameters allow for advanced tuning. The default values are sufficient when running concurrent high-throughput jobs on a large cluster.
+### SSL
-{% include generated/network_netty_configuration.html %}
+Flink's network connections can be secured via SSL. Please refer to the [SSL Setup Docs]({{site.baseurl}}/ops/security-ssl.html) for detailed setup guide and background.
-### Web Frontend
+{% include generated/security_ssl_section.html %}
-{% include generated/web_configuration.html %}
-### File Systems
+### Auth with External Systems
-{% include generated/file_system_configuration.html %}
+**ZooKeeper Authentication / Authorization**
-### Compiler/Optimizer
+These options are necessary when connecting to a secured ZooKeeper quorum.
-{% include generated/optimizer_configuration.html %}
+{% include generated/security_auth_zk_section.html %}
-### Runtime Algorithms
+**Kerberos-based Authentication / Authorization**
-{% include generated/algorithm_configuration.html %}
+Please refer to the [Flink and Kerberos Docs]({{site.baseurl}}/ops/security-kerberos.html) for a setup guide and a list of external system to which Flink can authenticate itself via Kerberos.
-### Resource Manager
+{% include generated/security_auth_kerberos_section.html %}
-The configuration keys in this section are independent of the used resource management framework (YARN, Mesos, Standalone, ...)
+----
+----
-{% include generated/resource_manager_configuration.html %}
+# Resource Orchestration Frameworks
+
+This section contains options related to integrating Flink with resource orchestration frameworks, like Kubernetes, Yarn, Mesos, etc.
-### Shuffle Service
+Note that is not always necessary to integrate Flink with the resource orchestration framework.
+For example, you can easily deploy Flink applications on Kubernetes without Flink knowing that it runs on Kubernetes (and without specifying any of the Kubernetes config options here.) See [this setup guide]({{site.baseurl}}/ops/deployment/kubernetes.html) for an example.
-{% include generated/shuffle_service_configuration.html %}
+The options in this section are necessary for setups where Flink itself actively requests and releases resources from the orchestrators.
### YARN
{% include generated/yarn_config_configuration.html %}
+### Kubernetes
+
+{% include generated/kubernetes_config_configuration.html %}
+
### Mesos
{% include generated/mesos_configuration.html %}
-#### Mesos TaskManager
+**Mesos TaskManager**
{% include generated/mesos_task_manager_configuration.html %}
-### Kubernetes
+----
+----
-{% include generated/kubernetes_config_configuration.html %}
+# State Backends
-### High Availability (HA)
+Please refer to the [State Backend Documentation]({{site.baseurl}}/ops/state/state_backends.html) for background on State Backends.
-{% include generated/high_availability_configuration.html %}
+### RocksDB State Backend
-#### ZooKeeper-based HA Mode
+These are the options commonly needed to configure the RocksDB state backend. See the [Advanced RocksDB Backend Section](#advanced-rocksdb-state-backends-options) for options necessary for advanced low level configurations and trouble-shooting.
-{% include generated/high_availability_zookeeper_configuration.html %}
+{% include generated/state_backend_rocksdb_section.html %}
-### ZooKeeper Security
+----
+----
-{% include generated/zoo_keeper_configuration.html %}
+# Metrics
-### Kerberos-based Security
+Please refer to the [metrics system documentation]({{site.baseurl}}/monitoring/metrics.html) for background on Flink's metrics infrastructure.
-{% include generated/kerberos_configuration.html %}
+{% include generated/metric_configuration.html %}
-### Environment
+### RocksDB Native Metrics
-{% include generated/environment_configuration.html %}
+Flink can report metrics from RocksDB's native code, for applications using the RocksDB state backend.
+The metrics here are scoped to the operators and then further broken down by column family; values are reported as unsigned longs.
-### Pipeline
+<div class="alert alert-warning">
+ <strong>Note:</strong> Enabling RocksDB's native metrics may cause degraded performance and should be set carefully.
+</div>
-{% include generated/pipeline_configuration.html %}
-{% include generated/stream_pipeline_configuration.html %}
+{% include generated/rocks_db_native_metric_configuration.html %}
-### Checkpointing
+----
+----
-{% include generated/checkpointing_configuration.html %}
-{% include generated/execution_checkpointing_configuration.html %}
+# History Server
-### RocksDB State Backend
+The history server keeps the information of completed jobs (graphs, runtimes, statistics). To enable it, you have to enable "job archiving" in the JobManager (`jobmanager.archive.fs.dir`).
-{% include generated/rocks_db_configuration.html %}
+See the [History Server Docs]({{site.baseurl}}/monitoring/historyserver.html) for details.
-### RocksDB Configurable Options
-Specific RocksDB configurable options, provided by Flink, to create a corresponding `ConfigurableOptionsFactory`.
-And the created one would be used as default `OptionsFactory` in `RocksDBStateBackend`
-unless user define a `OptionsFactory` and set via `RocksDBStateBackend.setOptions(optionsFactory)`
+{% include generated/history_server_configuration.html %}
-{% include generated/rocks_db_configurable_configuration.html %}
+----
+----
+
+# Experimental
+
+*Options for experimental features in Flink.*
### Queryable State
-{% include generated/queryable_state_configuration.html %}
+*Queryable State* is an experimental features that gives lets you access Flink's internal state like a key/value store.
+See the [Queryable State Docs]({{site.baseurl}}/dev/stream/state/queryable_state.html) for details.
-### Metrics
+{% include generated/queryable_state_configuration.html %}
-{% include generated/metric_configuration.html %}
+----
+----
-### RocksDB Native Metrics
-Certain RocksDB native metrics may be forwarded to Flink's metrics reporter.
-All native metrics are scoped to operators and then further broken down by column family; values are reported as unsigned longs.
+# Debugging & Expert Tuning
<div class="alert alert-warning">
- <strong>Note:</strong> Enabling native metrics may cause degraded performance and should be set carefully.
+ The options below here are meant for expert users and for fixing/debugging problems. Most setups should not need to configure these options.
</div>
-{% include generated/rocks_db_native_metric_configuration.html %}
+### Class Loading
-### History Server
+Flink dynamically loads the code for jobs submitted to a session cluster. In addition, Flink tries to hide many dependencies in the classpath from the application. This helps to reduce dependency conflicts between the application code and the dependencies in the classpath.
-You have to configure `jobmanager.archive.fs.dir` in order to archive terminated jobs and add it to the list of monitored directories via `historyserver.archive.fs.dir` if you want to display them via the HistoryServer's web frontend.
+Please refer to the [Debugging Classloading Docs]({{site.baseurl}}/monitoring/debugging_classloading.html) for details.
-- `jobmanager.archive.fs.dir`: Directory to upload information about terminated jobs to. You have to add this directory to the list of monitored directories of the history server via `historyserver.archive.fs.dir`.
+{% include generated/expert_class_loading_section.html %}
-{% include generated/history_server_configuration.html %}
+### Advanced State Backends Options
+
+{% include generated/expert_state_backends_section.html %}
+
+### Advanced RocksDB State Backends Options
+
+Advanced options to tune RocksDB and RocksDB checkpoints.
+
+{% include generated/expert_rocksdb_section.html %}
-### Python
+**RocksDB Configurable Options**
-{% include generated/python_configuration.html %}
+These options give fine-grained control over the behavior and resoures of ColumnFamilies.
+With the introduction of `state.backend.rocksdb.memory.managed` and `state.backend.rocksdb.memory.fixed-per-slot` (Apache Flink 1.10), it should be only necessary to use the options here for advanced performance tuning. These options here can also be specified in the application program via `RocksDBStateBackend.setOptions(PptionsFactory)`.
-## Legacy
+{% include generated/rocks_db_configurable_configuration.html %}
+
+### Advanced Fault Tolerance Options
+
+*These parameters can help with problems related to failover and to components erroneously considering each other as failed.*
+
+{% include generated/expert_fault_tolerance_section.html %}
+
+### Advanced Scheduling Options
+
+*These parameters can help with fine-tuning scheduling for specific situations.*
+
+{% include generated/expert_scheduling_section.html %}
+
+### Advanced High-availability Options
+
+{% include generated/expert_high_availability_section.html %}
+
+### Advanced High-availability ZooKeeper Options
+
+{% include generated/expert_high_availability_zk_section.html %}
+
+### Advanced SSL Security Options
+
+{% include generated/expert_security_ssl_section.html %}
+
+### Advanced Options for the REST endpoint and Client
+
+{% include generated/expert_rest_section.html %}
+
+### Advanced Options for Flink Web UI
+
+{% include generated/web_configuration.html %}
+
+### Full Flink Master Options
+
+**Master / JobManager**
+
+{% include generated/all_jobmanager_section.html %}
+
+**Blob Server**
+
+The Blob Server is a component in the Flink Master / JobManager. It is used for distribution of objects that are too large to be attached to a RPC message and that benefit from caching (like Jar files or large serialized code objects).
+
+{% include generated/blob_server_configuration.html %}
+
+**ResourceManager**
+
+These configuration keys control basic Resource Manager behavior, independent of the used resource orchestration management framework (YARN, Mesos, etc.)
+
+{% include generated/resource_manager_configuration.html %}
+
+### Full TaskManagerOptions
+
+{% include generated/all_taskmanager_section.html %}
+
+**Data Transport Network Stack**
+
+These options are for the network stack that handles the streaming and batch data exchanges between TaskManagers.
+
+{% include generated/all_taskmanager_network_section.html %}
+
+### RPC / Akka
+
+Flink uses Akka for RPC between components (JobManager/TaskManager/ResourceManager).
+Flink does not use Akka for data transport.
+
+{% include generated/akka_configuration.html %}
+
+----
+----
+
+# Startup Script Environment
+
+{% include generated/environment_configuration.html %}
+
+----
+----
+
+# Deprecated Options
+
+These options relate to parts of Flink that are not actively developed any more.
+These options may be removed in a future release.
+
+**DataSet API Optimizer**
+
+{% include generated/optimizer_configuration.html %}
+
+**DataSet API Runtime Algorithms**
+
+{% include generated/algorithm_configuration.html %}
+
+**DataSet File Sinks**
+
+{% include generated/deprecated_file_sinks_section.html %}
+
+----
+----
+
+# Backup
+
+#### Execution
+
+{% include generated/deployment_configuration.html %}
+{% include generated/savepoint_config_configuration.html %}
+{% include generated/execution_configuration.html %}
+
+#### Pipeline
+
+{% include generated/pipeline_configuration.html %}
+{% include generated/stream_pipeline_configuration.html %}
+
+#### Checkpointing
+
+{% include generated/execution_checkpointing_configuration.html %}
-- `mode`: Execution mode of Flink. Possible values are `legacy` and `new`. In order to start the legacy components, you have to specify `legacy` (DEFAULT: `new`).
+----
+----
-## Background
+# Background
### Configuring the Network Buffers
@@ -325,7 +519,7 @@ When starting a Flink application, users can supply the default number of slots
### Configuration Runtime Environment Variables
You have to set config with prefix `containerized.master.env.` and `containerized.taskmanager.env.` in order to set redefined environment variable in ApplicationMaster and TaskManager.
-- `containerized.master.env.`: Prefix for passing custom environment variables to Flink's master process.
+- `containerized.master.env.`: Prefix for passing custom environment variables to Flink's master process.
For example for passing LD_LIBRARY_PATH as an env variable to the AppMaster, set containerized.master.env.LD_LIBRARY_PATH: "/usr/lib/native"
in the flink-conf.yaml.
- `containerized.taskmanager.env.`: Similar to the above, this configuration prefix allows setting custom environment variables for the workers (TaskManagers).
diff --git a/flink-annotations/src/main/java/org/apache/flink/annotation/docs/Documentation.java b/flink-annotations/src/main/java/org/apache/flink/annotation/docs/Documentation.java
index e6c507f..e7dbce3 100644
--- a/flink-annotations/src/main/java/org/apache/flink/annotation/docs/Documentation.java
+++ b/flink-annotations/src/main/java/org/apache/flink/annotation/docs/Documentation.java
@@ -51,13 +51,6 @@ public final class Documentation {
@Retention(RetentionPolicy.RUNTIME)
@Internal
public @interface Section {
- int POSITION_MEMORY = 10;
- int POSITION_PARALLELISM_SLOTS = 20;
- int POSITION_FAULT_TOLERANCE = 30;
- int POSITION_HIGH_AVAILABILITY = 40;
- int POSITION_SECURITY = 50;
-
- String SECTION_COMMON = "common";
/**
* The sections in the config docs where this option should be included.
@@ -71,6 +64,43 @@ public final class Documentation {
}
/**
+ * Constants for section names.
+ */
+ public static final class Sections {
+
+ public static final String COMMON_HOST_PORT = "common_host_port";
+ public static final String COMMON_STATE_BACKENDS = "common_state_backends";
+ public static final String COMMON_HIGH_AVAILABILITY = "common_high_availability";
+ public static final String COMMON_HIGH_AVAILABILITY_ZOOKEEPER = "common_high_availability_zk";
+ public static final String COMMON_MEMORY = "common_memory";
+ public static final String COMMON_MISCELLANEOUS = "common_miscellaneous";
+
+ public static final String SECURITY_SSL = "security_ssl";
+ public static final String SECURITY_AUTH_KERBEROS = "security_auth_kerberos";
+ public static final String SECURITY_AUTH_ZOOKEEPER = "security_auth_zk";
+
+ public static final String STATE_BACKEND_ROCKSDB = "state_backend_rocksdb";
+
+ public static final String EXPERT_CLASS_LOADING = "expert_class_loading";
+ public static final String EXPERT_SCHEDULING = "expert_scheduling";
+ public static final String EXPERT_FAULT_TOLERANCE = "expert_fault_tolerance";
+ public static final String EXPERT_STATE_BACKENDS = "expert_state_backends";
+ public static final String EXPERT_REST = "expert_rest";
+ public static final String EXPERT_HIGH_AVAILABILITY = "expert_high_availability";
+ public static final String EXPERT_ZOOKEEPER_HIGH_AVAILABILITY = "expert_high_availability_zk";
+ public static final String EXPERT_SECURITY_SSL = "expert_security_ssl";
+ public static final String EXPERT_ROCKSDB = "expert_rocksdb";
+
+ public static final String ALL_JOB_MANAGER = "all_jobmanager";
+ public static final String ALL_TASK_MANAGER = "all_taskmanager";
+ public static final String ALL_TASK_MANAGER_NETWORK = "all_taskmanager_network";
+
+ public static final String DEPRECATED_FILE_SINKS = "deprecated_file_sinks";
+
+ private Sections() {}
+ }
+
+ /**
* Annotation used on table config options for adding meta data labels.
*
* <p>The {@link TableOption#execMode()} argument indicates the execution mode the config works for
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/CheckpointingOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/CheckpointingOptions.java
index 566bac6..4e18eea 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/CheckpointingOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/CheckpointingOptions.java
@@ -32,14 +32,15 @@ public class CheckpointingOptions {
/** The state backend to be used to store and checkpoint state. */
@Documentation.Section(
- value = {Documentation.Section.SECTION_COMMON},
- position = Documentation.Section.POSITION_FAULT_TOLERANCE)
+ value = Documentation.Sections.COMMON_STATE_BACKENDS,
+ position = 1)
public static final ConfigOption<String> STATE_BACKEND = ConfigOptions
.key("state.backend")
.noDefaultValue()
.withDescription("The state backend to be used to store and checkpoint state.");
/** The maximum number of completed checkpoints to retain.*/
+ @Documentation.Section(Documentation.Sections.COMMON_STATE_BACKENDS)
public static final ConfigOption<Integer> MAX_RETAINED_CHECKPOINTS = ConfigOptions
.key("state.checkpoints.num-retained")
.defaultValue(1)
@@ -50,6 +51,7 @@ public class CheckpointingOptions {
*
* <p>Some state backends may not support asynchronous snapshots, or only support
* asynchronous snapshots, and ignore this option. */
+ @Documentation.Section(Documentation.Sections.EXPERT_STATE_BACKENDS)
public static final ConfigOption<Boolean> ASYNC_SNAPSHOTS = ConfigOptions
.key("state.backend.async")
.defaultValue(true)
@@ -63,6 +65,7 @@ public class CheckpointingOptions {
*
* <p>Some state backends may not support incremental checkpoints and ignore
* this option.*/
+ @Documentation.Section(Documentation.Sections.COMMON_STATE_BACKENDS)
public static final ConfigOption<Boolean> INCREMENTAL_CHECKPOINTS = ConfigOptions
.key("state.backend.incremental")
.defaultValue(false)
@@ -78,6 +81,7 @@ public class CheckpointingOptions {
* Currently, MemoryStateBackend does not support local recovery and ignore
* this option.
*/
+ @Documentation.Section(Documentation.Sections.COMMON_STATE_BACKENDS)
public static final ConfigOption<Boolean> LOCAL_RECOVERY = ConfigOptions
.key("state.backend.local-recovery")
.defaultValue(false)
@@ -92,6 +96,7 @@ public class CheckpointingOptions {
* Currently, MemoryStateBackend does not support local recovery and ignore
* this option.
*/
+ @Documentation.Section(Documentation.Sections.COMMON_STATE_BACKENDS)
public static final ConfigOption<String> LOCAL_RECOVERY_TASK_MANAGER_STATE_ROOT_DIRS = ConfigOptions
.key("taskmanager.state.local.root-dirs")
.noDefaultValue()
@@ -106,8 +111,8 @@ public class CheckpointingOptions {
/** The default directory for savepoints. Used by the state backends that write
* savepoints to file systems (MemoryStateBackend, FsStateBackend, RocksDBStateBackend). */
@Documentation.Section(
- value = {Documentation.Section.SECTION_COMMON},
- position = Documentation.Section.POSITION_FAULT_TOLERANCE)
+ value = Documentation.Sections.COMMON_STATE_BACKENDS,
+ position = 3)
public static final ConfigOption<String> SAVEPOINT_DIRECTORY = ConfigOptions
.key("state.savepoints.dir")
.noDefaultValue()
@@ -118,8 +123,8 @@ public class CheckpointingOptions {
/** The default directory used for storing the data files and meta data of checkpoints in a Flink supported filesystem.
* The storage path must be accessible from all participating processes/nodes(i.e. all TaskManagers and JobManagers).*/
@Documentation.Section(
- value = {Documentation.Section.SECTION_COMMON},
- position = Documentation.Section.POSITION_FAULT_TOLERANCE)
+ value = Documentation.Sections.COMMON_STATE_BACKENDS,
+ position = 2)
public static final ConfigOption<String> CHECKPOINTS_DIRECTORY = ConfigOptions
.key("state.checkpoints.dir")
.noDefaultValue()
@@ -130,6 +135,7 @@ public class CheckpointingOptions {
/** The minimum size of state data files. All state chunks smaller than that
* are stored inline in the root checkpoint metadata file. */
+ @Documentation.Section(Documentation.Sections.EXPERT_STATE_BACKENDS)
public static final ConfigOption<Integer> FS_SMALL_FILE_THRESHOLD = ConfigOptions
.key("state.backend.fs.memory-threshold")
.defaultValue(1024)
@@ -139,6 +145,7 @@ public class CheckpointingOptions {
/**
* The default size of the write buffer for the checkpoint streams that write to file systems.
*/
+ @Documentation.Section(Documentation.Sections.EXPERT_STATE_BACKENDS)
public static final ConfigOption<Integer> FS_WRITE_BUFFER_SIZE = ConfigOptions
.key("state.backend.fs.write-buffer-size")
.defaultValue(4 * 1024)
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/ClusterOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/ClusterOptions.java
index b1462a9..de5479f 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/ClusterOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/ClusterOptions.java
@@ -19,6 +19,7 @@
package org.apache.flink.configuration;
import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.annotation.docs.Documentation;
import org.apache.flink.configuration.description.Description;
import static org.apache.flink.configuration.description.TextElement.code;
@@ -29,31 +30,37 @@ import static org.apache.flink.configuration.description.TextElement.code;
@PublicEvolving
public class ClusterOptions {
+ @Documentation.Section(Documentation.Sections.EXPERT_FAULT_TOLERANCE)
public static final ConfigOption<Long> INITIAL_REGISTRATION_TIMEOUT = ConfigOptions
.key("cluster.registration.initial-timeout")
.defaultValue(100L)
.withDescription("Initial registration timeout between cluster components in milliseconds.");
+ @Documentation.Section(Documentation.Sections.EXPERT_FAULT_TOLERANCE)
public static final ConfigOption<Long> MAX_REGISTRATION_TIMEOUT = ConfigOptions
.key("cluster.registration.max-timeout")
.defaultValue(30000L)
.withDescription("Maximum registration timeout between cluster components in milliseconds.");
+ @Documentation.Section(Documentation.Sections.EXPERT_FAULT_TOLERANCE)
public static final ConfigOption<Long> ERROR_REGISTRATION_DELAY = ConfigOptions
.key("cluster.registration.error-delay")
.defaultValue(10000L)
.withDescription("The pause made after an registration attempt caused an exception (other than timeout) in milliseconds.");
+ @Documentation.Section(Documentation.Sections.EXPERT_FAULT_TOLERANCE)
public static final ConfigOption<Long> REFUSED_REGISTRATION_DELAY = ConfigOptions
.key("cluster.registration.refused-registration-delay")
.defaultValue(30000L)
.withDescription("The pause made after the registration attempt was refused in milliseconds.");
+ @Documentation.Section(Documentation.Sections.EXPERT_FAULT_TOLERANCE)
public static final ConfigOption<Long> CLUSTER_SERVICES_SHUTDOWN_TIMEOUT = ConfigOptions
.key("cluster.services.shutdown-timeout")
.defaultValue(30000L)
.withDescription("The shutdown timeout for cluster services like executors in milliseconds.");
+ @Documentation.Section(Documentation.Sections.EXPERT_SCHEDULING)
public static final ConfigOption<Boolean> EVENLY_SPREAD_OUT_SLOTS_STRATEGY = ConfigOptions
.key("cluster.evenly-spread-out-slots")
.defaultValue(false)
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java
index 2449b4e..484f604 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java
@@ -34,8 +34,7 @@ import static org.apache.flink.configuration.ConfigOptions.key;
*/
@PublicEvolving
@ConfigGroups(groups = {
- @ConfigGroup(name = "Environment", keyPrefix = "env"),
- @ConfigGroup(name = "FileSystem", keyPrefix = "fs")
+ @ConfigGroup(name = "Environment", keyPrefix = "env")
})
public class CoreOptions {
@@ -54,6 +53,7 @@ public class CoreOptions {
*
* <p>Exceptions to the rules are defined via {@link #ALWAYS_PARENT_FIRST_LOADER_PATTERNS}.
*/
+ @Documentation.Section(Documentation.Sections.EXPERT_CLASS_LOADING)
public static final ConfigOption<String> CLASSLOADER_RESOLVE_ORDER = ConfigOptions
.key("classloader.resolve-order")
.defaultValue("child-first")
@@ -92,6 +92,7 @@ public class CoreOptions {
* log bindings.</li>
* </ul>
*/
+ @Documentation.Section(Documentation.Sections.EXPERT_CLASS_LOADING)
public static final ConfigOption<String> ALWAYS_PARENT_FIRST_LOADER_PATTERNS = ConfigOptions
.key("classloader.parent-first-patterns.default")
.defaultValue("java.;scala.;org.apache.flink.;com.esotericsoftware.kryo;org.apache.hadoop.;javax.annotation.;org.slf4j;org.apache.log4j;org.apache.logging;org.apache.commons.logging;ch.qos.logback;org.xml;javax.xml;org.apache.xerces;org.w3c")
@@ -101,6 +102,7 @@ public class CoreOptions {
" the fully qualified class name. This setting should generally not be modified. To add another pattern we" +
" recommend to use \"classloader.parent-first-patterns.additional\" instead.");
+ @Documentation.Section(Documentation.Sections.EXPERT_CLASS_LOADING)
public static final ConfigOption<String> ALWAYS_PARENT_FIRST_LOADER_PATTERNS_ADDITIONAL = ConfigOptions
.key("classloader.parent-first-patterns.additional")
.defaultValue("")
@@ -240,6 +242,7 @@ public class CoreOptions {
* ",", "|", or the system's {@link java.io.File#pathSeparator}.
*/
@Documentation.OverrideDefault("'LOCAL_DIRS' on Yarn. '_FLINK_TMP_DIR' on Mesos. System.getProperty(\"java.io.tmpdir\") in standalone.")
+ @Documentation.Section(Documentation.Sections.COMMON_MISCELLANEOUS)
public static final ConfigOption<String> TMP_DIRS =
key("io.tmp.dirs")
.defaultValue(System.getProperty("java.io.tmpdir"))
@@ -250,9 +253,6 @@ public class CoreOptions {
// program
// ------------------------------------------------------------------------
- @Documentation.Section(
- value = {Documentation.Section.SECTION_COMMON},
- position = Documentation.Section.POSITION_PARALLELISM_SLOTS)
public static final ConfigOption<Integer> DEFAULT_PARALLELISM = ConfigOptions
.key("parallelism.default")
.defaultValue(1)
@@ -265,6 +265,7 @@ public class CoreOptions {
/**
* The default filesystem scheme, used for paths that do not declare a scheme explicitly.
*/
+ @Documentation.Section(Documentation.Sections.COMMON_MISCELLANEOUS)
public static final ConfigOption<String> DEFAULT_FILESYSTEM_SCHEME = ConfigOptions
.key("fs.default-scheme")
.noDefaultValue()
@@ -274,6 +275,7 @@ public class CoreOptions {
/**
* Specifies whether file output writers should overwrite existing files by default.
*/
+ @Documentation.Section(Documentation.Sections.DEPRECATED_FILE_SINKS)
public static final ConfigOption<Boolean> FILESYTEM_DEFAULT_OVERRIDE =
key("fs.overwrite-files")
.defaultValue(false)
@@ -283,6 +285,7 @@ public class CoreOptions {
/**
* Specifies whether the file systems should always create a directory for the output, even with a parallelism of one.
*/
+ @Documentation.Section(Documentation.Sections.DEPRECATED_FILE_SINKS)
public static final ConfigOption<Boolean> FILESYSTEM_OUTPUT_ALWAYS_CREATE_DIRECTORY =
key("fs.output.always-create-directory")
.defaultValue(false)
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/HeartbeatManagerOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/HeartbeatManagerOptions.java
index 0fe4341..b0ccf0d 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/HeartbeatManagerOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/HeartbeatManagerOptions.java
@@ -19,6 +19,7 @@
package org.apache.flink.configuration;
import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.annotation.docs.Documentation;
import static org.apache.flink.configuration.ConfigOptions.key;
@@ -29,12 +30,14 @@ import static org.apache.flink.configuration.ConfigOptions.key;
public class HeartbeatManagerOptions {
/** Time interval for requesting heartbeat from sender side. */
+ @Documentation.Section(Documentation.Sections.EXPERT_FAULT_TOLERANCE)
public static final ConfigOption<Long> HEARTBEAT_INTERVAL =
key("heartbeat.interval")
.defaultValue(10000L)
.withDescription("Time interval for requesting heartbeat from sender side.");
/** Timeout for requesting and receiving heartbeat for both sender and receiver sides. */
+ @Documentation.Section(Documentation.Sections.EXPERT_FAULT_TOLERANCE)
public static final ConfigOption<Long> HEARTBEAT_TIMEOUT =
key("heartbeat.timeout")
.defaultValue(50000L)
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/HighAvailabilityOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/HighAvailabilityOptions.java
index 0087878..8930d00 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/HighAvailabilityOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/HighAvailabilityOptions.java
@@ -19,8 +19,6 @@
package org.apache.flink.configuration;
import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.annotation.docs.ConfigGroup;
-import org.apache.flink.annotation.docs.ConfigGroups;
import org.apache.flink.annotation.docs.Documentation;
import org.apache.flink.configuration.description.Description;
@@ -29,10 +27,6 @@ import static org.apache.flink.configuration.ConfigOptions.key;
/**
* The set of configuration options relating to high-availability settings.
*/
-@PublicEvolving
-@ConfigGroups(groups = {
- @ConfigGroup(name = "HighAvailabilityZookeeper", keyPrefix = "high-availability.zookeeper")
-})
public class HighAvailabilityOptions {
// ------------------------------------------------------------------------
@@ -45,9 +39,7 @@ public class HighAvailabilityOptions {
* To enable high-availability, set this mode to "ZOOKEEPER".
* Can also be set to FQN of HighAvailability factory class.
*/
- @Documentation.Section(
- value = {Documentation.Section.SECTION_COMMON},
- position = Documentation.Section.POSITION_HIGH_AVAILABILITY)
+ @Documentation.Section(Documentation.Sections.COMMON_HIGH_AVAILABILITY)
public static final ConfigOption<String> HA_MODE =
key("high-availability")
.defaultValue("NONE")
@@ -59,6 +51,7 @@ public class HighAvailabilityOptions {
* The ID of the Flink cluster, used to separate multiple Flink clusters
* Needs to be set for standalone clusters, is automatically inferred in YARN and Mesos.
*/
+ @Documentation.Section(Documentation.Sections.COMMON_HIGH_AVAILABILITY)
public static final ConfigOption<String> HA_CLUSTER_ID =
key("high-availability.cluster-id")
.defaultValue("/default")
@@ -69,9 +62,7 @@ public class HighAvailabilityOptions {
/**
* File system path (URI) where Flink persists metadata in high-availability setups.
*/
- @Documentation.Section(
- value = {Documentation.Section.SECTION_COMMON},
- position = Documentation.Section.POSITION_HIGH_AVAILABILITY)
+ @Documentation.Section(Documentation.Sections.COMMON_HIGH_AVAILABILITY)
public static final ConfigOption<String> HA_STORAGE_PATH =
key("high-availability.storageDir")
.noDefaultValue()
@@ -85,6 +76,7 @@ public class HighAvailabilityOptions {
/**
* Optional port (range) used by the job manager in high-availability mode.
*/
+ @Documentation.Section(Documentation.Sections.EXPERT_HIGH_AVAILABILITY)
public static final ConfigOption<String> HA_JOB_MANAGER_PORT_RANGE =
key("high-availability.jobmanager.port")
.defaultValue("0")
@@ -98,6 +90,7 @@ public class HighAvailabilityOptions {
/**
* The ZooKeeper quorum to use, when running Flink in a high-availability mode with ZooKeeper.
*/
+ @Documentation.Section(Documentation.Sections.COMMON_HIGH_AVAILABILITY_ZOOKEEPER)
public static final ConfigOption<String> HA_ZOOKEEPER_QUORUM =
key("high-availability.zookeeper.quorum")
.noDefaultValue()
@@ -107,12 +100,14 @@ public class HighAvailabilityOptions {
/**
* The root path under which Flink stores its entries in ZooKeeper.
*/
+ @Documentation.Section(Documentation.Sections.COMMON_HIGH_AVAILABILITY_ZOOKEEPER)
public static final ConfigOption<String> HA_ZOOKEEPER_ROOT =
key("high-availability.zookeeper.path.root")
.defaultValue("/flink")
.withDeprecatedKeys("recovery.zookeeper.path.root")
.withDescription("The root path under which Flink stores its entries in ZooKeeper.");
+ @Documentation.Section(Documentation.Sections.EXPERT_ZOOKEEPER_HIGH_AVAILABILITY)
public static final ConfigOption<String> HA_ZOOKEEPER_LATCH_PATH =
key("high-availability.zookeeper.path.latch")
.defaultValue("/leaderlatch")
@@ -120,12 +115,14 @@ public class HighAvailabilityOptions {
.withDescription("Defines the znode of the leader latch which is used to elect the leader.");
/** ZooKeeper root path (ZNode) for job graphs. */
+ @Documentation.Section(Documentation.Sections.EXPERT_ZOOKEEPER_HIGH_AVAILABILITY)
public static final ConfigOption<String> HA_ZOOKEEPER_JOBGRAPHS_PATH =
key("high-availability.zookeeper.path.jobgraphs")
.defaultValue("/jobgraphs")
.withDeprecatedKeys("recovery.zookeeper.path.jobgraphs")
.withDescription("ZooKeeper root path (ZNode) for job graphs");
+ @Documentation.Section(Documentation.Sections.EXPERT_ZOOKEEPER_HIGH_AVAILABILITY)
public static final ConfigOption<String> HA_ZOOKEEPER_LEADER_PATH =
key("high-availability.zookeeper.path.leader")
.defaultValue("/leader")
@@ -134,6 +131,7 @@ public class HighAvailabilityOptions {
" leader session ID.");
/** ZooKeeper root path (ZNode) for completed checkpoints. */
+ @Documentation.Section(Documentation.Sections.EXPERT_ZOOKEEPER_HIGH_AVAILABILITY)
public static final ConfigOption<String> HA_ZOOKEEPER_CHECKPOINTS_PATH =
key("high-availability.zookeeper.path.checkpoints")
.defaultValue("/checkpoints")
@@ -141,6 +139,7 @@ public class HighAvailabilityOptions {
.withDescription("ZooKeeper root path (ZNode) for completed checkpoints.");
/** ZooKeeper root path (ZNode) for checkpoint counters. */
+ @Documentation.Section(Documentation.Sections.EXPERT_ZOOKEEPER_HIGH_AVAILABILITY)
public static final ConfigOption<String> HA_ZOOKEEPER_CHECKPOINT_COUNTER_PATH =
key("high-availability.zookeeper.path.checkpoint-counter")
.defaultValue("/checkpoint-counter")
@@ -149,6 +148,7 @@ public class HighAvailabilityOptions {
/** ZooKeeper root path (ZNode) for Mesos workers. */
@PublicEvolving
+ @Documentation.Section(Documentation.Sections.EXPERT_ZOOKEEPER_HIGH_AVAILABILITY)
public static final ConfigOption<String> HA_ZOOKEEPER_MESOS_WORKERS_PATH =
key("high-availability.zookeeper.path.mesos-workers")
.defaultValue("/mesos-workers")
@@ -161,34 +161,40 @@ public class HighAvailabilityOptions {
// ZooKeeper Client Settings
// ------------------------------------------------------------------------
+ @Documentation.Section(Documentation.Sections.EXPERT_ZOOKEEPER_HIGH_AVAILABILITY)
public static final ConfigOption<Integer> ZOOKEEPER_SESSION_TIMEOUT =
key("high-availability.zookeeper.client.session-timeout")
.defaultValue(60000)
.withDeprecatedKeys("recovery.zookeeper.client.session-timeout")
.withDescription("Defines the session timeout for the ZooKeeper session in ms.");
+ @Documentation.Section(Documentation.Sections.EXPERT_ZOOKEEPER_HIGH_AVAILABILITY)
public static final ConfigOption<Integer> ZOOKEEPER_CONNECTION_TIMEOUT =
key("high-availability.zookeeper.client.connection-timeout")
.defaultValue(15000)
.withDeprecatedKeys("recovery.zookeeper.client.connection-timeout")
.withDescription("Defines the connection timeout for ZooKeeper in ms.");
+ @Documentation.Section(Documentation.Sections.EXPERT_ZOOKEEPER_HIGH_AVAILABILITY)
public static final ConfigOption<Integer> ZOOKEEPER_RETRY_WAIT =
key("high-availability.zookeeper.client.retry-wait")
.defaultValue(5000)
.withDeprecatedKeys("recovery.zookeeper.client.retry-wait")
.withDescription("Defines the pause between consecutive retries in ms.");
+ @Documentation.Section(Documentation.Sections.EXPERT_ZOOKEEPER_HIGH_AVAILABILITY)
public static final ConfigOption<Integer> ZOOKEEPER_MAX_RETRY_ATTEMPTS =
key("high-availability.zookeeper.client.max-retry-attempts")
.defaultValue(3)
.withDeprecatedKeys("recovery.zookeeper.client.max-retry-attempts")
.withDescription("Defines the number of connection retries before the client gives up.");
+ @Documentation.Section(Documentation.Sections.EXPERT_ZOOKEEPER_HIGH_AVAILABILITY)
public static final ConfigOption<String> ZOOKEEPER_RUNNING_JOB_REGISTRY_PATH =
key("high-availability.zookeeper.path.running-registry")
.defaultValue("/running_job_registry/");
+ @Documentation.Section(Documentation.Sections.EXPERT_ZOOKEEPER_HIGH_AVAILABILITY)
public static final ConfigOption<String> ZOOKEEPER_CLIENT_ACL =
key("high-availability.zookeeper.client.acl")
.defaultValue("open")
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
index 47755e2..efeb340 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
@@ -42,6 +42,7 @@ public class JobManagerOptions {
* leader-election service (like ZooKeeper) is used to elect and discover the JobManager
* leader from potentially multiple standby JobManagers.
*/
+ @Documentation.Section({Documentation.Sections.COMMON_HOST_PORT, Documentation.Sections.ALL_JOB_MANAGER})
public static final ConfigOption<String> ADDRESS =
key("jobmanager.rpc.address")
.noDefaultValue()
@@ -64,6 +65,7 @@ public class JobManagerOptions {
* leader-election service (like ZooKeeper) is used to elect and discover the JobManager
* leader from potentially multiple standby JobManagers.
*/
+ @Documentation.Section({Documentation.Sections.COMMON_HOST_PORT, Documentation.Sections.ALL_JOB_MANAGER})
public static final ConfigOption<Integer> PORT =
key("jobmanager.rpc.port")
.defaultValue(6123)
@@ -79,9 +81,7 @@ public class JobManagerOptions {
/**
* JVM heap size for the JobManager with memory size.
*/
- @Documentation.Section(
- value = {Documentation.Section.SECTION_COMMON},
- position = Documentation.Section.POSITION_MEMORY)
+ @Documentation.Section(Documentation.Sections.ALL_JOB_MANAGER)
public static final ConfigOption<String> JOB_MANAGER_HEAP_MEMORY =
key("jobmanager.heap.size")
.defaultValue("1024m")
@@ -100,6 +100,7 @@ public class JobManagerOptions {
/**
* The maximum number of prior execution attempts kept in history.
*/
+ @Documentation.Section(Documentation.Sections.ALL_JOB_MANAGER)
public static final ConfigOption<Integer> MAX_ATTEMPTS_HISTORY_SIZE =
key("jobmanager.execution.attempts-history-size")
.defaultValue(16)
@@ -114,6 +115,8 @@ public class JobManagerOptions {
* failover strategy would also restart failed tasks individually.
* The new "region" strategy supersedes "individual" strategy and should always work.
*/
+ @Documentation.Section({Documentation.Sections.ALL_JOB_MANAGER, Documentation.Sections.EXPERT_FAULT_TOLERANCE})
+ @Documentation.OverrideDefault("region")
public static final ConfigOption<String> EXECUTION_FAILOVER_STRATEGY =
key("jobmanager.execution.failover-strategy")
.defaultValue("full")
@@ -132,6 +135,7 @@ public class JobManagerOptions {
/**
* The location where the JobManager stores the archives of completed jobs.
*/
+ @Documentation.Section(Documentation.Sections.ALL_JOB_MANAGER)
public static final ConfigOption<String> ARCHIVE_DIR =
key("jobmanager.archive.fs.dir")
.noDefaultValue()
@@ -141,6 +145,7 @@ public class JobManagerOptions {
* The job store cache size in bytes which is used to keep completed
* jobs in memory.
*/
+ @Documentation.Section(Documentation.Sections.ALL_JOB_MANAGER)
public static final ConfigOption<Long> JOB_STORE_CACHE_SIZE =
key("jobstore.cache-size")
.defaultValue(50L * 1024L * 1024L)
@@ -149,6 +154,7 @@ public class JobManagerOptions {
/**
* The time in seconds after which a completed job expires and is purged from the job store.
*/
+ @Documentation.Section(Documentation.Sections.ALL_JOB_MANAGER)
public static final ConfigOption<Long> JOB_STORE_EXPIRATION_TIME =
key("jobstore.expiration-time")
.defaultValue(60L * 60L)
@@ -157,6 +163,7 @@ public class JobManagerOptions {
/**
* The max number of completed jobs that can be kept in the job store.
*/
+ @Documentation.Section(Documentation.Sections.ALL_JOB_MANAGER)
public static final ConfigOption<Integer> JOB_STORE_MAX_CAPACITY =
key("jobstore.max-capacity")
.defaultValue(Integer.MAX_VALUE)
@@ -165,6 +172,7 @@ public class JobManagerOptions {
/**
* The timeout in milliseconds for requesting a slot from Slot Pool.
*/
+ @Documentation.Section(Documentation.Sections.EXPERT_SCHEDULING)
public static final ConfigOption<Long> SLOT_REQUEST_TIMEOUT =
key("slot.request.timeout")
.defaultValue(5L * 60L * 1000L)
@@ -173,6 +181,7 @@ public class JobManagerOptions {
/**
* The timeout in milliseconds for a idle slot in Slot Pool.
*/
+ @Documentation.Section(Documentation.Sections.EXPERT_SCHEDULING)
public static final ConfigOption<Long> SLOT_IDLE_TIMEOUT =
key("slot.idle.timeout")
// default matches heartbeat.timeout so that sticky allocation is not lost on timeouts for local recovery
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/NettyShuffleEnvironmentOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/NettyShuffleEnvironmentOptions.java
index ca4c131..8670183 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/NettyShuffleEnvironmentOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/NettyShuffleEnvironmentOptions.java
@@ -19,8 +19,6 @@
package org.apache.flink.configuration;
import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.annotation.docs.ConfigGroup;
-import org.apache.flink.annotation.docs.ConfigGroups;
import org.apache.flink.annotation.docs.Documentation;
import static org.apache.flink.configuration.ConfigOptions.key;
@@ -29,7 +27,6 @@ import static org.apache.flink.configuration.ConfigOptions.key;
* The set of configuration options relating to network stack.
*/
@PublicEvolving
-@ConfigGroups(groups = @ConfigGroup(name = "NetworkNetty", keyPrefix = "taskmanager.network.netty"))
public class NettyShuffleEnvironmentOptions {
// ------------------------------------------------------------------------
@@ -40,6 +37,7 @@ public class NettyShuffleEnvironmentOptions {
* The default network port the task manager expects to receive transfer envelopes on. The {@code 0} means that
* the TaskManager searches for a free port.
*/
+ @Documentation.Section({Documentation.Sections.COMMON_HOST_PORT, Documentation.Sections.ALL_TASK_MANAGER})
public static final ConfigOption<Integer> DATA_PORT =
key("taskmanager.data.port")
.defaultValue(0)
@@ -48,6 +46,7 @@ public class NettyShuffleEnvironmentOptions {
/**
* Config parameter to override SSL support for taskmanager's data transport.
*/
+ @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER)
public static final ConfigOption<Boolean> DATA_SSL_ENABLED =
key("taskmanager.data.ssl.enabled")
.defaultValue(true)
@@ -61,6 +60,7 @@ public class NettyShuffleEnvironmentOptions {
* IO bounded scenario when data compression ratio is high. Currently, shuffle data compression is an experimental
* feature and the config option can be changed in the future.
*/
+ @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER_NETWORK)
public static final ConfigOption<Boolean> BLOCKING_SHUFFLE_COMPRESSION_ENABLED =
key("taskmanager.network.blocking-shuffle.compression.enabled")
.defaultValue(false)
@@ -82,6 +82,7 @@ public class NettyShuffleEnvironmentOptions {
* Boolean flag to enable/disable more detailed metrics about inbound/outbound network queue
* lengths.
*/
+ @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER_NETWORK)
public static final ConfigOption<Boolean> NETWORK_DETAILED_METRICS =
key("taskmanager.network.detailed-metrics")
.defaultValue(false)
@@ -141,6 +142,7 @@ public class NettyShuffleEnvironmentOptions {
*
* <p>Reasoning: 1 buffer for in-flight data in the subpartition + 1 buffer for parallel serialization.
*/
+ @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER_NETWORK)
public static final ConfigOption<Integer> NETWORK_BUFFERS_PER_CHANNEL =
key("taskmanager.network.memory.buffers-per-channel")
.defaultValue(2)
@@ -152,6 +154,7 @@ public class NettyShuffleEnvironmentOptions {
/**
* Number of extra network buffers to use for each outgoing/incoming gate (result partition/input gate).
*/
+ @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER_NETWORK)
public static final ConfigOption<Integer> NETWORK_EXTRA_BUFFERS_PER_GATE =
key("taskmanager.network.memory.floating-buffers-per-gate")
.defaultValue(8)
@@ -173,6 +176,7 @@ public class NettyShuffleEnvironmentOptions {
"tasks have occupied all the buffers and the downstream tasks are waiting for the exclusive buffers. The timeout breaks" +
"the tie by failing the request of exclusive buffers and ask users to increase the number of total buffers.");
+ @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER_NETWORK)
public static final ConfigOption<String> NETWORK_BLOCKING_SHUFFLE_TYPE =
key("taskmanager.network.blocking-shuffle.type")
.defaultValue("file")
@@ -185,36 +189,42 @@ public class NettyShuffleEnvironmentOptions {
// Netty Options
// ------------------------------------------------------------------------
+ @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER_NETWORK)
public static final ConfigOption<Integer> NUM_ARENAS =
key("taskmanager.network.netty.num-arenas")
.defaultValue(-1)
.withDeprecatedKeys("taskmanager.net.num-arenas")
.withDescription("The number of Netty arenas.");
+ @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER_NETWORK)
public static final ConfigOption<Integer> NUM_THREADS_SERVER =
key("taskmanager.network.netty.server.numThreads")
.defaultValue(-1)
.withDeprecatedKeys("taskmanager.net.server.numThreads")
.withDescription("The number of Netty server threads.");
+ @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER_NETWORK)
public static final ConfigOption<Integer> NUM_THREADS_CLIENT =
key("taskmanager.network.netty.client.numThreads")
.defaultValue(-1)
.withDeprecatedKeys("taskmanager.net.client.numThreads")
.withDescription("The number of Netty client threads.");
+ @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER_NETWORK)
public static final ConfigOption<Integer> CONNECT_BACKLOG =
key("taskmanager.network.netty.server.backlog")
.defaultValue(0) // default: 0 => Netty's default
.withDeprecatedKeys("taskmanager.net.server.backlog")
.withDescription("The netty server connection backlog.");
+ @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER_NETWORK)
public static final ConfigOption<Integer> CLIENT_CONNECT_TIMEOUT_SECONDS =
key("taskmanager.network.netty.client.connectTimeoutSec")
.defaultValue(120) // default: 120s = 2min
.withDeprecatedKeys("taskmanager.net.client.connectTimeoutSec")
.withDescription("The Netty client connection timeout.");
+ @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER_NETWORK)
public static final ConfigOption<Integer> SEND_RECEIVE_BUFFER_SIZE =
key("taskmanager.network.netty.sendReceiveBufferSize")
.defaultValue(0) // default: 0 => Netty's default
@@ -222,6 +232,7 @@ public class NettyShuffleEnvironmentOptions {
.withDescription("The Netty send and receive buffer size. This defaults to the system buffer size" +
" (cat /proc/sys/net/ipv4/tcp_[rw]mem) and is 4 MiB in modern Linux.");
+ @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER_NETWORK)
public static final ConfigOption<String> TRANSPORT_TYPE =
key("taskmanager.network.netty.transport")
.defaultValue("auto")
@@ -237,6 +248,7 @@ public class NettyShuffleEnvironmentOptions {
/**
* Minimum backoff for partition requests of input channels.
*/
+ @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER_NETWORK)
public static final ConfigOption<Integer> NETWORK_REQUEST_BACKOFF_INITIAL =
key("taskmanager.network.request-backoff.initial")
.defaultValue(100)
@@ -246,6 +258,7 @@ public class NettyShuffleEnvironmentOptions {
/**
* Maximum backoff for partition requests of input channels.
*/
+ @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER_NETWORK)
public static final ConfigOption<Integer> NETWORK_REQUEST_BACKOFF_MAX =
key("taskmanager.network.request-backoff.max")
.defaultValue(10000)
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/ResourceManagerOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/ResourceManagerOptions.java
index 869e97f..fde942b 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/ResourceManagerOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/ResourceManagerOptions.java
@@ -36,8 +36,9 @@ public class ResourceManagerOptions {
.withDescription("Timeout for jobs which don't have a job manager as leader assigned.");
/**
- * The number of resource managers start.
+ * This option is not used any more.
*/
+ @Deprecated
public static final ConfigOption<Integer> LOCAL_NUMBER_RESOURCE_MANAGER = ConfigOptions
.key("local.number-resourcemanager")
.defaultValue(1)
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/RestOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/RestOptions.java
index 733e5bf..457d0d2 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/RestOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/RestOptions.java
@@ -19,6 +19,7 @@
package org.apache.flink.configuration;
import org.apache.flink.annotation.Internal;
+import org.apache.flink.annotation.docs.Documentation;
import org.apache.flink.configuration.description.Description;
import static org.apache.flink.configuration.ConfigOptions.key;
@@ -35,6 +36,7 @@ public class RestOptions {
/**
* The address that the server binds itself to.
*/
+ @Documentation.Section(Documentation.Sections.COMMON_HOST_PORT)
public static final ConfigOption<String> BIND_ADDRESS =
key("rest.bind-address")
.noDefaultValue()
@@ -45,6 +47,7 @@ public class RestOptions {
/**
* The port range that the server could bind itself to.
*/
+ @Documentation.Section(Documentation.Sections.COMMON_HOST_PORT)
public static final ConfigOption<String> BIND_PORT =
key("rest.bind-port")
.defaultValue("8081")
@@ -58,6 +61,7 @@ public class RestOptions {
/**
* The address that should be used by clients to connect to the server.
*/
+ @Documentation.Section(Documentation.Sections.COMMON_HOST_PORT)
public static final ConfigOption<String> ADDRESS =
key("rest.address")
.noDefaultValue()
@@ -68,6 +72,7 @@ public class RestOptions {
* The port that the REST client connects to and the REST server binds to if {@link #BIND_PORT}
* has not been specified.
*/
+ @Documentation.Section(Documentation.Sections.COMMON_HOST_PORT)
public static final ConfigOption<Integer> PORT =
key(REST_PORT_KEY)
.defaultValue(8081)
@@ -81,6 +86,7 @@ public class RestOptions {
* The time in ms that the client waits for the leader address, e.g., Dispatcher or
* WebMonitorEndpoint.
*/
+ @Documentation.Section(Documentation.Sections.EXPERT_REST)
public static final ConfigOption<Long> AWAIT_LEADER_TIMEOUT =
key("rest.await-leader-timeout")
.defaultValue(30_000L)
@@ -91,6 +97,7 @@ public class RestOptions {
* The number of retries the client will attempt if a retryable operations fails.
* @see #RETRY_DELAY
*/
+ @Documentation.Section(Documentation.Sections.EXPERT_REST)
public static final ConfigOption<Integer> RETRY_MAX_ATTEMPTS =
key("rest.retry.max-attempts")
.defaultValue(20)
@@ -101,6 +108,7 @@ public class RestOptions {
* The time in ms that the client waits between retries.
* @see #RETRY_MAX_ATTEMPTS
*/
+ @Documentation.Section(Documentation.Sections.EXPERT_REST)
public static final ConfigOption<Long> RETRY_DELAY =
key("rest.retry.delay")
.defaultValue(3_000L)
@@ -110,6 +118,7 @@ public class RestOptions {
/**
* The maximum time in ms for the client to establish a TCP connection.
*/
+ @Documentation.Section(Documentation.Sections.EXPERT_REST)
public static final ConfigOption<Long> CONNECTION_TIMEOUT =
key("rest.connection-timeout")
.defaultValue(15_000L)
@@ -118,6 +127,7 @@ public class RestOptions {
/**
* The maximum time in ms for a connection to stay idle before failing.
*/
+ @Documentation.Section(Documentation.Sections.EXPERT_REST)
public static final ConfigOption<Long> IDLENESS_TIMEOUT =
key("rest.idleness-timeout")
.defaultValue(5L * 60L * 1_000L) // 5 minutes
@@ -126,6 +136,7 @@ public class RestOptions {
/**
* The maximum content length that the server will handle.
*/
+ @Documentation.Section(Documentation.Sections.EXPERT_REST)
public static final ConfigOption<Integer> SERVER_MAX_CONTENT_LENGTH =
key("rest.server.max-content-length")
.defaultValue(104_857_600)
@@ -134,16 +145,19 @@ public class RestOptions {
/**
* The maximum content length that the client will handle.
*/
+ @Documentation.Section(Documentation.Sections.EXPERT_REST)
public static final ConfigOption<Integer> CLIENT_MAX_CONTENT_LENGTH =
key("rest.client.max-content-length")
.defaultValue(104_857_600)
.withDescription("The maximum content length in bytes that the client will handle.");
+ @Documentation.Section(Documentation.Sections.EXPERT_REST)
public static final ConfigOption<Integer> SERVER_NUM_THREADS =
key("rest.server.numThreads")
.defaultValue(4)
.withDescription("The number of threads for the asynchronous processing of requests.");
+ @Documentation.Section(Documentation.Sections.EXPERT_REST)
public static final ConfigOption<Integer> SERVER_THREAD_PRIORITY = key("rest.server.thread-priority")
.defaultValue(Thread.NORM_PRIORITY)
.withDescription("Thread priority of the REST server's executor for processing asynchronous requests. " +
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/SecurityOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/SecurityOptions.java
index 8297cfd..0e4d153 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/SecurityOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/SecurityOptions.java
@@ -18,9 +18,6 @@
package org.apache.flink.configuration;
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.annotation.docs.ConfigGroup;
-import org.apache.flink.annotation.docs.ConfigGroups;
import org.apache.flink.annotation.docs.Documentation;
import org.apache.flink.configuration.description.Description;
@@ -33,34 +30,33 @@ import static org.apache.flink.configuration.description.TextElement.text;
/**
* The set of configuration options relating to security.
*/
-@PublicEvolving
-@ConfigGroups(groups = {
- @ConfigGroup(name = "Kerberos", keyPrefix = "security.kerberos"),
- @ConfigGroup(name = "ZooKeeper", keyPrefix = "zookeeper")
-})
public class SecurityOptions {
// ------------------------------------------------------------------------
// Kerberos Options
// ------------------------------------------------------------------------
+ @Documentation.Section(Documentation.Sections.SECURITY_AUTH_KERBEROS)
public static final ConfigOption<String> KERBEROS_LOGIN_PRINCIPAL =
key("security.kerberos.login.principal")
.noDefaultValue()
.withDeprecatedKeys("security.principal")
.withDescription("Kerberos principal name associated with the keytab.");
+ @Documentation.Section(Documentation.Sections.SECURITY_AUTH_KERBEROS)
public static final ConfigOption<String> KERBEROS_LOGIN_KEYTAB =
key("security.kerberos.login.keytab")
.noDefaultValue()
.withDeprecatedKeys("security.keytab")
.withDescription("Absolute path to a Kerberos keytab file that contains the user credentials.");
+ @Documentation.Section(Documentation.Sections.SECURITY_AUTH_KERBEROS)
public static final ConfigOption<Boolean> KERBEROS_LOGIN_USETICKETCACHE =
key("security.kerberos.login.use-ticket-cache")
.defaultValue(true)
.withDescription("Indicates whether to read from your Kerberos ticket cache.");
+ @Documentation.Section(Documentation.Sections.SECURITY_AUTH_KERBEROS)
public static final ConfigOption<String> KERBEROS_LOGIN_CONTEXTS =
key("security.kerberos.login.contexts")
.noDefaultValue()
@@ -73,14 +69,17 @@ public class SecurityOptions {
// ZooKeeper Security Options
// ------------------------------------------------------------------------
+ @Documentation.Section(Documentation.Sections.SECURITY_AUTH_ZOOKEEPER)
public static final ConfigOption<Boolean> ZOOKEEPER_SASL_DISABLE =
key("zookeeper.sasl.disable")
.defaultValue(false);
+ @Documentation.Section(Documentation.Sections.SECURITY_AUTH_ZOOKEEPER)
public static final ConfigOption<String> ZOOKEEPER_SASL_SERVICE_NAME =
key("zookeeper.sasl.service-name")
.defaultValue("zookeeper");
+ @Documentation.Section(Documentation.Sections.SECURITY_AUTH_ZOOKEEPER)
public static final ConfigOption<String> ZOOKEEPER_SASL_LOGIN_CONTEXT_NAME =
key("zookeeper.sasl.login-context-name")
.defaultValue("Client");
@@ -106,9 +105,7 @@ public class SecurityOptions {
/**
* Enable SSL for internal communication (akka rpc, netty data transport, blob server).
*/
- @Documentation.Section(
- value = {Documentation.Section.SECTION_COMMON},
- position = Documentation.Section.POSITION_SECURITY)
+ @Documentation.Section(Documentation.Sections.SECURITY_SSL)
public static final ConfigOption<Boolean> SSL_INTERNAL_ENABLED =
key("security.ssl.internal.enabled")
.defaultValue(false)
@@ -119,9 +116,7 @@ public class SecurityOptions {
/**
* Enable SSL for external REST endpoints.
*/
- @Documentation.Section(
- value = {Documentation.Section.SECTION_COMMON},
- position = Documentation.Section.POSITION_SECURITY)
+ @Documentation.Section(Documentation.Sections.SECURITY_SSL)
public static final ConfigOption<Boolean> SSL_REST_ENABLED =
key("security.ssl.rest.enabled")
.defaultValue(false)
@@ -130,6 +125,7 @@ public class SecurityOptions {
/**
* Enable mututal SSL authentication for external REST endpoints.
*/
+ @Documentation.Section(Documentation.Sections.SECURITY_SSL)
public static final ConfigOption<Boolean> SSL_REST_AUTHENTICATION_ENABLED =
key("security.ssl.rest.authentication-enabled")
.defaultValue(false)
@@ -140,6 +136,7 @@ public class SecurityOptions {
/**
* The Java keystore file containing the flink endpoint key and certificate.
*/
+ @Documentation.ExcludeFromDocumentation("The SSL Setup encourages separate configs for internal and REST security.")
public static final ConfigOption<String> SSL_KEYSTORE =
key("security.ssl.keystore")
.noDefaultValue()
@@ -148,6 +145,7 @@ public class SecurityOptions {
/**
* Secret to decrypt the keystore file.
*/
+ @Documentation.ExcludeFromDocumentation("The SSL Setup encourages separate configs for internal and REST security.")
public static final ConfigOption<String> SSL_KEYSTORE_PASSWORD =
key("security.ssl.keystore-password")
.noDefaultValue()
@@ -156,6 +154,7 @@ public class SecurityOptions {
/**
* Secret to decrypt the server key.
*/
+ @Documentation.ExcludeFromDocumentation("The SSL Setup encourages separate configs for internal and REST security.")
public static final ConfigOption<String> SSL_KEY_PASSWORD =
key("security.ssl.key-password")
.noDefaultValue()
@@ -164,6 +163,7 @@ public class SecurityOptions {
/**
* The truststore file containing the public CA certificates to verify the ssl peers.
*/
+ @Documentation.ExcludeFromDocumentation("The SSL Setup encourages separate configs for internal and REST security.")
public static final ConfigOption<String> SSL_TRUSTSTORE =
key("security.ssl.truststore")
.noDefaultValue()
@@ -173,6 +173,7 @@ public class SecurityOptions {
/**
* Secret to decrypt the truststore.
*/
+ @Documentation.ExcludeFromDocumentation("The SSL Setup encourages separate configs for internal and REST security.")
public static final ConfigOption<String> SSL_TRUSTSTORE_PASSWORD =
key("security.ssl.truststore-password")
.noDefaultValue()
@@ -183,6 +184,7 @@ public class SecurityOptions {
/**
* For internal SSL, the Java keystore file containing the private key and certificate.
*/
+ @Documentation.Section(Documentation.Sections.SECURITY_SSL)
public static final ConfigOption<String> SSL_INTERNAL_KEYSTORE =
key("security.ssl.internal.keystore")
.noDefaultValue()
@@ -192,6 +194,7 @@ public class SecurityOptions {
/**
* For internal SSL, the password to decrypt the keystore file containing the certificate.
*/
+ @Documentation.Section(Documentation.Sections.SECURITY_SSL)
public static final ConfigOption<String> SSL_INTERNAL_KEYSTORE_PASSWORD =
key("security.ssl.internal.keystore-password")
.noDefaultValue()
@@ -201,6 +204,7 @@ public class SecurityOptions {
/**
* For internal SSL, the password to decrypt the private key.
*/
+ @Documentation.Section(Documentation.Sections.SECURITY_SSL)
public static final ConfigOption<String> SSL_INTERNAL_KEY_PASSWORD =
key("security.ssl.internal.key-password")
.noDefaultValue()
@@ -210,6 +214,7 @@ public class SecurityOptions {
/**
* For internal SSL, the truststore file containing the public CA certificates to verify the ssl peers.
*/
+ @Documentation.Section(Documentation.Sections.SECURITY_SSL)
public static final ConfigOption<String> SSL_INTERNAL_TRUSTSTORE =
key("security.ssl.internal.truststore")
.noDefaultValue()
@@ -219,6 +224,7 @@ public class SecurityOptions {
/**
* For internal SSL, the secret to decrypt the truststore.
*/
+ @Documentation.Section(Documentation.Sections.SECURITY_SSL)
public static final ConfigOption<String> SSL_INTERNAL_TRUSTSTORE_PASSWORD =
key("security.ssl.internal.truststore-password")
.noDefaultValue()
@@ -228,6 +234,7 @@ public class SecurityOptions {
/**
* For internal SSL, the sha1 fingerprint of the internal certificate to verify the client.
*/
+ @Documentation.Section(Documentation.Sections.SECURITY_SSL)
public static final ConfigOption<String> SSL_INTERNAL_CERT_FINGERPRINT =
key("security.ssl.internal.cert.fingerprint")
.noDefaultValue()
@@ -240,6 +247,7 @@ public class SecurityOptions {
/**
* For external (REST) SSL, the Java keystore file containing the private key and certificate.
*/
+ @Documentation.Section(Documentation.Sections.SECURITY_SSL)
public static final ConfigOption<String> SSL_REST_KEYSTORE =
key("security.ssl.rest.keystore")
.noDefaultValue()
@@ -249,6 +257,7 @@ public class SecurityOptions {
/**
* For external (REST) SSL, the password to decrypt the keystore file containing the certificate.
*/
+ @Documentation.Section(Documentation.Sections.SECURITY_SSL)
public static final ConfigOption<String> SSL_REST_KEYSTORE_PASSWORD =
key("security.ssl.rest.keystore-password")
.noDefaultValue()
@@ -258,6 +267,7 @@ public class SecurityOptions {
/**
* For external (REST) SSL, the password to decrypt the private key.
*/
+ @Documentation.Section(Documentation.Sections.SECURITY_SSL)
public static final ConfigOption<String> SSL_REST_KEY_PASSWORD =
key("security.ssl.rest.key-password")
.noDefaultValue()
@@ -267,6 +277,7 @@ public class SecurityOptions {
/**
* For external (REST) SSL, the truststore file containing the public CA certificates to verify the ssl peers.
*/
+ @Documentation.Section(Documentation.Sections.SECURITY_SSL)
public static final ConfigOption<String> SSL_REST_TRUSTSTORE =
key("security.ssl.rest.truststore")
.noDefaultValue()
@@ -276,6 +287,7 @@ public class SecurityOptions {
/**
* For external (REST) SSL, the secret to decrypt the truststore.
*/
+ @Documentation.Section(Documentation.Sections.SECURITY_SSL)
public static final ConfigOption<String> SSL_REST_TRUSTSTORE_PASSWORD =
key("security.ssl.rest.truststore-password")
.noDefaultValue()
@@ -285,6 +297,7 @@ public class SecurityOptions {
/**
* For external (REST) SSL, the sha1 fingerprint of the rest client certificate to verify.
*/
+ @Documentation.Section(Documentation.Sections.SECURITY_SSL)
public static final ConfigOption<String> SSL_REST_CERT_FINGERPRINT =
key("security.ssl.rest.cert.fingerprint")
.noDefaultValue()
@@ -297,6 +310,7 @@ public class SecurityOptions {
/**
* SSL protocol version to be supported.
*/
+ @Documentation.Section(Documentation.Sections.SECURITY_SSL)
public static final ConfigOption<String> SSL_PROTOCOL =
key("security.ssl.protocol")
.defaultValue("TLSv1.2")
@@ -308,6 +322,7 @@ public class SecurityOptions {
*
* <p>More options here - http://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html#ciphersuites
*/
+ @Documentation.Section(Documentation.Sections.SECURITY_SSL)
public static final ConfigOption<String> SSL_ALGORITHMS =
key("security.ssl.algorithms")
.defaultValue("TLS_RSA_WITH_AES_128_CBC_SHA")
@@ -321,6 +336,7 @@ public class SecurityOptions {
/**
* Flag to enable/disable hostname verification for the ssl connections.
*/
+ @Documentation.Section(Documentation.Sections.SECURITY_SSL)
public static final ConfigOption<Boolean> SSL_VERIFY_HOSTNAME =
key("security.ssl.verify-hostname")
.defaultValue(true)
@@ -329,6 +345,7 @@ public class SecurityOptions {
/**
* SSL engine provider.
*/
+ @Documentation.Section(Documentation.Sections.EXPERT_SECURITY_SSL)
public static final ConfigOption<String> SSL_PROVIDER =
key("security.ssl.provider")
.defaultValue("JDK")
@@ -365,6 +382,7 @@ public class SecurityOptions {
/**
* SSL session cache size.
*/
+ @Documentation.Section(Documentation.Sections.EXPERT_SECURITY_SSL)
public static final ConfigOption<Integer> SSL_INTERNAL_SESSION_CACHE_SIZE =
key("security.ssl.internal.session-cache-size")
.defaultValue(-1)
@@ -377,6 +395,7 @@ public class SecurityOptions {
/**
* SSL session timeout.
*/
+ @Documentation.Section(Documentation.Sections.EXPERT_SECURITY_SSL)
public static final ConfigOption<Integer> SSL_INTERNAL_SESSION_TIMEOUT =
key("security.ssl.internal.session-timeout")
.defaultValue(-1)
@@ -386,6 +405,7 @@ public class SecurityOptions {
/**
* SSL session timeout during handshakes.
*/
+ @Documentation.Section(Documentation.Sections.EXPERT_SECURITY_SSL)
public static final ConfigOption<Integer> SSL_INTERNAL_HANDSHAKE_TIMEOUT =
key("security.ssl.internal.handshake-timeout")
.defaultValue(-1)
@@ -395,6 +415,7 @@ public class SecurityOptions {
/**
* SSL session timeout after flushing the <tt>close_notify</tt> message.
*/
+ @Documentation.Section(Documentation.Sections.EXPERT_SECURITY_SSL)
public static final ConfigOption<Integer> SSL_INTERNAL_CLOSE_NOTIFY_FLUSH_TIMEOUT =
key("security.ssl.internal.close-notify-flush-timeout")
.defaultValue(-1)
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java
index 05ad75c..a5014c8 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java
@@ -72,6 +72,7 @@ public class TaskManagerOptions {
/**
* Whether to kill the TaskManager when the task thread throws an OutOfMemoryError.
*/
+ @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER)
public static final ConfigOption<Boolean> KILL_ON_OUT_OF_MEMORY =
key("taskmanager.jvm-exit-on-oom")
.booleanType()
@@ -96,6 +97,7 @@ public class TaskManagerOptions {
* The config parameter defining the task manager's hostname.
* Overrides {@link #HOST_BIND_POLICY} automatic address binding.
*/
+ @Documentation.Section({Documentation.Sections.COMMON_HOST_PORT, Documentation.Sections.ALL_TASK_MANAGER})
public static final ConfigOption<String> HOST =
key("taskmanager.host")
.stringType()
@@ -109,6 +111,7 @@ public class TaskManagerOptions {
* The default network port range the task manager expects incoming IPC connections. The {@code "0"} means that
* the TaskManager searches for a free port.
*/
+ @Documentation.Section({Documentation.Sections.COMMON_HOST_PORT, Documentation.Sections.ALL_TASK_MANAGER})
public static final ConfigOption<String> RPC_PORT =
key("taskmanager.rpc.port")
.stringType()
@@ -121,6 +124,7 @@ public class TaskManagerOptions {
* The initial registration backoff between two consecutive registration attempts. The backoff
* is doubled for each new registration attempt until it reaches the maximum registration backoff.
*/
+ @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER)
public static final ConfigOption<Duration> INITIAL_REGISTRATION_BACKOFF =
key("taskmanager.registration.initial-backoff")
.durationType()
@@ -132,6 +136,7 @@ public class TaskManagerOptions {
/**
* The maximum registration backoff between two consecutive registration attempts.
*/
+ @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER)
public static final ConfigOption<Duration> REGISTRATION_MAX_BACKOFF =
key("taskmanager.registration.max-backoff")
.durationType()
@@ -143,6 +148,7 @@ public class TaskManagerOptions {
/**
* The backoff after a registration has been refused by the job manager before retrying to connect.
*/
+ @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER)
public static final ConfigOption<Duration> REFUSED_REGISTRATION_BACKOFF =
key("taskmanager.registration.refused-backoff")
.durationType()
@@ -154,6 +160,7 @@ public class TaskManagerOptions {
* Defines the timeout it can take for the TaskManager registration. If the duration is
* exceeded without a successful registration, then the TaskManager terminates.
*/
+ @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER)
public static final ConfigOption<Duration> REGISTRATION_TIMEOUT =
key("taskmanager.registration.timeout")
.durationType()
@@ -165,9 +172,7 @@ public class TaskManagerOptions {
/**
* The config parameter defining the number of task slots of a task manager.
*/
- @Documentation.Section(
- value = {Documentation.Section.SECTION_COMMON},
- position = Documentation.Section.POSITION_PARALLELISM_SLOTS)
+ @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER)
public static final ConfigOption<Integer> NUM_TASK_SLOTS =
key("taskmanager.numberOfTaskSlots")
.intType()
@@ -179,6 +184,7 @@ public class TaskManagerOptions {
" is typically proportional to the number of physical CPU cores that the TaskManager's machine has" +
" (e.g., equal to the number of cores, or half the number of cores).");
+ @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER)
public static final ConfigOption<Boolean> DEBUG_MEMORY_LOG =
key("taskmanager.debug.memory.log")
.booleanType()
@@ -186,6 +192,7 @@ public class TaskManagerOptions {
.withDeprecatedKeys("taskmanager.debug.memory.startLogThread")
.withDescription("Flag indicating whether to start a thread, which repeatedly logs the memory usage of the JVM.");
+ @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER)
public static final ConfigOption<Long> DEBUG_MEMORY_USAGE_LOG_INTERVAL_MS =
key("taskmanager.debug.memory.log-interval")
.longType()
@@ -200,6 +207,7 @@ public class TaskManagerOptions {
/**
* Size of memory buffers used by the network stack and the memory manager.
*/
+ @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER)
public static final ConfigOption<MemorySize> MEMORY_SEGMENT_SIZE =
key("taskmanager.memory.segment-size")
.memoryType()
@@ -210,6 +218,7 @@ public class TaskManagerOptions {
* The config parameter for automatically defining the TaskManager's binding address,
* if {@link #HOST} configuration option is not set.
*/
+ @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER)
public static final ConfigOption<String> HOST_BIND_POLICY =
key("taskmanager.network.bind-policy")
.stringType()
@@ -254,9 +263,7 @@ public class TaskManagerOptions {
/**
* Total Process Memory size for the TaskExecutors.
*/
- @Documentation.Section(
- value = {Documentation.Section.SECTION_COMMON},
- position = Documentation.Section.POSITION_MEMORY)
+ @Documentation.Section(Documentation.Sections.COMMON_MEMORY)
public static final ConfigOption<MemorySize> TOTAL_PROCESS_MEMORY =
key("taskmanager.memory.process.size")
.memoryType()
@@ -270,9 +277,7 @@ public class TaskManagerOptions {
/**
* Total Flink Memory size for the TaskExecutors.
*/
- @Documentation.Section(
- value = {Documentation.Section.SECTION_COMMON},
- position = Documentation.Section.POSITION_MEMORY)
+ @Documentation.Section(Documentation.Sections.COMMON_MEMORY)
public static final ConfigOption<MemorySize> TOTAL_FLINK_MEMORY =
key("taskmanager.memory.flink.size")
.memoryType()
@@ -286,6 +291,7 @@ public class TaskManagerOptions {
/**
* Framework Heap Memory size for TaskExecutors.
*/
+ @Documentation.Section(Documentation.Sections.COMMON_MEMORY)
public static final ConfigOption<MemorySize> FRAMEWORK_HEAP_MEMORY =
key("taskmanager.memory.framework.heap.size")
.memoryType()
@@ -296,6 +302,7 @@ public class TaskManagerOptions {
/**
* Framework Off-Heap Memory size for TaskExecutors.
*/
+ @Documentation.Section(Documentation.Sections.COMMON_MEMORY)
public static final ConfigOption<MemorySize> FRAMEWORK_OFF_HEAP_MEMORY =
key("taskmanager.memory.framework.off-heap.size")
.memoryType()
@@ -308,6 +315,7 @@ public class TaskManagerOptions {
/**
* Task Heap Memory size for TaskExecutors.
*/
+ @Documentation.Section(Documentation.Sections.COMMON_MEMORY)
public static final ConfigOption<MemorySize> TASK_HEAP_MEMORY =
key("taskmanager.memory.task.heap.size")
.memoryType()
@@ -319,6 +327,7 @@ public class TaskManagerOptions {
/**
* Task Off-Heap Memory size for TaskExecutors.
*/
+ @Documentation.Section(Documentation.Sections.COMMON_MEMORY)
public static final ConfigOption<MemorySize> TASK_OFF_HEAP_MEMORY =
key("taskmanager.memory.task.off-heap.size")
.memoryType()
@@ -330,6 +339,7 @@ public class TaskManagerOptions {
/**
* Managed Memory size for TaskExecutors.
*/
+ @Documentation.Section(Documentation.Sections.COMMON_MEMORY)
public static final ConfigOption<MemorySize> MANAGED_MEMORY_SIZE =
key("taskmanager.memory.managed.size")
.memoryType()
@@ -345,6 +355,7 @@ public class TaskManagerOptions {
/**
* Fraction of Total Flink Memory to be used as Managed Memory, if {@link #MANAGED_MEMORY_SIZE} is not specified.
*/
+ @Documentation.Section(Documentation.Sections.COMMON_MEMORY)
public static final ConfigOption<Float> MANAGED_MEMORY_FRACTION =
key("taskmanager.memory.managed.fraction")
.floatType()
@@ -355,6 +366,7 @@ public class TaskManagerOptions {
/**
* Min Network Memory size for TaskExecutors.
*/
+ @Documentation.Section(Documentation.Sections.COMMON_MEMORY)
public static final ConfigOption<MemorySize> NETWORK_MEMORY_MIN =
key("taskmanager.memory.network.min")
.memoryType()
@@ -369,6 +381,7 @@ public class TaskManagerOptions {
/**
* Max Network Memory size for TaskExecutors.
*/
+ @Documentation.Section(Documentation.Sections.COMMON_MEMORY)
public static final ConfigOption<MemorySize> NETWORK_MEMORY_MAX =
key("taskmanager.memory.network.max")
.memoryType()
@@ -383,6 +396,7 @@ public class TaskManagerOptions {
/**
* Fraction of Total Flink Memory to be used as Network Memory.
*/
+ @Documentation.Section(Documentation.Sections.COMMON_MEMORY)
public static final ConfigOption<Float> NETWORK_MEMORY_FRACTION =
key("taskmanager.memory.network.fraction")
.floatType()
@@ -397,6 +411,7 @@ public class TaskManagerOptions {
/**
* JVM Metaspace Size for the TaskExecutors.
*/
+ @Documentation.Section(Documentation.Sections.COMMON_MEMORY)
public static final ConfigOption<MemorySize> JVM_METASPACE =
key("taskmanager.memory.jvm-metaspace.size")
.memoryType()
@@ -406,6 +421,7 @@ public class TaskManagerOptions {
/**
* Min JVM Overhead size for the TaskExecutors.
*/
+ @Documentation.Section(Documentation.Sections.COMMON_MEMORY)
public static final ConfigOption<MemorySize> JVM_OVERHEAD_MIN =
key("taskmanager.memory.jvm-overhead.min")
.memoryType()
@@ -420,6 +436,7 @@ public class TaskManagerOptions {
/**
* Max JVM Overhead size for the TaskExecutors.
*/
+ @Documentation.Section(Documentation.Sections.COMMON_MEMORY)
public static final ConfigOption<MemorySize> JVM_OVERHEAD_MAX =
key("taskmanager.memory.jvm-overhead.max")
.memoryType()
@@ -434,6 +451,7 @@ public class TaskManagerOptions {
/**
* Fraction of Total Process Memory to be reserved for JVM Overhead.
*/
+ @Documentation.Section(Documentation.Sections.COMMON_MEMORY)
public static final ConfigOption<Float> JVM_OVERHEAD_FRACTION =
key("taskmanager.memory.jvm-overhead.fraction")
.floatType()
@@ -454,6 +472,7 @@ public class TaskManagerOptions {
* Time interval in milliseconds between two successive task cancellation
* attempts.
*/
+ @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER)
public static final ConfigOption<Long> TASK_CANCELLATION_INTERVAL =
key("task.cancellation.interval")
.longType()
@@ -466,6 +485,7 @@ public class TaskManagerOptions {
* leads to a fatal TaskManager error. A value of <code>0</code> deactivates
* the watch dog.
*/
+ @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER)
public static final ConfigOption<Long> TASK_CANCELLATION_TIMEOUT =
key("task.cancellation.timeout")
.longType()
@@ -477,6 +497,7 @@ public class TaskManagerOptions {
* This configures how long we wait for the timers in milliseconds to finish all pending timer threads
* when the stream task is cancelled.
*/
+ @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER)
public static final ConfigOption<Long> TASK_CANCELLATION_TIMEOUT_TIMERS = ConfigOptions
.key("task.cancellation.timers.timeout")
.longType()
@@ -492,6 +513,7 @@ public class TaskManagerOptions {
*
* <p>The default value of {@code -1} indicates that there is no limit.
*/
+ @Documentation.ExcludeFromDocumentation("With flow control, there is no alignment spilling any more")
public static final ConfigOption<Long> TASK_CHECKPOINT_ALIGNMENT_BYTES_LIMIT =
key("task.checkpoint.alignment.max-size")
.longType()
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/WebOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/WebOptions.java
index d7a41b1..762f376 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/WebOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/WebOptions.java
@@ -32,6 +32,7 @@ public class WebOptions {
/**
* Config parameter defining the runtime monitor web-frontend server address.
*/
+ @Deprecated
public static final ConfigOption<String> ADDRESS =
key("web.address")
.noDefaultValue()
@@ -71,6 +72,7 @@ public class WebOptions {
/**
* Config parameter to override SSL support for the JobManager Web UI.
*/
+ @Deprecated
public static final ConfigOption<Boolean> SSL_ENABLED =
key("web.ssl.enabled")
.defaultValue(true)
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBOptions.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBOptions.java
index c147c38..932d036 100644
--- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBOptions.java
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBOptions.java
@@ -18,6 +18,7 @@
package org.apache.flink.contrib.streaming.state;
+import org.apache.flink.annotation.docs.Documentation;
import org.apache.flink.configuration.ConfigOption;
import org.apache.flink.configuration.ConfigOptions;
import org.apache.flink.configuration.MemorySize;
@@ -35,6 +36,7 @@ import static org.apache.flink.contrib.streaming.state.RocksDBStateBackend.Prior
public class RocksDBOptions {
/** The local directory (on the TaskManager) where RocksDB puts its files. */
+ @Documentation.Section(Documentation.Sections.EXPERT_ROCKSDB)
public static final ConfigOption<String> LOCAL_DIRECTORIES = ConfigOptions
.key("state.backend.rocksdb.localdir")
.noDefaultValue()
@@ -44,6 +46,7 @@ public class RocksDBOptions {
/**
* Choice of timer service implementation.
*/
+ @Documentation.Section(Documentation.Sections.STATE_BACKEND_ROCKSDB)
public static final ConfigOption<String> TIMER_SERVICE_FACTORY = ConfigOptions
.key("state.backend.rocksdb.timer-service.factory")
.defaultValue(ROCKSDB.name())
@@ -54,6 +57,7 @@ public class RocksDBOptions {
/**
* The number of threads used to transfer (download and upload) files in RocksDBStateBackend.
*/
+ @Documentation.Section(Documentation.Sections.EXPERT_ROCKSDB)
public static final ConfigOption<Integer> CHECKPOINT_TRANSFER_THREAD_NUM = ConfigOptions
.key("state.backend.rocksdb.checkpoint.transfer.thread.num")
.defaultValue(1)
@@ -75,6 +79,7 @@ public class RocksDBOptions {
/**
* The predefined settings for RocksDB DBOptions and ColumnFamilyOptions by Flink community.
*/
+ @Documentation.Section(Documentation.Sections.EXPERT_ROCKSDB)
public static final ConfigOption<String> PREDEFINED_OPTIONS = ConfigOptions
.key("state.backend.rocksdb.predefined-options")
.defaultValue(DEFAULT.name())
@@ -86,6 +91,7 @@ public class RocksDBOptions {
/**
* The options factory class for RocksDB to create DBOptions and ColumnFamilyOptions.
*/
+ @Documentation.Section(Documentation.Sections.EXPERT_ROCKSDB)
public static final ConfigOption<String> OPTIONS_FACTORY = ConfigOptions
.key("state.backend.rocksdb.options-factory")
.defaultValue(DefaultConfigurableOptionsFactory.class.getName())
@@ -93,6 +99,7 @@ public class RocksDBOptions {
"The default options factory is %s, and it would read the configured options which provided in 'RocksDBConfigurableOptions'.",
DefaultConfigurableOptionsFactory.class.getName()));
+ @Documentation.Section(Documentation.Sections.STATE_BACKEND_ROCKSDB)
public static final ConfigOption<Boolean> USE_MANAGED_MEMORY = ConfigOptions
.key("state.backend.rocksdb.memory.managed")
.booleanType()
@@ -101,6 +108,7 @@ public class RocksDBOptions {
"managed memory budget of the task slot, and divide the memory over write buffers, indexes, " +
"block caches, etc. That way, the three major uses of memory of RocksDB will be capped.");
+ @Documentation.Section(Documentation.Sections.STATE_BACKEND_ROCKSDB)
public static final ConfigOption<MemorySize> FIX_PER_SLOT_MEMORY_SIZE = ConfigOptions
.key("state.backend.rocksdb.memory.fixed-per-slot")
.memoryType()
@@ -111,6 +119,7 @@ public class RocksDBOptions {
"are set, then each RocksDB column family state has its own memory caches (as controlled by the column " +
"family options).", USE_MANAGED_MEMORY.key(), USE_MANAGED_MEMORY.key()));
+ @Documentation.Section(Documentation.Sections.STATE_BACKEND_ROCKSDB)
public static final ConfigOption<Double> WRITE_BUFFER_RATIO = ConfigOptions
.key("state.backend.rocksdb.memory.write-buffer-ratio")
.doubleType()
@@ -121,6 +130,7 @@ public class RocksDBOptions {
USE_MANAGED_MEMORY.key(),
FIX_PER_SLOT_MEMORY_SIZE.key()));
+ @Documentation.Section(Documentation.Sections.STATE_BACKEND_ROCKSDB)
public static final ConfigOption<Double> HIGH_PRIORITY_POOL_RATIO = ConfigOptions
.key("state.backend.rocksdb.memory.high-prio-pool-ratio")
.doubleType()