You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by gu...@apache.org on 2024/09/03 12:20:54 UTC
(flink) 02/03: [FLINK-36198][network] Remove deprecated config options related to legacy hybrid shuffle
This is an automated email from the ASF dual-hosted git repository.
guoweijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit ae1d7c1c8e6fb21bfe76c3ae734eec3eb6836368
Author: Weijie Guo <re...@163.com>
AuthorDate: Tue Sep 3 12:07:46 2024 +0800
[FLINK-36198][network] Remove deprecated config options related to legacy hybrid shuffle
---
.../generated/all_taskmanager_network_section.html | 2 +-
.../netty_shuffle_environment_configuration.html | 2 +-
.../NettyShuffleEnvironmentOptions.java | 68 +---------------------
3 files changed, 3 insertions(+), 69 deletions(-)
diff --git a/docs/layouts/shortcodes/generated/all_taskmanager_network_section.html b/docs/layouts/shortcodes/generated/all_taskmanager_network_section.html
index 14e7532b050..56ad08d2f30 100644
--- a/docs/layouts/shortcodes/generated/all_taskmanager_network_section.html
+++ b/docs/layouts/shortcodes/generated/all_taskmanager_network_section.html
@@ -30,7 +30,7 @@
<td><h5>taskmanager.network.hybrid-shuffle.remote.path</h5></td>
<td style="word-wrap: break-word;">(none)</td>
<td>String</td>
- <td>The option is used to configure the base path of remote storage for hybrid shuffle. The shuffle data will be stored in remote storage when the disk space is not enough. Note: If the option is configured and taskmanager.network.hybrid-shuffle.enable-new-mode is false, this option will be ignored. If the option is not configured and taskmanager.network.hybrid-shuffle.enable-new-mode is true, the remote storage will be disabled.</td>
+ <td>The option is used to configure the base path of remote storage for hybrid shuffle. The shuffle data will be stored in remote storage when the disk space is not enough. Note: If this option is not configured the remote storage will be disabled.</td>
</tr>
<tr>
<td><h5>taskmanager.network.memory.buffer-debloat.enabled</h5></td>
diff --git a/docs/layouts/shortcodes/generated/netty_shuffle_environment_configuration.html b/docs/layouts/shortcodes/generated/netty_shuffle_environment_configuration.html
index cfd54b05540..a071530f563 100644
--- a/docs/layouts/shortcodes/generated/netty_shuffle_environment_configuration.html
+++ b/docs/layouts/shortcodes/generated/netty_shuffle_environment_configuration.html
@@ -48,7 +48,7 @@
<td><h5>taskmanager.network.hybrid-shuffle.remote.path</h5></td>
<td style="word-wrap: break-word;">(none)</td>
<td>String</td>
- <td>The option is used to configure the base path of remote storage for hybrid shuffle. The shuffle data will be stored in remote storage when the disk space is not enough. Note: If the option is configured and taskmanager.network.hybrid-shuffle.enable-new-mode is false, this option will be ignored. If the option is not configured and taskmanager.network.hybrid-shuffle.enable-new-mode is true, the remote storage will be disabled.</td>
+ <td>The option is used to configure the base path of remote storage for hybrid shuffle. The shuffle data will be stored in remote storage when the disk space is not enough. Note: If this option is not configured the remote storage will be disabled.</td>
</tr>
<tr>
<td><h5>taskmanager.network.memory.read-buffer.required-per-gate.max</h5></td>
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/NettyShuffleEnvironmentOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/NettyShuffleEnvironmentOptions.java
index 72cf1404cb4..a819f28486b 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/NettyShuffleEnvironmentOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/NettyShuffleEnvironmentOptions.java
@@ -31,10 +31,6 @@ import static org.apache.flink.configuration.description.TextElement.code;
/** The set of configuration options relating to network stack. */
@PublicEvolving
public class NettyShuffleEnvironmentOptions {
-
- private static final String HYBRID_SHUFFLE_NEW_MODE_OPTION_NAME =
- "taskmanager.network.hybrid-shuffle.enable-new-mode";
-
private static final String HYBRID_SHUFFLE_REMOTE_STORAGE_BASE_PATH_OPTION_NAME =
"taskmanager.network.hybrid-shuffle.remote.path";
@@ -357,44 +353,6 @@ public class NettyShuffleEnvironmentOptions {
// this raw value must be changed correspondingly
"taskmanager.memory.framework.off-heap.batch-shuffle.size"));
- /**
- * Region group size of hybrid spilled file data index.
- *
- * @deprecated The option is only used in the legacy hybrid shuffle mode. It is deprecated in
- * 1.20 and will be totally removed in 2.0, as the legacy hybrid shuffle mode will be
- * removed in 2.0.
- */
- @Deprecated
- public static final ConfigOption<Integer> HYBRID_SHUFFLE_SPILLED_INDEX_REGION_GROUP_SIZE =
- key("taskmanager.network.hybrid-shuffle.spill-index-region-group-size")
- .intType()
- .defaultValue(1024)
- .withDeprecatedKeys(
- "taskmanager.network.hybrid-shuffle.spill-index-segment-size")
- .withDescription(
- "Controls the region group size(in bytes) of hybrid spilled file data index. "
- + "Note: This option will be ignored if "
- + HYBRID_SHUFFLE_NEW_MODE_OPTION_NAME
- + " is set true.");
-
- /**
- * Max number of hybrid retained regions in memory.
- *
- * @deprecated The option is only used in the legacy hybrid shuffle mode. It is deprecated in
- * 1.20 and will be totally removed in 2.0, as the legacy hybrid shuffle mode will be
- * removed in 2.0.
- */
- @Deprecated
- public static final ConfigOption<Long> HYBRID_SHUFFLE_NUM_RETAINED_IN_MEMORY_REGIONS_MAX =
- key("taskmanager.network.hybrid-shuffle.num-retained-in-memory-regions-max")
- .longType()
- .defaultValue(1024 * 1024L)
- .withDescription(
- "Controls the max number of hybrid retained regions in memory. "
- + "Note: This option will be ignored if "
- + HYBRID_SHUFFLE_NEW_MODE_OPTION_NAME
- + " is set true. ");
-
/**
* Number of max buffers can be used for each output subpartition.
*
@@ -471,25 +429,6 @@ public class NettyShuffleEnvironmentOptions {
+ "tasks have occupied all the buffers and the downstream tasks are waiting for the exclusive buffers. The timeout breaks"
+ "the tie by failing the request of exclusive buffers and ask users to increase the number of total buffers.");
- /**
- * The option to enable the new mode of hybrid shuffle.
- *
- * @deprecated This option is deprecated in 1.20 and will be totally removed in 2.0, as the
- * legacy hybrid shuffle mode will be removed in 2.0.
- */
- @Deprecated @Experimental
- public static final ConfigOption<Boolean> NETWORK_HYBRID_SHUFFLE_ENABLE_NEW_MODE =
- ConfigOptions.key(HYBRID_SHUFFLE_NEW_MODE_OPTION_NAME)
- .booleanType()
- .defaultValue(true)
- .withDescription(
- "The option is used to enable the new mode of hybrid shuffle, which has resolved existing issues in the legacy mode. First, the new mode "
- + "uses less required network memory. Second, the new mode can store shuffle data in remote storage when the disk space is not "
- + "enough, which could avoid insufficient disk space errors and is only supported when "
- + HYBRID_SHUFFLE_REMOTE_STORAGE_BASE_PATH_OPTION_NAME
- + " is configured. The new mode is currently in an experimental phase. It can be set to false to fallback to the legacy mode "
- + " if something unexpected. Once the new mode reaches a stable state, the legacy mode as well as the option will be removed.");
-
/** The option to configure the tiered factory creator remote class name for hybrid shuffle. */
@Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER_NETWORK)
@Experimental
@@ -518,12 +457,7 @@ public class NettyShuffleEnvironmentOptions {
.withDescription(
"The option is used to configure the base path of remote storage for hybrid shuffle. The shuffle data will be stored in "
+ "remote storage when the disk space is not enough. "
- + "Note: If the option is configured and "
- + HYBRID_SHUFFLE_NEW_MODE_OPTION_NAME
- + " is false, this option will be ignored. "
- + "If the option is not configured and "
- + HYBRID_SHUFFLE_NEW_MODE_OPTION_NAME
- + " is true, the remote storage will be disabled.");
+ + "Note: If this option is not configured the remote storage will be disabled.");
/**
* @deprecated The hash-based blocking shuffle is deprecated in 1.20 and will be totally removed