You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by et...@apache.org on 2023/04/04 11:15:35 UTC
[incubator-celeborn] branch main updated: [CELEBORN-505] Fix typo of SHUFFLE_CHUCK_SIZE (#1411)
This is an automated email from the ASF dual-hosted git repository.
ethanfeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new cad2836e8 [CELEBORN-505] Fix typo of SHUFFLE_CHUCK_SIZE (#1411)
cad2836e8 is described below
commit cad2836e855f9e740c1ac34238689cfaa0d82f29
Author: Angerszhuuuu <an...@gmail.com>
AuthorDate: Tue Apr 4 19:15:30 2023 +0800
[CELEBORN-505] Fix typo of SHUFFLE_CHUCK_SIZE (#1411)
---
common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala | 6 +++---
docs/configuration/client.md | 2 +-
docs/configuration/worker.md | 2 +-
.../celeborn/service/deploy/worker/storage/FileWriterSuiteJ.java | 2 +-
4 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
index 8a92c8e14..fdc9c3eb3 100644
--- a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
+++ b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
@@ -526,7 +526,7 @@ class CelebornConf(loadDefaults: Boolean) extends Cloneable with Logging with Se
def shuffleForceFallbackEnabled: Boolean = get(SHUFFLE_FORCE_FALLBACK_ENABLED)
def shuffleForceFallbackPartitionThreshold: Long = get(SHUFFLE_FORCE_FALLBACK_PARTITION_THRESHOLD)
def shuffleManagerPort: Int = get(SHUFFLE_MANAGER_PORT)
- def shuffleChunkSize: Long = get(SHUFFLE_CHUCK_SIZE)
+ def shuffleChunkSize: Long = get(SHUFFLE_CHUNK_SIZE)
def registerShuffleMaxRetry: Int = get(SHUFFLE_REGISTER_MAX_RETRIES)
def registerShuffleRetryWaitMs: Long = get(SHUFFLE_REGISTER_RETRY_WAIT)
def reserveSlotsMaxRetries: Int = get(RESERVE_SLOTS_MAX_RETRIES)
@@ -1442,8 +1442,8 @@ object CelebornConf extends Logging {
.booleanConf
.createWithDefault(true)
- val SHUFFLE_CHUCK_SIZE: ConfigEntry[Long] =
- buildConf("celeborn.shuffle.chuck.size")
+ val SHUFFLE_CHUNK_SIZE: ConfigEntry[Long] =
+ buildConf("celeborn.shuffle.chunk.size")
.withAlternative("rss.chunk.size")
.categories("client", "worker")
.version("0.2.0")
diff --git a/docs/configuration/client.md b/docs/configuration/client.md
index 103da136b..064eda1c7 100644
--- a/docs/configuration/client.md
+++ b/docs/configuration/client.md
@@ -64,7 +64,7 @@ license: |
| celeborn.shuffle.batchHandleReleasePartition.enabled | false | When true, LifecycleManager will handle release partition request in batch. Otherwise, LifecycleManager will process release partition request immediately | 0.3.0 |
| celeborn.shuffle.batchHandleReleasePartition.interval | 5s | Interval for LifecycleManager to schedule handling release partition requests in batch. | 0.3.0 |
| celeborn.shuffle.batchHandleReleasePartition.threads | 8 | Threads number for LifecycleManager to handle release partition request in batch. | 0.3.0 |
-| celeborn.shuffle.chuck.size | 8m | Max chunk size of reducer's merged shuffle data. For example, if a reducer's shuffle data is 128M and the data will need 16 fetch chunk requests to fetch. | 0.2.0 |
+| celeborn.shuffle.chunk.size | 8m | Max chunk size of reducer's merged shuffle data. For example, if a reducer's shuffle data is 128M and the data will need 16 fetch chunk requests to fetch. | 0.2.0 |
| celeborn.shuffle.compression.codec | LZ4 | The codec used to compress shuffle data. By default, Celeborn provides two codecs: `lz4` and `zstd`. | 0.2.0 |
| celeborn.shuffle.compression.zstd.level | 1 | Compression level for Zstd compression codec, its value should be an integer between -5 and 22. Increasing the compression level will result in better compression at the expense of more CPU and memory. | 0.2.0 |
| celeborn.shuffle.expired.checkInterval | 60s | Interval for client to check expired shuffles. | 0.2.0 |
diff --git a/docs/configuration/worker.md b/docs/configuration/worker.md
index 50b0e11ab..da1101376 100644
--- a/docs/configuration/worker.md
+++ b/docs/configuration/worker.md
@@ -27,7 +27,7 @@ license: |
| celeborn.metrics.extraLabels | | If default metric labels are not enough, extra metric labels can be customized. Labels' pattern is: `<label1_key>=<label1_value>[,<label2_key>=<label2_value>]*`; e.g. `env=prod,version=1` | 0.3.0 |
| celeborn.metrics.sample.rate | 1.0 | It controls if Celeborn collect timer metrics for some operations. Its value should be in [0.0, 1.0]. | 0.2.0 |
| celeborn.metrics.timer.slidingWindow.size | 4096 | The sliding window size of timer metric. | 0.2.0 |
-| celeborn.shuffle.chuck.size | 8m | Max chunk size of reducer's merged shuffle data. For example, if a reducer's shuffle data is 128M and the data will need 16 fetch chunk requests to fetch. | 0.2.0 |
+| celeborn.shuffle.chunk.size | 8m | Max chunk size of reducer's merged shuffle data. For example, if a reducer's shuffle data is 128M and the data will need 16 fetch chunk requests to fetch. | 0.2.0 |
| celeborn.shuffle.minPartitionSizeToEstimate | 8mb | Ignore partition size smaller than this configuration of partition size for estimation. | 0.2.0 |
| celeborn.shuffle.partitionSplit.min | 1m | Min size for a partition to split | 0.2.0 |
| celeborn.storage.hdfs.dir | <undefined> | HDFS dir configuration for Celeborn to access HDFS. | 0.2.0 |
diff --git a/worker/src/test/java/org/apache/celeborn/service/deploy/worker/storage/FileWriterSuiteJ.java b/worker/src/test/java/org/apache/celeborn/service/deploy/worker/storage/FileWriterSuiteJ.java
index c36982644..ab8b9d026 100644
--- a/worker/src/test/java/org/apache/celeborn/service/deploy/worker/storage/FileWriterSuiteJ.java
+++ b/worker/src/test/java/org/apache/celeborn/service/deploy/worker/storage/FileWriterSuiteJ.java
@@ -103,7 +103,7 @@ public class FileWriterSuiteJ {
@BeforeClass
public static void beforeAll() {
tempDir = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "celeborn");
- CONF.set("celeborn.shuffle.chuck.size", "1k");
+ CONF.set("celeborn.shuffle.chunk.size", "1k");
source = Mockito.mock(WorkerSource.class);
Mockito.doAnswer(