You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@celeborn.apache.org by "waitinfuture (via GitHub)" <gi...@apache.org> on 2023/05/24 16:01:39 UTC

[GitHub] [incubator-celeborn] waitinfuture commented on a diff in pull request #1501: [CELEBORN-595] Rename and refactor the configuration doc.

waitinfuture commented on code in PR #1501:
URL: https://github.com/apache/incubator-celeborn/pull/1501#discussion_r1204412373


##########
common/src/main/scala/org/apache/celeborn/common/internal/config/ConfigEntry.scala:
##########
@@ -19,6 +19,7 @@ package org.apache.celeborn.common.internal.config
 
 import java.util.concurrent.ConcurrentHashMap
 
+import org.apache.celeborn.common.CelebornConf

Review Comment:
   Seems L20 and L22 are unused.



##########
common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala:
##########
@@ -1130,10 +1131,11 @@ object CelebornConf extends Logging {
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("30s")
 
-  val HA_CLIENT_RPC_ASK_TIMEOUT: ConfigEntry[Long] =
-    buildConf("celeborn.rpc.haClient.askTimeout")
+  val CLUSTER_CLIENT_RPC_ASK_TIMEOUT: ConfigEntry[Long] =
+    buildConf("celeborn.cluster.client.rpc.askTimeout")

Review Comment:
   I think ```celeborn.cluster.client.rpc.askTimeout``` is even harder to understand. This config is used in ```RssHARetryClient```, seems the original name is fine.



##########
common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala:
##########
@@ -1247,6 +1249,18 @@ object CelebornConf extends Logging {
       .booleanConf
       .createWithDefault(false)
 
+  val MAX_CHUNKS_BEING_TRANSFERRED: ConfigEntry[Long] =
+    buildConf("celeborn.<module>.io.maxChunksBeingTransferred")

Review Comment:
   Is there more than one ```<module>``` here?



##########
common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala:
##########
@@ -1247,6 +1249,18 @@ object CelebornConf extends Logging {
       .booleanConf
       .createWithDefault(false)
 
+  val MAX_CHUNKS_BEING_TRANSFERRED: ConfigEntry[Long] =
+    buildConf("celeborn.<module>.io.maxChunksBeingTransferred")
+      .withAlternative("rss.shuffle.maxChunksBeingTransferred")
+      .categories("network")
+      .doc("The max number of chunks allowed to be transferred at the same time on shuffle service. Note " +
+        "that new incoming connections will be closed when the max number is hit. The client will retry " +
+        "according to the shuffle retry configs (see `celeborn.shuffle.io.maxRetries` and " +

Review Comment:
   ```celeborn.shuffle.io.maxRetries``` does not exist.



##########
common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala:
##########
@@ -1370,23 +1437,183 @@ object CelebornConf extends Logging {
       .createWithDefault(false)
 
   val PUSH_TIMEOUT_CHECK_INTERVAL: ConfigEntry[Long] =
-    buildConf("celeborn.push.timeoutCheck.interval")
-      .categories("client", "worker")
+    buildConf("celeborn.channel.push.timeoutCheck.interval")
+      .withAlternative("celeborn.push.timeoutCheck.interval")
+      .categories("network")
       .doc("Interval for checking push data timeout.")
       .version("0.3.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("5s")
 
   val PUSH_TIMEOUT_CHECK_THREADS: ConfigEntry[Int] =
-    buildConf("celeborn.push.timeoutCheck.threads")
-      .categories("client", "worker")
+    buildConf("celeborn.channel.push.timeoutCheck.threads")
+      .withAlternative("celeborn.push.timeoutCheck.threads")
+      .categories("network")
       .doc("Threads num for checking push data timeout.")
       .version("0.3.0")
       .intConf
       .createWithDefault(16)
 
+  val PUSH_STAGE_END_TIMEOUT: ConfigEntry[Long] =
+    buildConf("celeborn.client.push.stageEnd.timeout")
+      .withAlternative("celeborn.push.stageEnd.timeout")
+      .withAlternative("rss.stage.end.timeout")
+      .categories("client")
+      .doc(s"Timeout for waiting StageEnd. " +
+        s"During this process, there are `${COMMIT_FILE_REQUEST_MAX_RETRY.key}` times for retry opportunities for committing files" +
+        s"and 1 times for releasing slots request. User can customize this value according to your setting. " +
+        s"By default, the value is the max timeout value `${NETWORK_IO_CONNECTION_TIMEOUT.key}`.")
+      .version("0.2.0")
+      .fallbackConf(NETWORK_IO_CONNECTION_TIMEOUT)
+
+  val PUSH_LIMIT_STRATEGY: ConfigEntry[String] =
+    buildConf("celeborn.client.push.limit.strategy")
+      .withAlternative("celeborn.push.limit.strategy")
+      .categories("client")
+      .doc("The strategy used to control the push speed. " +
+        "Valid strategies are SIMPLE and SLOWSTART. the SLOWSTART strategy is usually cooperate with " +
+        "congest control mechanism in the worker side.")
+      .version("0.3.0")
+      .stringConf
+      .transform(_.toUpperCase(Locale.ROOT))
+      .checkValues(Set("SIMPLE", "SLOWSTART"))
+      .createWithDefaultString("SIMPLE")
+
+  val PUSH_SLOW_START_INITIAL_SLEEP_TIME: ConfigEntry[Long] =
+    buildConf("celeborn.client.push.slowStart.initialSleepTime")
+      .withAlternative("celeborn.push.slowStart.initialSleepTime")

Review Comment:
   ditto



##########
common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala:
##########
@@ -1469,7 +1733,8 @@ object CelebornConf extends Logging {
       .createWithDefault(false)
 
   val WORKER_EXCLUDED_EXPIRE_TIMEOUT: ConfigEntry[Long] =
-    buildConf("celeborn.worker.excluded.expireTimeout")
+    buildConf("celeborn.client.worker.excluded.expireTimeout")

Review Comment:
   Better to use ```celeborn.client.excludedWorker.expireTimeout```



##########
common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala:
##########
@@ -1485,18 +1750,31 @@ object CelebornConf extends Logging {
       .booleanConf
       .createWithDefault(true)
 
+  val DFS_SHUFFLE_CHUNK_SIZE: ConfigEntry[Long] =
+    buildConf("celeborn.client.shuffle.dfs.read.chunk.size")
+      .withAlternative("celeborn.shuffle.chunk.size")
+      .withAlternative("rss.chunk.size")
+      .categories("client")
+      .version("0.2.0")
+      .doc("Max chunk size of reducer's merged shuffle data. For example, if a reducer's " +
+        "shuffle data is 128M and the data will need 16 fetch chunk requests to fetch.")
+      .bytesConf(ByteUnit.BYTE)
+      .createWithDefaultString("8m")
+
   val SHUFFLE_CHUNK_SIZE: ConfigEntry[Long] =
-    buildConf("celeborn.shuffle.chunk.size")
+    buildConf("celeborn.worker.shuffle.chunk.size")

Review Comment:
   celeborn.worker.chunk.size



##########
common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala:
##########
@@ -1370,23 +1437,183 @@ object CelebornConf extends Logging {
       .createWithDefault(false)
 
   val PUSH_TIMEOUT_CHECK_INTERVAL: ConfigEntry[Long] =
-    buildConf("celeborn.push.timeoutCheck.interval")
-      .categories("client", "worker")
+    buildConf("celeborn.channel.push.timeoutCheck.interval")

Review Comment:
   The namespace of ```celeborn.channel``` is not easy to understand.



##########
common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala:
##########
@@ -1247,6 +1249,18 @@ object CelebornConf extends Logging {
       .booleanConf
       .createWithDefault(false)
 
+  val MAX_CHUNKS_BEING_TRANSFERRED: ConfigEntry[Long] =
+    buildConf("celeborn.<module>.io.maxChunksBeingTransferred")
+      .withAlternative("rss.shuffle.maxChunksBeingTransferred")
+      .categories("network")
+      .doc("The max number of chunks allowed to be transferred at the same time on shuffle service. Note " +
+        "that new incoming connections will be closed when the max number is hit. The client will retry " +
+        "according to the shuffle retry configs (see `celeborn.shuffle.io.maxRetries` and " +
+        "`celeborn.shuffle.io.retryWait`), if those limits are reached the task will fail with fetch failure.")

Review Comment:
   ```celeborn.shuffle.io.retryWait``` does not exist.



##########
common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala:
##########
@@ -1370,23 +1437,183 @@ object CelebornConf extends Logging {
       .createWithDefault(false)
 
   val PUSH_TIMEOUT_CHECK_INTERVAL: ConfigEntry[Long] =
-    buildConf("celeborn.push.timeoutCheck.interval")
-      .categories("client", "worker")
+    buildConf("celeborn.channel.push.timeoutCheck.interval")
+      .withAlternative("celeborn.push.timeoutCheck.interval")
+      .categories("network")
       .doc("Interval for checking push data timeout.")
       .version("0.3.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("5s")
 
   val PUSH_TIMEOUT_CHECK_THREADS: ConfigEntry[Int] =
-    buildConf("celeborn.push.timeoutCheck.threads")
-      .categories("client", "worker")
+    buildConf("celeborn.channel.push.timeoutCheck.threads")
+      .withAlternative("celeborn.push.timeoutCheck.threads")
+      .categories("network")
       .doc("Threads num for checking push data timeout.")
       .version("0.3.0")
       .intConf
       .createWithDefault(16)
 
+  val PUSH_STAGE_END_TIMEOUT: ConfigEntry[Long] =
+    buildConf("celeborn.client.push.stageEnd.timeout")
+      .withAlternative("celeborn.push.stageEnd.timeout")
+      .withAlternative("rss.stage.end.timeout")
+      .categories("client")
+      .doc(s"Timeout for waiting StageEnd. " +
+        s"During this process, there are `${COMMIT_FILE_REQUEST_MAX_RETRY.key}` times for retry opportunities for committing files" +
+        s"and 1 times for releasing slots request. User can customize this value according to your setting. " +
+        s"By default, the value is the max timeout value `${NETWORK_IO_CONNECTION_TIMEOUT.key}`.")
+      .version("0.2.0")
+      .fallbackConf(NETWORK_IO_CONNECTION_TIMEOUT)
+
+  val PUSH_LIMIT_STRATEGY: ConfigEntry[String] =
+    buildConf("celeborn.client.push.limit.strategy")
+      .withAlternative("celeborn.push.limit.strategy")

Review Comment:
   ditto



##########
common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala:
##########
@@ -1309,16 +1314,74 @@ object CelebornConf extends Logging {
       .booleanConf
       .createWithDefault(false)
 
+  val RESERVE_SLOTS_RACKAWARE_ENABLED: ConfigEntry[Boolean] =
+    buildConf("celeborn.client.reserveSlots.rackware.enabled")
+      .categories("client")
+      .version("0.3.0")
+      .doc("Whether need to place different replicates on different racks when allocating slots.")
+      .booleanConf
+      .createWithDefault(false)
+
+  val RESERVE_SLOTS_MAX_RETRIES: ConfigEntry[Int] =
+    buildConf("celeborn.client.reserveSlots.maxRetries")
+      .withAlternative("celeborn.slots.reserve.maxRetries")
+      .withAlternative("rss.reserve.slots.max.retry")
+      .categories("client")
+      .version("0.2.0")
+      .doc("Max retry times for client to reserve slots.")
+      .intConf
+      .createWithDefault(3)
+
+  val RESERVE_SLOTS_RETRY_WAIT: ConfigEntry[Long] =
+    buildConf("celeborn.client.reserveSlots.retryWait")
+      .withAlternative("celeborn.slots.reserve.retryWait")
+      .withAlternative("rss.reserve.slots.retry.wait")
+      .categories("client")
+      .version("0.2.0")
+      .doc("Wait time before next retry if reserve slots failed.")
+      .timeConf(TimeUnit.MILLISECONDS)
+      .createWithDefaultString("3s")
+
+  val COMMIT_FILE_REQUEST_MAX_RETRY: ConfigEntry[Int] =
+    buildConf("celeborn.client.rpc.requestCommitFiles.maxRetries")
+      .withAlternative("celeborn.rpc.requestCommitFiles.maxRetries")

Review Comment:
   No need alternative for 0.3.0



##########
common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala:
##########
@@ -1259,21 +1273,10 @@ object CelebornConf extends Logging {
       .bytesConf(ByteUnit.BYTE)
       .createWithDefaultString("2m")
 
-  val MAX_CHUNKS_BEING_TRANSFERRED: ConfigEntry[Long] =
-    buildConf("celeborn.shuffle.maxChunksBeingTransferred")
-      .withAlternative("rss.shuffle.maxChunksBeingTransferred")
-      .categories("network")
-      .doc("The max number of chunks allowed to be transferred at the same time on shuffle service. Note " +
-        "that new incoming connections will be closed when the max number is hit. The client will retry " +
-        "according to the shuffle retry configs (see `celeborn.shuffle.io.maxRetries` and " +
-        "`celeborn.shuffle.io.retryWait`), if those limits are reached the task will fail with fetch failure.")
-      .version("0.2.0")
-      .longConf
-      .createWithDefault(Long.MaxValue)
-
   val MASTER_ENDPOINTS: ConfigEntry[Seq[String]] =
-    buildConf("celeborn.master.endpoints")
-      .categories("client", "worker")
+    buildConf("celeborn.cluster.master.endpoints")

Review Comment:
   I'm not sure if adding ```cluster``` is a good idea, because ```master``` is already straightforward.



##########
common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala:
##########
@@ -1370,23 +1437,183 @@ object CelebornConf extends Logging {
       .createWithDefault(false)
 
   val PUSH_TIMEOUT_CHECK_INTERVAL: ConfigEntry[Long] =
-    buildConf("celeborn.push.timeoutCheck.interval")
-      .categories("client", "worker")
+    buildConf("celeborn.channel.push.timeoutCheck.interval")
+      .withAlternative("celeborn.push.timeoutCheck.interval")
+      .categories("network")
       .doc("Interval for checking push data timeout.")
       .version("0.3.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("5s")
 
   val PUSH_TIMEOUT_CHECK_THREADS: ConfigEntry[Int] =
-    buildConf("celeborn.push.timeoutCheck.threads")
-      .categories("client", "worker")
+    buildConf("celeborn.channel.push.timeoutCheck.threads")
+      .withAlternative("celeborn.push.timeoutCheck.threads")
+      .categories("network")
       .doc("Threads num for checking push data timeout.")
       .version("0.3.0")
       .intConf
       .createWithDefault(16)
 
+  val PUSH_STAGE_END_TIMEOUT: ConfigEntry[Long] =
+    buildConf("celeborn.client.push.stageEnd.timeout")
+      .withAlternative("celeborn.push.stageEnd.timeout")
+      .withAlternative("rss.stage.end.timeout")
+      .categories("client")
+      .doc(s"Timeout for waiting StageEnd. " +
+        s"During this process, there are `${COMMIT_FILE_REQUEST_MAX_RETRY.key}` times for retry opportunities for committing files" +
+        s"and 1 times for releasing slots request. User can customize this value according to your setting. " +
+        s"By default, the value is the max timeout value `${NETWORK_IO_CONNECTION_TIMEOUT.key}`.")
+      .version("0.2.0")
+      .fallbackConf(NETWORK_IO_CONNECTION_TIMEOUT)
+
+  val PUSH_LIMIT_STRATEGY: ConfigEntry[String] =
+    buildConf("celeborn.client.push.limit.strategy")
+      .withAlternative("celeborn.push.limit.strategy")
+      .categories("client")
+      .doc("The strategy used to control the push speed. " +
+        "Valid strategies are SIMPLE and SLOWSTART. the SLOWSTART strategy is usually cooperate with " +
+        "congest control mechanism in the worker side.")
+      .version("0.3.0")
+      .stringConf
+      .transform(_.toUpperCase(Locale.ROOT))
+      .checkValues(Set("SIMPLE", "SLOWSTART"))
+      .createWithDefaultString("SIMPLE")
+
+  val PUSH_SLOW_START_INITIAL_SLEEP_TIME: ConfigEntry[Long] =
+    buildConf("celeborn.client.push.slowStart.initialSleepTime")
+      .withAlternative("celeborn.push.slowStart.initialSleepTime")
+      .categories("client")
+      .version("0.3.0")
+      .doc(s"The initial sleep time if the current max in flight requests is 0")
+      .timeConf(TimeUnit.MILLISECONDS)
+      .createWithDefaultString("500ms")
+
+  val PUSH_SLOW_START_MAX_SLEEP_TIME: ConfigEntry[Long] =
+    buildConf("celeborn.client.push.slowStart.maxSleepTime")
+      .withAlternative("celeborn.push.slowStart.maxSleepTime")

Review Comment:
   ditto



##########
common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala:
##########
@@ -1485,18 +1750,31 @@ object CelebornConf extends Logging {
       .booleanConf
       .createWithDefault(true)
 
+  val DFS_SHUFFLE_CHUNK_SIZE: ConfigEntry[Long] =
+    buildConf("celeborn.client.shuffle.dfs.read.chunk.size")

Review Comment:
   celeborn.client.read.dfs.chunk.size



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@celeborn.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org