You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by zh...@apache.org on 2023/05/30 07:14:17 UTC
[incubator-celeborn] branch main updated: [CELEBORN-595] Rename and refactor the configuration doc. (#1501)
This is an automated email from the ASF dual-hosted git repository.
zhouky pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 62681ba85 [CELEBORN-595] Rename and refactor the configuration doc. (#1501)
62681ba85 is described below
commit 62681ba85de35b00d1a360e5ea1bf5363634184c
Author: Angerszhuuuu <an...@gmail.com>
AuthorDate: Tue May 30 15:14:12 2023 +0800
[CELEBORN-595] Rename and refactor the configuration doc. (#1501)
---
.../shuffle/celeborn/RssShuffleWriterSuiteJ.java | 18 +-
.../shuffle/celeborn/RssShuffleManagerSuite.scala | 9 +-
.../shuffle/celeborn/RssShuffleWriterSuiteJ.java | 18 +-
.../shuffle/celeborn/RssShuffleManagerSuite.scala | 9 +-
.../celeborn/client/ShuffleClientBaseSuiteJ.java | 6 +-
.../celeborn/client/ShuffleClientSuiteJ.java | 6 +-
.../celeborn/client/write/DataPushQueueSuitJ.java | 2 +-
.../celeborn/common/haclient/RssHARetryClient.java | 4 +-
.../common/network/util/TransportConf.java | 17 +-
.../org/apache/celeborn/common/CelebornConf.scala | 540 +++++++++++++--------
.../common/haclient/RssHARetryClientSuiteJ.java | 10 +-
.../common/write/SlowStartPushStrategyTest.java | 18 +-
.../org/apache/celeborn/ConfigurationSuite.scala | 4 +
.../apache/celeborn/common/CelebornConfSuite.scala | 48 +-
conf/celeborn-defaults.conf.template | 6 +-
docs/configuration/client.md | 124 +++--
docs/configuration/columnar-shuffle.md | 12 +-
docs/configuration/{columnar-shuffle.md => ha.md} | 12 +-
docs/configuration/index.md | 30 +-
docs/configuration/master.md | 41 +-
docs/configuration/metrics.md | 11 +-
docs/configuration/network.md | 9 +-
docs/configuration/worker.md | 58 +--
.../celeborn/service/deploy/master/Master.scala | 4 +-
.../master/clustermeta/ha/RatisBaseSuiteJ.java | 2 +-
.../ha/RatisMasterStatusSystemSuiteJ.java | 6 +-
.../deploy/master/AppDiskUsageMetricSuite.scala | 4 +-
.../service/deploy/master/MasterSuite.scala | 12 +-
.../celeborn/tests/client/ShuffleClientSuite.scala | 10 +-
.../celeborn/tests/spark/PushDataTimeoutTest.scala | 15 +-
.../tests/spark/RetryCommitFilesTest.scala | 3 +-
.../celeborn/tests/spark/RetryReviveTest.scala | 3 +-
.../tests/spark/RssNonPipelineSortSuite.scala | 5 +-
.../tests/spark/RssPipelineSortSuite.scala | 5 +-
.../tests/spark/ShuffleFallbackSuite.scala | 5 +-
.../celeborn/tests/spark/SkewJoinSuite.scala | 9 +-
.../celeborn/tests/spark/SparkTestBase.scala | 7 +-
toolkit/scripts/genConfs.py | 6 +-
.../service/deploy/worker/FetchHandler.scala | 13 +-
.../deploy/worker/storage/DeviceMonitor.scala | 4 +-
.../deploy/worker/storage/StorageManager.scala | 6 +-
.../worker/storage/CreditStreamManagerSuiteJ.java | 18 +-
.../deploy/worker/storage/FileWriterSuiteJ.java | 20 +-
.../storage/MapPartitionFileWriterSuiteJ.java | 18 +-
.../worker/storage/PartitionFilesSorterSuiteJ.java | 22 +-
.../celeborn/service/deploy/HeartbeatFeature.scala | 22 +-
.../service/deploy/MiniClusterFeature.scala | 14 +-
.../service/deploy/cluster/ReadWriteTestBase.scala | 8 +-
.../deploy/worker/storage/WorkerSuite.scala | 4 +-
49 files changed, 686 insertions(+), 571 deletions(-)
diff --git a/client-spark/spark-2/src/test/java/org/apache/spark/shuffle/celeborn/RssShuffleWriterSuiteJ.java b/client-spark/spark-2/src/test/java/org/apache/spark/shuffle/celeborn/RssShuffleWriterSuiteJ.java
index f71a34f7a..a14d0d567 100644
--- a/client-spark/spark-2/src/test/java/org/apache/spark/shuffle/celeborn/RssShuffleWriterSuiteJ.java
+++ b/client-spark/spark-2/src/test/java/org/apache/spark/shuffle/celeborn/RssShuffleWriterSuiteJ.java
@@ -172,7 +172,8 @@ public class RssShuffleWriterSuiteJ {
@Test
public void testMergeSmallBlock() throws Exception {
final KryoSerializer serializer = new KryoSerializer(sparkConf);
- final CelebornConf conf = new CelebornConf().set("celeborn.push.buffer.max.size", "1024");
+ final CelebornConf conf =
+ new CelebornConf().set(CelebornConf.PUSH_BUFFER_MAX_SIZE().key(), "1024");
check(10000, conf, serializer, true);
check(10000, conf, serializer, false);
}
@@ -180,7 +181,8 @@ public class RssShuffleWriterSuiteJ {
@Test
public void testMergeSmallBlockWithFastWrite() throws Exception {
final UnsafeRowSerializer serializer = new UnsafeRowSerializer(2, null);
- final CelebornConf conf = new CelebornConf().set("celeborn.push.buffer.max.size", "1024");
+ final CelebornConf conf =
+ new CelebornConf().set(CelebornConf.PUSH_BUFFER_MAX_SIZE().key(), "1024");
check(10000, conf, serializer, true);
check(10000, conf, serializer, false);
}
@@ -188,7 +190,8 @@ public class RssShuffleWriterSuiteJ {
@Test
public void testGiantRecord() throws Exception {
final KryoSerializer serializer = new KryoSerializer(sparkConf);
- final CelebornConf conf = new CelebornConf().set("celeborn.push.buffer.max.size", "5");
+ final CelebornConf conf =
+ new CelebornConf().set(CelebornConf.PUSH_BUFFER_MAX_SIZE().key(), "5");
check(10000, conf, serializer, true);
check(10000, conf, serializer, false);
}
@@ -196,7 +199,8 @@ public class RssShuffleWriterSuiteJ {
@Test
public void testGiantRecordWithFastWrite() throws Exception {
final UnsafeRowSerializer serializer = new UnsafeRowSerializer(2, null);
- final CelebornConf conf = new CelebornConf().set("celeborn.push.buffer.max.size", "5");
+ final CelebornConf conf =
+ new CelebornConf().set(CelebornConf.PUSH_BUFFER_MAX_SIZE().key(), "5");
check(10000, conf, serializer, true);
check(10000, conf, serializer, false);
}
@@ -204,7 +208,8 @@ public class RssShuffleWriterSuiteJ {
@Test
public void testGiantRecordAndMergeSmallBlock() throws Exception {
final KryoSerializer serializer = new KryoSerializer(sparkConf);
- final CelebornConf conf = new CelebornConf().set("celeborn.push.buffer.max.size", "128");
+ final CelebornConf conf =
+ new CelebornConf().set(CelebornConf.PUSH_BUFFER_MAX_SIZE().key(), "128");
check(2 << 30, conf, serializer, true);
check(2 << 30, conf, serializer, false);
}
@@ -212,7 +217,8 @@ public class RssShuffleWriterSuiteJ {
@Test
public void testGiantRecordAndMergeSmallBlockWithFastWrite() throws Exception {
final UnsafeRowSerializer serializer = new UnsafeRowSerializer(2, null);
- final CelebornConf conf = new CelebornConf().set("celeborn.push.buffer.max.size", "128");
+ final CelebornConf conf =
+ new CelebornConf().set(CelebornConf.PUSH_BUFFER_MAX_SIZE().key(), "128");
check(2 << 30, conf, serializer, true);
check(2 << 30, conf, serializer, false);
}
diff --git a/client-spark/spark-2/src/test/scala/org/apache/spark/shuffle/celeborn/RssShuffleManagerSuite.scala b/client-spark/spark-2/src/test/scala/org/apache/spark/shuffle/celeborn/RssShuffleManagerSuite.scala
index a63e28c2b..e6c8a4171 100644
--- a/client-spark/spark-2/src/test/scala/org/apache/spark/shuffle/celeborn/RssShuffleManagerSuite.scala
+++ b/client-spark/spark-2/src/test/scala/org/apache/spark/shuffle/celeborn/RssShuffleManagerSuite.scala
@@ -22,6 +22,7 @@ import org.junit
import org.junit.runner.RunWith
import org.junit.runners.JUnit4
+import org.apache.celeborn.common.CelebornConf
import org.apache.celeborn.common.internal.Logging
@RunWith(classOf[JUnit4])
@@ -31,8 +32,8 @@ class RssShuffleManagerSuite extends Logging {
def testFallBack(): Unit = {
val conf = new SparkConf().setIfMissing("spark.master", "local")
.setIfMissing("spark.shuffle.manager", "org.apache.spark.shuffle.celeborn.RssShuffleManager")
- .set("spark.celeborn.master.endpoints", "localhost:9097")
- .set("spark.celeborn.push.replicate.enabled", "false")
+ .set(s"spark.${CelebornConf.MASTER_ENDPOINTS.key}", "localhost:9097")
+ .set(s"spark.${CelebornConf.PUSH_REPLICATE_ENABLED.key}", "false")
.set("spark.shuffle.service.enabled", "false")
.set("spark.shuffle.useOldFetchProtocol", "true")
.set("spark.sql.adaptive.enabled", "true")
@@ -49,10 +50,10 @@ class RssShuffleManagerSuite extends Logging {
def testClusterNotAvailable(): Unit = {
val conf = new SparkConf().setIfMissing("spark.master", "local")
.setIfMissing("spark.shuffle.manager", "org.apache.spark.shuffle.celeborn.RssShuffleManager")
- .set("spark.celeborn.master.endpoints", "localhost:9097")
+ .set(s"spark.${CelebornConf.MASTER_ENDPOINTS.key}", "localhost:9097")
+ .set(s"spark.${CelebornConf.PUSH_REPLICATE_ENABLED.key}", "false")
.set("spark.shuffle.service.enabled", "false")
.set("spark.shuffle.useOldFetchProtocol", "true")
- .set("spark.celeborn.push.replicate.enabled", "false")
.set("spark.sql.adaptive.enabled", "false")
.setAppName("test")
val sc = new SparkContext(conf)
diff --git a/client-spark/spark-3/src/test/java/org/apache/spark/shuffle/celeborn/RssShuffleWriterSuiteJ.java b/client-spark/spark-3/src/test/java/org/apache/spark/shuffle/celeborn/RssShuffleWriterSuiteJ.java
index 02cff40a6..340168372 100644
--- a/client-spark/spark-3/src/test/java/org/apache/spark/shuffle/celeborn/RssShuffleWriterSuiteJ.java
+++ b/client-spark/spark-3/src/test/java/org/apache/spark/shuffle/celeborn/RssShuffleWriterSuiteJ.java
@@ -158,42 +158,48 @@ public class RssShuffleWriterSuiteJ {
@Test
public void testMergeSmallBlock() throws Exception {
final KryoSerializer serializer = new KryoSerializer(sparkConf);
- final CelebornConf conf = new CelebornConf().set("celeborn.push.buffer.max.size", "1024");
+ final CelebornConf conf =
+ new CelebornConf().set(CelebornConf.PUSH_BUFFER_MAX_SIZE().key(), "1024");
check(10000, conf, serializer);
}
@Test
public void testMergeSmallBlockWithFastWrite() throws Exception {
final UnsafeRowSerializer serializer = new UnsafeRowSerializer(2, null);
- final CelebornConf conf = new CelebornConf().set("celeborn.push.buffer.max.size", "1024");
+ final CelebornConf conf =
+ new CelebornConf().set(CelebornConf.PUSH_BUFFER_MAX_SIZE().key(), "1024");
check(10000, conf, serializer);
}
@Test
public void testGiantRecord() throws Exception {
final KryoSerializer serializer = new KryoSerializer(sparkConf);
- final CelebornConf conf = new CelebornConf().set("celeborn.push.buffer.max.size", "5");
+ final CelebornConf conf =
+ new CelebornConf().set(CelebornConf.PUSH_BUFFER_MAX_SIZE().key(), "5");
check(10000, conf, serializer);
}
@Test
public void testGiantRecordWithFastWrite() throws Exception {
final UnsafeRowSerializer serializer = new UnsafeRowSerializer(2, null);
- final CelebornConf conf = new CelebornConf().set("celeborn.push.buffer.max.size", "5");
+ final CelebornConf conf =
+ new CelebornConf().set(CelebornConf.PUSH_BUFFER_MAX_SIZE().key(), "5");
check(10000, conf, serializer);
}
@Test
public void testGiantRecordAndMergeSmallBlock() throws Exception {
final KryoSerializer serializer = new KryoSerializer(sparkConf);
- final CelebornConf conf = new CelebornConf().set("celeborn.push.buffer.max.size", "128");
+ final CelebornConf conf =
+ new CelebornConf().set(CelebornConf.PUSH_BUFFER_MAX_SIZE().key(), "128");
check(2 << 30, conf, serializer);
}
@Test
public void testGiantRecordAndMergeSmallBlockWithFastWrite() throws Exception {
final UnsafeRowSerializer serializer = new UnsafeRowSerializer(2, null);
- final CelebornConf conf = new CelebornConf().set("celeborn.push.buffer.max.size", "128");
+ final CelebornConf conf =
+ new CelebornConf().set(CelebornConf.PUSH_BUFFER_MAX_SIZE().key(), "128");
check(2 << 30, conf, serializer);
}
diff --git a/client-spark/spark-3/src/test/scala/org/apache/spark/shuffle/celeborn/RssShuffleManagerSuite.scala b/client-spark/spark-3/src/test/scala/org/apache/spark/shuffle/celeborn/RssShuffleManagerSuite.scala
index df24675ce..6e2377ec6 100644
--- a/client-spark/spark-3/src/test/scala/org/apache/spark/shuffle/celeborn/RssShuffleManagerSuite.scala
+++ b/client-spark/spark-3/src/test/scala/org/apache/spark/shuffle/celeborn/RssShuffleManagerSuite.scala
@@ -22,6 +22,7 @@ import org.junit
import org.junit.runner.RunWith
import org.junit.runners.JUnit4
+import org.apache.celeborn.common.CelebornConf
import org.apache.celeborn.common.internal.Logging
@RunWith(classOf[JUnit4])
@@ -30,10 +31,10 @@ class RssShuffleManagerSuite extends Logging {
def testFallBack(): Unit = {
val conf = new SparkConf().setIfMissing("spark.master", "local")
.setIfMissing("spark.shuffle.manager", "org.apache.spark.shuffle.celeborn.RssShuffleManager")
- .set("spark.celeborn.master.endpoints", "localhost:9097")
+ .set(s"spark.${CelebornConf.MASTER_ENDPOINTS.key}", "localhost:9097")
+ .set(s"spark.${CelebornConf.PUSH_REPLICATE_ENABLED.key}", "false")
.set("spark.shuffle.service.enabled", "false")
.set("spark.shuffle.useOldFetchProtocol", "true")
- .set("spark.celeborn.push.replicate.enabled", "false")
.set("spark.sql.adaptive.enabled", "true")
.setAppName("test")
val sc = new SparkContext(conf)
@@ -48,10 +49,10 @@ class RssShuffleManagerSuite extends Logging {
def testClusterNotAvailable(): Unit = {
val conf = new SparkConf().setIfMissing("spark.master", "local")
.setIfMissing("spark.shuffle.manager", "org.apache.spark.shuffle.celeborn.RssShuffleManager")
- .set("spark.celeborn.master.endpoints", "localhost:9097")
+ .set(s"spark.${CelebornConf.MASTER_ENDPOINTS.key}", "localhost:9097")
+ .set(s"spark.${CelebornConf.PUSH_REPLICATE_ENABLED.key}", "false")
.set("spark.shuffle.service.enabled", "false")
.set("spark.shuffle.useOldFetchProtocol", "true")
- .set("spark.celeborn.push.replicate.enabled", "false")
.set("spark.sql.adaptive.enabled", "false")
.setAppName("test")
val sc = new SparkContext(conf)
diff --git a/client/src/test/java/org/apache/celeborn/client/ShuffleClientBaseSuiteJ.java b/client/src/test/java/org/apache/celeborn/client/ShuffleClientBaseSuiteJ.java
index 482d9820f..d2536c064 100644
--- a/client/src/test/java/org/apache/celeborn/client/ShuffleClientBaseSuiteJ.java
+++ b/client/src/test/java/org/apache/celeborn/client/ShuffleClientBaseSuiteJ.java
@@ -82,9 +82,9 @@ public abstract class ShuffleClientBaseSuiteJ {
protected CelebornConf setupEnv(CompressionCodec codec) throws IOException, InterruptedException {
CelebornConf conf = new CelebornConf();
- conf.set("celeborn.shuffle.compression.codec", codec.name());
- conf.set("celeborn.push.retry.threads", "1");
- conf.set("celeborn.push.buffer.max.size", "1K");
+ conf.set(CelebornConf.SHUFFLE_COMPRESSION_CODEC().key(), codec.name());
+ conf.set(CelebornConf.PUSH_RETRY_THREADS().key(), "1");
+ conf.set(CelebornConf.PUSH_BUFFER_MAX_SIZE().key(), "1K");
shuffleClient = new ShuffleClientImpl(conf, new UserIdentifier("mock", "mock"));
masterLocation.setPeer(slaveLocation);
diff --git a/client/src/test/java/org/apache/celeborn/client/ShuffleClientSuiteJ.java b/client/src/test/java/org/apache/celeborn/client/ShuffleClientSuiteJ.java
index 10dd5e96a..f623374b2 100644
--- a/client/src/test/java/org/apache/celeborn/client/ShuffleClientSuiteJ.java
+++ b/client/src/test/java/org/apache/celeborn/client/ShuffleClientSuiteJ.java
@@ -174,9 +174,9 @@ public class ShuffleClientSuiteJ {
private CelebornConf setupEnv(CompressionCodec codec) throws IOException, InterruptedException {
CelebornConf conf = new CelebornConf();
- conf.set("celeborn.shuffle.compression.codec", codec.name());
- conf.set("celeborn.push.retry.threads", "1");
- conf.set("celeborn.push.buffer.max.size", "1K");
+ conf.set(CelebornConf.SHUFFLE_COMPRESSION_CODEC().key(), codec.name());
+ conf.set(CelebornConf.PUSH_RETRY_THREADS().key(), "1");
+ conf.set(CelebornConf.PUSH_BUFFER_MAX_SIZE().key(), "1K");
shuffleClient = new ShuffleClientImpl(conf, new UserIdentifier("mock", "mock"));
masterLocation.setPeer(slaveLocation);
diff --git a/client/src/test/java/org/apache/celeborn/client/write/DataPushQueueSuitJ.java b/client/src/test/java/org/apache/celeborn/client/write/DataPushQueueSuitJ.java
index 70850377d..307819200 100644
--- a/client/src/test/java/org/apache/celeborn/client/write/DataPushQueueSuitJ.java
+++ b/client/src/test/java/org/apache/celeborn/client/write/DataPushQueueSuitJ.java
@@ -80,7 +80,7 @@ public class DataPushQueueSuitJ {
Map<Integer, Integer> partitionBatchIdMap = new HashMap<>();
CelebornConf conf = new CelebornConf();
- conf.set("celeborn.push.maxReqsInFlight", "2");
+ conf.set(CelebornConf.PUSH_MAX_REQS_IN_FLIGHT().key(), "2");
String app = "APP-1";
int shuffleId = 0;
diff --git a/common/src/main/java/org/apache/celeborn/common/haclient/RssHARetryClient.java b/common/src/main/java/org/apache/celeborn/common/haclient/RssHARetryClient.java
index 3b5063a1f..360de2af0 100644
--- a/common/src/main/java/org/apache/celeborn/common/haclient/RssHARetryClient.java
+++ b/common/src/main/java/org/apache/celeborn/common/haclient/RssHARetryClient.java
@@ -63,8 +63,8 @@ public class RssHARetryClient {
public RssHARetryClient(RpcEnv rpcEnv, CelebornConf conf) {
this.rpcEnv = rpcEnv;
this.masterEndpoints = conf.masterEndpoints();
- this.maxRetries = Math.max(masterEndpoints.length, conf.clientMaxTries());
- this.rpcTimeout = conf.haClientRpcAskTimeout();
+ this.maxRetries = Math.max(masterEndpoints.length, conf.masterClientMaxRetries());
+ this.rpcTimeout = conf.masterClientRpcAskTimeout();
this.rpcEndpointRef = new AtomicReference<>();
this.oneWayMessageSender = ThreadUtils.newDaemonSingleThreadExecutor("One-Way-Message-Sender");
}
diff --git a/common/src/main/java/org/apache/celeborn/common/network/util/TransportConf.java b/common/src/main/java/org/apache/celeborn/common/network/util/TransportConf.java
index 1e863a2b3..29d7e5dc6 100644
--- a/common/src/main/java/org/apache/celeborn/common/network/util/TransportConf.java
+++ b/common/src/main/java/org/apache/celeborn/common/network/util/TransportConf.java
@@ -130,30 +130,19 @@ public class TransportConf {
return celebornConf.networkIoVerboseMetrics(module);
}
- /**
- * The max number of chunks allowed to be transferred at the same time on shuffle service. Note
- * that new incoming connections will be closed when the max number is hit. The client will retry
- * according to the shuffle retry configs (see `celeborn.shuffle.io.maxRetries` and
- * `celeborn.shuffle.io.retryWait`), if those limits are reached the task will fail with fetch
- * failure.
- */
- public long maxChunksBeingTransferred() {
- return celebornConf.networkIoMaxChunksBeingTransferred(module);
- }
-
public CelebornConf getCelebornConf() {
return celebornConf;
}
public int pushDataTimeoutCheckerThreads() {
- return celebornConf.pushDataTimeoutCheckerThreads();
+ return celebornConf.pushDataTimeoutCheckerThreads(module);
}
public long pushDataTimeoutCheckIntervalMs() {
- return celebornConf.pushTimeoutCheckInterval();
+ return celebornConf.pushDataTimeoutCheckInterval(module);
}
public long clientHearbeatInterval() {
- return celebornConf.clientHeartbeatInterval();
+ return celebornConf.clientHeartbeatInterval(module);
}
}
diff --git a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
index eea6ea8fb..8a7defc5c 100644
--- a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
+++ b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
@@ -380,9 +380,6 @@ class CelebornConf(loadDefaults: Boolean) extends Cloneable with Logging with Se
new RpcTimeout(get(RPC_LOOKUP_TIMEOUT).milli, RPC_LOOKUP_TIMEOUT.key)
def rpcAskTimeout: RpcTimeout =
new RpcTimeout(get(RPC_ASK_TIMEOUT).milli, RPC_ASK_TIMEOUT.key)
- def haClientRpcAskTimeout: RpcTimeout =
- new RpcTimeout(get(HA_CLIENT_RPC_ASK_TIMEOUT).milli, HA_CLIENT_RPC_ASK_TIMEOUT.key)
-
def networkIoMode(module: String): String = {
val key = NETWORK_IO_MODE.key.replace("<module>", module)
get(key, NETWORK_IO_MODE.defaultValue.get)
@@ -458,9 +455,8 @@ class CelebornConf(loadDefaults: Boolean) extends Cloneable with Logging with Se
getBoolean(key, NETWORK_VERBOSE_METRICS.defaultValue.get)
}
- def networkIoMaxChunksBeingTransferred(module: String): Long = {
- val key = MAX_CHUNKS_BEING_TRANSFERRED.key.replace("<module>", module)
- getSizeAsBytes(key, MAX_CHUNKS_BEING_TRANSFERRED.defaultValueString)
+ def shuffleIoMaxChunksBeingTransferred: Long = {
+ get(MAX_CHUNKS_BEING_TRANSFERRED)
}
// //////////////////////////////////////////////////////
@@ -475,15 +471,12 @@ class CelebornConf(loadDefaults: Boolean) extends Cloneable with Logging with Se
get(SLOTS_ASSIGN_LOADAWARE_FETCHTIME_WEIGHT)
def slotsAssignExtraSlots: Int = get(SLOTS_ASSIGN_EXTRA_SLOTS)
def slotsAssignPolicy: SlotsAssignPolicy = SlotsAssignPolicy.valueOf(get(SLOTS_ASSIGN_POLICY))
- def initialEstimatedPartitionSize: Long = get(SHUFFLE_INITIAL_ESRIMATED_PARTITION_SIZE)
+ def initialEstimatedPartitionSize: Long = get(SHUFFLE_INITIAL_ESTIMATED_PARTITION_SIZE)
def estimatedPartitionSizeUpdaterInitialDelay: Long =
- get(SHUFFLE_ESTIMATED_PARTITION_SIZE_UPDATE_INITIAL_DELAY)
+ get(ESTIMATED_PARTITION_SIZE_UPDATE_INITIAL_DELAY)
def estimatedPartitionSizeForEstimationUpdateInterval: Long =
- get(SHUFFLE_ESTIMATED_PARTITION_SIZE_UPDATE_INTERVAL)
- def metricsAppTopDiskUsageCount: Int = get(METRICS_APP_TOP_DISK_USAGE_COUNT)
- def metricsAppTopDiskUsageWindowSize: Int = get(METRICS_APP_TOP_DISK_USAGE_WINDOW_SIZE)
- def metricsAppTopDiskUsageInterval: Long = get(METRICS_APP_TOP_DISK_USAGE_INTERVAL)
- def metricsResourceConsumptionInterval: Long = get(METRICS_RESOURCE_CONSUMPTION_INTERVAL)
+ get(ESTIMATED_PARTITION_SIZE_UPDATE_INTERVAL)
+ def masterResourceConsumptionInterval: Long = get(MASTER_RESOURCE_CONSUMPTION_INTERVAL)
// //////////////////////////////////////////////////////
// Worker //
@@ -499,20 +492,19 @@ class CelebornConf(loadDefaults: Boolean) extends Cloneable with Logging with Se
def workerReplicateIoThreads: Option[Int] = get(WORKER_REPLICATE_IO_THREADS)
def registerWorkerTimeout: Long = get(WORKER_REGISTER_TIMEOUT)
- def workerNonEmptyDirExpireDuration: Long = get(WORKER_NON_EMPTY_DIR_EXPIRE_DURATION)
+ def workerApplicationDataKeepAliveTime: Long = get(WORKER_APPLICATION_DATA_KEEP_ALIVE_TIME)
def workerWorkingDir: String = get(WORKER_WORKING_DIR)
def workerCloseIdleConnections: Boolean = get(WORKER_CLOSE_IDLE_CONNECTIONS)
def workerReplicateFastFailDuration: Long = get(WORKER_REPLICATE_FAST_FAIL_DURATION)
def workerReplicateRandomConnectionEnabled: Boolean =
get(WORKER_REPLICATE_RANDOM_CONNECTION_ENABLED)
- def workerDeviceStatusCheckTimeout: Long = get(WORKER_DEVICE_STATUS_CHECK_TIMEOUT)
def workerCheckFileCleanMaxRetries: Int = get(WORKER_CHECK_FILE_CLEAN_MAX_RETRIES)
def workerCheckFileCleanTimeout: Long = get(WORKER_CHECK_FILE_CLEAN_TIMEOUT)
def workerHeartbeatTimeout: Long = get(WORKER_HEARTBEAT_TIMEOUT)
def workerReplicateThreads: Int = get(WORKER_REPLICATE_THREADS)
def workerCommitThreads: Int = get(WORKER_COMMIT_THREADS)
def workerShuffleCommitTimeout: Long = get(WORKER_SHUFFLE_COMMIT_TIMEOUT)
- def minPartitionSizeToEstimate: Long = get(SHUFFLE_MIN_PARTITION_SIZE_TO_ESTIMATE)
+ def minPartitionSizeToEstimate: Long = get(MIN_PARTITION_SIZE_TO_ESTIMATE)
def partitionSorterSortPartitionTimeout: Long = get(PARTITION_SORTER_SORT_TIMEOUT)
def partitionSorterReservedMemoryPerPartition: Long =
get(PARTITION_SORTER_PER_PARTITION_RESERVED_MEMORY)
@@ -525,7 +517,6 @@ class CelebornConf(loadDefaults: Boolean) extends Cloneable with Logging with Se
// //////////////////////////////////////////////////////
// Client //
// //////////////////////////////////////////////////////
- def clientMaxTries: Int = get(CLIENT_MAX_RETRIES)
def clientCloseIdleConnections: Boolean = get(CLIENT_CLOSE_IDLE_CONNECTIONS)
def shuffleWriterMode: ShuffleMode = ShuffleMode.valueOf(get(SHUFFLE_WRITER_MODE))
def shuffleForceFallbackEnabled: Boolean = get(SHUFFLE_FORCE_FALLBACK_ENABLED)
@@ -548,7 +539,10 @@ class CelebornConf(loadDefaults: Boolean) extends Cloneable with Logging with Se
def shufflePartitionType: PartitionType = PartitionType.valueOf(get(SHUFFLE_PARTITION_TYPE))
def requestCommitFilesMaxRetries: Int = get(COMMIT_FILE_REQUEST_MAX_RETRY)
def shuffleClientPushBlacklistEnabled: Boolean = get(SHUFFLE_CLIENT_PUSH_BLACKLIST_ENABLED)
- def clientHeartbeatInterval: Long = get(CLIENT_HEARTBEAT_INTERVAL)
+ def clientHeartbeatInterval(module: String): Long = {
+ val key = CHANNEL_HEARTBEAT_INTERVAL.key.replace("<module>", module)
+ getTimeAsMs(key, CHANNEL_HEARTBEAT_INTERVAL.defaultValueString)
+ }
// //////////////////////////////////////////////////////
// Shuffle Compression //
// //////////////////////////////////////////////////////
@@ -567,6 +561,11 @@ class CelebornConf(loadDefaults: Boolean) extends Cloneable with Logging with Se
}
}
+ def masterClientRpcAskTimeout: RpcTimeout =
+ new RpcTimeout(get(MASTER_CLIENT_RPC_ASK_TIMEOUT).milli, MASTER_CLIENT_RPC_ASK_TIMEOUT.key)
+
+ def masterClientMaxRetries: Int = get(MASTER_CLIENT_MAX_RETRIES)
+
def masterHost: String = get(MASTER_HOST)
def masterPort: Int = get(MASTER_PORT)
@@ -652,6 +651,9 @@ class CelebornConf(loadDefaults: Boolean) extends Cloneable with Logging with Se
def workerPrometheusMetricPort: Int = get(WORKER_PROMETHEUS_PORT)
def metricsExtraLabels: Map[String, String] =
get(METRICS_EXTRA_LABELS).map(Utils.parseMetricLabels(_)).toMap
+ def metricsAppTopDiskUsageCount: Int = get(METRICS_APP_TOP_DISK_USAGE_COUNT)
+ def metricsAppTopDiskUsageWindowSize: Int = get(METRICS_APP_TOP_DISK_USAGE_WINDOW_SIZE)
+ def metricsAppTopDiskUsageInterval: Long = get(METRICS_APP_TOP_DISK_USAGE_INTERVAL)
// //////////////////////////////////////////////////////
// Quota //
@@ -733,8 +735,14 @@ class CelebornConf(loadDefaults: Boolean) extends Cloneable with Logging with Se
def rpcCacheConcurrencyLevel: Int = get(RPC_CACHE_CONCURRENCY_LEVEL)
def rpcCacheExpireTime: Long = get(RPC_CACHE_EXPIRE_TIME)
def pushDataTimeoutMs: Long = get(PUSH_DATA_TIMEOUT)
- def pushDataTimeoutCheckerThreads: Int = get(PUSH_TIMEOUT_CHECK_THREADS)
- def pushTimeoutCheckInterval: Long = get(PUSH_TIMEOUT_CHECK_INTERVAL)
+ def pushDataTimeoutCheckerThreads(module: String): Int = {
+ val key = PUSH_TIMEOUT_CHECK_THREADS.key.replace("<module>", module)
+ getInt(key, PUSH_TIMEOUT_CHECK_THREADS.defaultValue.get)
+ }
+ def pushDataTimeoutCheckInterval(module: String): Long = {
+ val key = PUSH_TIMEOUT_CHECK_INTERVAL.key.replace("<module>", module)
+ getTimeAsMs(key, PUSH_TIMEOUT_CHECK_INTERVAL.defaultValueString)
+ }
// //////////////////////////////////////////////////////
// Graceful Shutdown & Recover //
@@ -743,7 +751,7 @@ class CelebornConf(loadDefaults: Boolean) extends Cloneable with Logging with Se
def shutdownTimeoutMs: Long = get(WORKER_GRACEFUL_SHUTDOWN_TIMEOUT)
def checkSlotsFinishedInterval: Long = get(WORKER_CHECK_SLOTS_FINISHED_INTERVAL)
def checkSlotsFinishedTimeoutMs: Long = get(WORKER_CHECK_SLOTS_FINISHED_TIMEOUT)
- def workerRecoverPath: String = get(WORKER_RECOVER_PATH)
+ def workerRecoverPath: String = get(WORKER_GRACEFUL_SHUTDOWN_RECOVER_PATH)
def partitionSorterCloseAwaitTimeMs: Long = get(PARTITION_SORTER_SHUTDOWN_TIMEOUT)
def workerFlusherShutdownTimeoutMs: Long = get(WORKER_FLUSHER_SHUTDOWN_TIMEOUT)
@@ -768,6 +776,7 @@ class CelebornConf(loadDefaults: Boolean) extends Cloneable with Logging with Se
def diskMonitorNotifyErrorThreshold: Int = get(WORKER_DISK_MONITOR_NOTIFY_ERROR_THRESHOLD)
def diskMonitorNotifyErrorExpireTimeout: Long =
get(WORKER_DISK_MONITOR_NOTIFY_ERROR_EXPIRE_TIMEOUT)
+ def diskMonitorStatusCheckTimeout: Long = get(WORKER_DEVICE_STATUS_CHECK_TIMEOUT)
def createWriterMaxAttempts: Int = get(WORKER_WRITER_CREATE_MAX_ATTEMPTS)
def workerStorageBaseDirPrefix: String = get(WORKER_STORAGE_BASE_DIR_PREFIX)
def workerStorageBaseDirNumber: Int = get(WORKER_STORAGE_BASE_DIR_COUNT)
@@ -1146,10 +1155,12 @@ object CelebornConf extends Logging {
.timeConf(TimeUnit.MILLISECONDS)
.createWithDefaultString("30s")
- val HA_CLIENT_RPC_ASK_TIMEOUT: ConfigEntry[Long] =
- buildConf("celeborn.rpc.haClient.askTimeout")
+ val MASTER_CLIENT_RPC_ASK_TIMEOUT: ConfigEntry[Long] =
+ buildConf("celeborn.masterClient.rpc.askTimeout")
+ .withAlternative("celeborn.rpc.haClient.askTimeout")
.withAlternative("rss.haclient.rpc.askTimeout")
- .categories("network")
+ .internal
+ .categories("client", "worker")
.version("0.2.0")
.doc("Timeout for HA client RPC ask operations.")
.fallbackConf(RPC_ASK_TIMEOUT)
@@ -1276,13 +1287,13 @@ object CelebornConf extends Logging {
.createWithDefaultString("2m")
val MAX_CHUNKS_BEING_TRANSFERRED: ConfigEntry[Long] =
- buildConf("celeborn.shuffle.maxChunksBeingTransferred")
+ buildConf("celeborn.shuffle.io.maxChunksBeingTransferred")
.withAlternative("rss.shuffle.maxChunksBeingTransferred")
.categories("network")
.doc("The max number of chunks allowed to be transferred at the same time on shuffle service. Note " +
"that new incoming connections will be closed when the max number is hit. The client will retry " +
- "according to the shuffle retry configs (see `celeborn.shuffle.io.maxRetries` and " +
- "`celeborn.shuffle.io.retryWait`), if those limits are reached the task will fail with fetch failure.")
+ "according to the shuffle retry configs (see `celeborn.<module>.io.maxRetries` and " +
+ "`celeborn.<module>.io.retryWait`), if those limits are reached the task will fail with fetch failure.")
.version("0.2.0")
.longConf
.createWithDefault(Long.MaxValue)
@@ -1303,7 +1314,8 @@ object CelebornConf extends Logging {
.createWithDefaultString(s"<localhost>:9097")
val SHUFFLE_WRITER_MODE: ConfigEntry[String] =
- buildConf("celeborn.shuffle.writer")
+ buildConf("celeborn.client.shuffle.writer")
+ .withAlternative("celeborn.shuffle.writer")
.withAlternative("rss.shuffle.writer.mode")
.categories("client")
.doc("Celeborn supports the following kind of shuffle writers. 1. hash: hash-based shuffle writer " +
@@ -1316,7 +1328,8 @@ object CelebornConf extends Logging {
.createWithDefault(ShuffleMode.HASH.name)
val PUSH_REPLICATE_ENABLED: ConfigEntry[Boolean] =
- buildConf("celeborn.push.replicate.enabled")
+ buildConf("celeborn.client.push.replicate.enabled")
+ .withAlternative("celeborn.push.replicate.enabled")
.withAlternative("rss.push.data.replicate")
.categories("client")
.doc("When true, Celeborn worker will replicate shuffle data to another Celeborn worker " +
@@ -1326,7 +1339,8 @@ object CelebornConf extends Logging {
.createWithDefault(false)
val PUSH_BUFFER_INITIAL_SIZE: ConfigEntry[Long] =
- buildConf("celeborn.push.buffer.initial.size")
+ buildConf("celeborn.client.push.buffer.initial.size")
+ .withAlternative("celeborn.push.buffer.initial.size")
.withAlternative("rss.push.data.buffer.initial.size")
.categories("client")
.version("0.2.0")
@@ -1334,7 +1348,8 @@ object CelebornConf extends Logging {
.createWithDefaultString("8k")
val PUSH_BUFFER_MAX_SIZE: ConfigEntry[Long] =
- buildConf("celeborn.push.buffer.max.size")
+ buildConf("celeborn.client.push.buffer.max.size")
+ .withAlternative("celeborn.push.buffer.max.size")
.withAlternative("rss.push.data.buffer.size")
.categories("client")
.version("0.2.0")
@@ -1347,7 +1362,8 @@ object CelebornConf extends Logging {
.createWithDefaultString("64k")
val PUSH_QUEUE_CAPACITY: ConfigEntry[Int] =
- buildConf("celeborn.push.queue.capacity")
+ buildConf("celeborn.client.push.queue.capacity")
+ .withAlternative("celeborn.push.queue.capacity")
.withAlternative("rss.push.data.queue.capacity")
.categories("client")
.version("0.2.0")
@@ -1358,7 +1374,8 @@ object CelebornConf extends Logging {
.createWithDefault(512)
val PUSH_MAX_REQS_IN_FLIGHT: ConfigEntry[Int] =
- buildConf("celeborn.push.maxReqsInFlight")
+ buildConf("celeborn.client.push.maxReqsInFlight")
+ .withAlternative("celeborn.push.maxReqsInFlight")
.withAlternative("rss.push.data.maxReqsInFlight")
.categories("client")
.version("0.2.0")
@@ -1369,7 +1386,7 @@ object CelebornConf extends Logging {
.createWithDefault(4)
val PUSH_MAX_REVIVE_TIMES: ConfigEntry[Int] =
- buildConf("celeborn.push.revive.maxRetries")
+ buildConf("celeborn.client.push.revive.maxRetries")
.categories("client")
.version("0.3.0")
.doc("Max retry times for reviving when celeborn push data failed.")
@@ -1377,32 +1394,42 @@ object CelebornConf extends Logging {
.createWithDefault(5)
val TEST_RETRY_REVIVE: ConfigEntry[Boolean] =
- buildConf("celeborn.test.retryRevive")
+ buildConf("celeborn.test.client.retryRevive")
+ .withAlternative("celeborn.test.retryRevive")
.internal
- .categories("client", "test")
+ .categories("test", "client")
.doc("Fail push data and request for test")
.version("0.2.0")
.booleanConf
.createWithDefault(false)
val PUSH_TIMEOUT_CHECK_INTERVAL: ConfigEntry[Long] =
- buildConf("celeborn.push.timeoutCheck.interval")
- .categories("client", "worker")
- .doc("Interval for checking push data timeout.")
+ buildConf("celeborn.<module>.push.timeoutCheck.interval")
+ .categories("network")
+ .doc("Interval for checking push data timeout. " +
+ s"If setting <module> to `${TransportModuleConstants.DATA_MODULE}`, " +
+ s"it works for shuffle client push data and should be configured in client side. " +
+ s"If setting <module> to `${TransportModuleConstants.REPLICATE_MODULE}`, " +
+ s"it works for worker replicate data to peer worker and should be configured in worker side.")
.version("0.3.0")
.timeConf(TimeUnit.MILLISECONDS)
.createWithDefaultString("5s")
val PUSH_TIMEOUT_CHECK_THREADS: ConfigEntry[Int] =
- buildConf("celeborn.push.timeoutCheck.threads")
- .categories("client", "worker")
- .doc("Threads num for checking push data timeout.")
+ buildConf("celeborn.<module>.push.timeoutCheck.threads")
+ .categories("network")
+ .doc("Threads num for checking push data timeout. " +
+ s"If setting <module> to `${TransportModuleConstants.DATA_MODULE}`, " +
+ s"it works for shuffle client push data and should be configured in client side. " +
+ s"If setting <module> to `${TransportModuleConstants.REPLICATE_MODULE}`, " +
+ s"it works for worker replicate data to peer worker and should be configured in worker side.")
.version("0.3.0")
.intConf
.createWithDefault(16)
val FETCH_TIMEOUT: ConfigEntry[Long] =
- buildConf("celeborn.fetch.timeout")
+ buildConf("celeborn.client.fetch.timeout")
+ .withAlternative("celeborn.fetch.timeout")
.withAlternative("rss.fetch.chunk.timeout")
.categories("client")
.version("0.2.0")
@@ -1411,7 +1438,8 @@ object CelebornConf extends Logging {
.createWithDefaultString("30s")
val FETCH_MAX_REQS_IN_FLIGHT: ConfigEntry[Int] =
- buildConf("celeborn.fetch.maxReqsInFlight")
+ buildConf("celeborn.client.fetch.maxReqsInFlight")
+ .withAlternative("celeborn.fetch.maxReqsInFlight")
.withAlternative("rss.fetch.chunk.maxReqsInFlight")
.categories("client")
.version("0.2.0")
@@ -1420,7 +1448,8 @@ object CelebornConf extends Logging {
.createWithDefault(3)
val CLIENT_RPC_MAX_PARALLELISM: ConfigEntry[Int] =
- buildConf("celeborn.rpc.maxParallelism")
+ buildConf("celeborn.client.rpc.maxParallelism")
+ .withAlternative("celeborn.rpc.maxParallelism")
.withAlternative("rss.rpc.max.parallelism")
.categories("client")
.version("0.2.0")
@@ -1429,7 +1458,8 @@ object CelebornConf extends Logging {
.createWithDefault(1024)
val FETCH_MAX_RETRIES_FOR_EACH_REPLICA: ConfigEntry[Int] =
- buildConf("celeborn.fetch.maxRetriesForEachReplica")
+ buildConf("celeborn.client.fetch.maxRetriesForEachReplica")
+ .withAlternative("celeborn.fetch.maxRetriesForEachReplica")
.withAlternative("celeborn.fetch.maxRetries")
.categories("client")
.version("0.2.0")
@@ -1438,16 +1468,18 @@ object CelebornConf extends Logging {
.createWithDefault(3)
val TEST_FETCH_FAILURE: ConfigEntry[Boolean] =
- buildConf("celeborn.test.fetchFailure")
+ buildConf("celeborn.test.client.fetchFailure")
+ .withAlternative("celeborn.test.fetchFailure")
.internal
- .categories("client", "test")
+ .categories("test", "client")
.version("0.2.0")
.doc("Whether to test fetch chunk failure")
.booleanConf
.createWithDefault(false)
val APPLICATION_HEARTBEAT_TIMEOUT: ConfigEntry[Long] =
- buildConf("celeborn.application.heartbeat.timeout")
+ buildConf("celeborn.master.heartbeat.application.timeout")
+ .withAlternative("celeborn.application.heartbeat.timeout")
.withAlternative("rss.application.timeout")
.categories("master")
.version("0.2.0")
@@ -1456,7 +1488,8 @@ object CelebornConf extends Logging {
.createWithDefaultString("300s")
val APPLICATION_HEARTBEAT_INTERVAL: ConfigEntry[Long] =
- buildConf("celeborn.application.heartbeatInterval")
+ buildConf("celeborn.client.application.heartbeatInterval")
+ .withAlternative("celeborn.application.heartbeatInterval")
.withAlternative("rss.application.heartbeatInterval")
.categories("client")
.version("0.2.0")
@@ -1465,7 +1498,8 @@ object CelebornConf extends Logging {
.createWithDefaultString("10s")
val SHUFFLE_EXPIRED_CHECK_INTERVAL: ConfigEntry[Long] =
- buildConf("celeborn.shuffle.expired.checkInterval")
+ buildConf("celeborn.client.shuffle.expired.checkInterval")
+ .withAlternative("celeborn.shuffle.expired.checkInterval")
.withAlternative("rss.remove.shuffle.delay")
.categories("client")
.version("0.2.0")
@@ -1485,7 +1519,8 @@ object CelebornConf extends Logging {
.createWithDefault(false)
val WORKER_EXCLUDED_EXPIRE_TIMEOUT: ConfigEntry[Long] =
- buildConf("celeborn.worker.excluded.expireTimeout")
+ buildConf("celeborn.client.excludedWorker.expireTimeout")
+ .withAlternative("celeborn.worker.excluded.expireTimeout")
.categories("client")
.version("0.2.0")
.doc("Timeout time for LifecycleManager to clear reserved excluded worker.")
@@ -1512,7 +1547,8 @@ object CelebornConf extends Logging {
.createWithDefaultString("8m")
val SHUFFLE_REGISTER_MAX_RETRIES: ConfigEntry[Int] =
- buildConf("celeborn.shuffle.register.maxRetries")
+ buildConf("celeborn.client.registerShuffle.maxRetries")
+ .withAlternative("celeborn.shuffle.register.maxRetries")
.withAlternative("rss.register.shuffle.max.retry")
.categories("client")
.version("0.2.0")
@@ -1521,7 +1557,8 @@ object CelebornConf extends Logging {
.createWithDefault(3)
val SHUFFLE_REGISTER_RETRY_WAIT: ConfigEntry[Long] =
- buildConf("celeborn.shuffle.register.retryWait")
+ buildConf("celeborn.client.registerShuffle.retryWait")
+ .withAlternative("celeborn.shuffle.register.retryWait")
.withAlternative("rss.register.shuffle.retry.wait")
.categories("client")
.version("0.2.0")
@@ -1530,7 +1567,8 @@ object CelebornConf extends Logging {
.createWithDefaultString("3s")
val RESERVE_SLOTS_MAX_RETRIES: ConfigEntry[Int] =
- buildConf("celeborn.slots.reserve.maxRetries")
+ buildConf("celeborn.client.reserveSlots.maxRetries")
+ .withAlternative("celeborn.slots.reserve.maxRetries")
.withAlternative("rss.reserve.slots.max.retry")
.categories("client")
.version("0.2.0")
@@ -1539,7 +1577,8 @@ object CelebornConf extends Logging {
.createWithDefault(3)
val RESERVE_SLOTS_RETRY_WAIT: ConfigEntry[Long] =
- buildConf("celeborn.slots.reserve.retryWait")
+ buildConf("celeborn.client.reserveSlots.retryWait")
+ .withAlternative("celeborn.slots.reserve.retryWait")
.withAlternative("rss.reserve.slots.retry.wait")
.categories("client")
.version("0.2.0")
@@ -1548,18 +1587,19 @@ object CelebornConf extends Logging {
.createWithDefaultString("3s")
val COMMIT_FILE_REQUEST_MAX_RETRY: ConfigEntry[Int] =
- buildConf("celeborn.rpc.requestCommitFiles.maxRetries")
+ buildConf("celeborn.client.rpc.requestCommitFiles.maxRetries")
.categories("client")
.doc("Max retry times for requestCommitFiles RPC.")
- .version("1.0.0")
+ .version("0.3.0")
.intConf
.checkValue(v => v > 0, "value must be positive")
.createWithDefault(2)
val TEST_RETRY_COMMIT_FILE: ConfigEntry[Boolean] =
- buildConf("celeborn.test.retryCommitFiles")
+ buildConf("celeborn.test.client.retryCommitFiles")
+ .withAlternative("celeborn.test.retryCommitFiles")
.internal
- .categories("client", "test")
+ .categories("test", "client")
.doc("Fail commitFile request for test")
.version("0.2.0")
.booleanConf
@@ -1593,13 +1633,14 @@ object CelebornConf extends Logging {
.checkValue(p => p >= 1024 && p < 65535, "invalid port")
.createWithDefault(9097)
- val HA_ENABLED: ConfigEntry[Boolean] = buildConf("celeborn.ha.enabled")
- .withAlternative("rss.ha.enabled")
- .categories("master")
- .version("0.2.0")
- .doc("When true, master nodes run as Raft cluster mode.")
- .booleanConf
- .createWithDefault(false)
+ val HA_ENABLED: ConfigEntry[Boolean] =
+ buildConf("celeborn.ha.enabled")
+ .withAlternative("rss.ha.enabled")
+ .categories("ha")
+ .version("0.2.0")
+ .doc("When true, master nodes run as Raft cluster mode.")
+ .booleanConf
+ .createWithDefault(false)
val HA_MASTER_NODE_ID: OptionalConfigEntry[String] =
buildConf("celeborn.ha.master.node.id")
@@ -1611,7 +1652,7 @@ object CelebornConf extends Logging {
val HA_MASTER_NODE_HOST: ConfigEntry[String] =
buildConf("celeborn.ha.master.node.<id>.host")
- .categories("master")
+ .categories("ha")
.doc("Host to bind of master node <id> in HA mode.")
.version("0.2.0")
.stringConf
@@ -1619,7 +1660,7 @@ object CelebornConf extends Logging {
val HA_MASTER_NODE_PORT: ConfigEntry[Int] =
buildConf("celeborn.ha.master.node.<id>.port")
- .categories("master")
+ .categories("ha")
.doc("Port to bind of master node <id> in HA mode.")
.version("0.2.0")
.intConf
@@ -1629,7 +1670,7 @@ object CelebornConf extends Logging {
val HA_MASTER_NODE_RATIS_HOST: OptionalConfigEntry[String] =
buildConf("celeborn.ha.master.node.<id>.ratis.host")
.internal
- .categories("master")
+ .categories("ha")
.doc("Ratis host to bind of master node <id> in HA mode. If not provided, " +
s"fallback to ${HA_MASTER_NODE_HOST.key}.")
.version("0.2.0")
@@ -1638,7 +1679,7 @@ object CelebornConf extends Logging {
val HA_MASTER_NODE_RATIS_PORT: ConfigEntry[Int] =
buildConf("celeborn.ha.master.node.<id>.ratis.port")
- .categories("master")
+ .categories("ha")
.doc("Ratis port to bind of master node <id> in HA mode.")
.version("0.2.0")
.intConf
@@ -1648,7 +1689,7 @@ object CelebornConf extends Logging {
val HA_MASTER_RATIS_RPC_TYPE: ConfigEntry[String] =
buildConf("celeborn.ha.master.ratis.raft.rpc.type")
.withAlternative("rss.ha.rpc.type")
- .categories("master")
+ .categories("ha")
.doc("RPC type for Ratis, available options: netty, grpc.")
.version("0.2.0")
.stringConf
@@ -1658,7 +1699,7 @@ object CelebornConf extends Logging {
val HA_MASTER_RATIS_STORAGE_DIR: ConfigEntry[String] =
buildConf("celeborn.ha.master.ratis.raft.server.storage.dir")
- .categories("master")
+ .categories("ha")
.withAlternative("rss.ha.storage.dir")
.version("0.2.0")
.stringConf
@@ -1668,7 +1709,7 @@ object CelebornConf extends Logging {
buildConf("celeborn.ha.master.ratis.raft.server.log.segment.size.max")
.withAlternative("rss.ha.ratis.segment.size")
.internal
- .categories("master")
+ .categories("ha")
.version("0.2.0")
.bytesConf(ByteUnit.BYTE)
.createWithDefaultString("4MB")
@@ -1677,7 +1718,7 @@ object CelebornConf extends Logging {
buildConf("celeborn.ha.master.ratis.raft.server.log.preallocated.size")
.withAlternative("rss.ratis.segment.preallocated.size")
.internal
- .categories("master")
+ .categories("ha")
.version("0.2.0")
.bytesConf(ByteUnit.BYTE)
.createWithDefaultString("4MB")
@@ -1686,7 +1727,7 @@ object CelebornConf extends Logging {
buildConf("celeborn.ha.master.ratis.raft.server.log.appender.buffer.element-limit")
.withAlternative("rss.ratis.log.appender.queue.num-elements")
.internal
- .categories("master")
+ .categories("ha")
.version("0.2.0")
.intConf
.createWithDefault(1024)
@@ -1695,7 +1736,7 @@ object CelebornConf extends Logging {
buildConf("celeborn.ha.master.ratis.raft.server.log.appender.buffer.byte-limit")
.withAlternative("rss.ratis.log.appender.queue.byte-limit")
.internal
- .categories("master")
+ .categories("ha")
.version("0.2.0")
.bytesConf(ByteUnit.BYTE)
.createWithDefaultString("32MB")
@@ -1703,7 +1744,7 @@ object CelebornConf extends Logging {
val HA_MASTER_RATIS_LOG_INSTABLL_SNAPSHOT_ENABLED: ConfigEntry[Boolean] =
buildConf("celeborn.ha.master.ratis.raft.server.log.appender.install.snapshot.enabled")
.internal
- .categories("master")
+ .categories("ha")
.version("0.2.0")
.booleanConf
.createWithDefault(true)
@@ -1712,7 +1753,7 @@ object CelebornConf extends Logging {
buildConf("celeborn.ha.master.ratis.raft.server.log.purge.gap")
.withAlternative("rss.ratis.log.purge.gap")
.internal
- .categories("master")
+ .categories("ha")
.version("0.2.0")
.intConf
.createWithDefault(1000000)
@@ -1721,7 +1762,7 @@ object CelebornConf extends Logging {
buildConf("celeborn.ha.master.ratis.raft.server.rpc.request.timeout")
.withAlternative("rss.ratis.server.request.timeout")
.internal
- .categories("master")
+ .categories("ha")
.version("0.2.0")
.timeConf(TimeUnit.SECONDS)
.createWithDefaultString("3s")
@@ -1730,7 +1771,7 @@ object CelebornConf extends Logging {
buildConf("celeborn.ha.master.ratis.raft.server.retrycache.expirytime")
.withAlternative("rss.ratis.server.retry.cache.timeout")
.internal
- .categories("master")
+ .categories("ha")
.version("0.2.0")
.timeConf(TimeUnit.SECONDS)
.createWithDefaultString("600s")
@@ -1739,7 +1780,7 @@ object CelebornConf extends Logging {
buildConf("celeborn.ha.master.ratis.raft.server.rpc.timeout.min")
.withAlternative("rss.ratis.minimum.timeout")
.internal
- .categories("master")
+ .categories("ha")
.version("0.2.0")
.timeConf(TimeUnit.SECONDS)
.createWithDefaultString("3s")
@@ -1747,7 +1788,7 @@ object CelebornConf extends Logging {
val HA_MASTER_RATIS_RPC_TIMEOUT_MAX: ConfigEntry[Long] =
buildConf("celeborn.ha.master.ratis.raft.server.rpc.timeout.max")
.internal
- .categories("master")
+ .categories("ha")
.version("0.2.0")
.timeConf(TimeUnit.SECONDS)
.createWithDefaultString("5s")
@@ -1755,7 +1796,7 @@ object CelebornConf extends Logging {
val HA_MASTER_RATIS_FIRSTELECTION_TIMEOUT_MIN: ConfigEntry[Long] =
buildConf("celeborn.ha.master.ratis.first.election.timeout.min")
.internal
- .categories("master")
+ .categories("ha")
.version("0.3.0")
.timeConf(TimeUnit.SECONDS)
.createWithDefaultString("3s")
@@ -1763,7 +1804,7 @@ object CelebornConf extends Logging {
val HA_MASTER_RATIS_FIRSTELECTION_TIMEOUT_MAX: ConfigEntry[Long] =
buildConf("celeborn.ha.master.ratis.first.election.timeout.max")
.internal
- .categories("master")
+ .categories("ha")
.version("0.3.0")
.timeConf(TimeUnit.SECONDS)
.createWithDefaultString("5s")
@@ -1771,7 +1812,7 @@ object CelebornConf extends Logging {
val HA_MASTER_RATIS_NOTIFICATION_NO_LEADER_TIMEOUT: ConfigEntry[Long] =
buildConf("celeborn.ha.master.ratis.raft.server.notification.no-leader.timeout")
.internal
- .categories("master")
+ .categories("ha")
.version("0.2.0")
.timeConf(TimeUnit.SECONDS)
.createWithDefaultString("30s")
@@ -1780,7 +1821,7 @@ object CelebornConf extends Logging {
buildConf("celeborn.ha.master.ratis.raft.server.rpc.slowness.timeout")
.withAlternative("rss.ratis.server.failure.timeout.duration")
.internal
- .categories("master")
+ .categories("ha")
.version("0.2.0")
.timeConf(TimeUnit.SECONDS)
.createWithDefaultString("120s")
@@ -1789,7 +1830,7 @@ object CelebornConf extends Logging {
buildConf("celeborn.ha.master.ratis.raft.server.role.check.interval")
.withAlternative("rss.ratis.server.role.check.interval")
.internal
- .categories("master")
+ .categories("ha")
.version("0.2.0")
.timeConf(TimeUnit.MILLISECONDS)
.createWithDefaultString("1s")
@@ -1798,7 +1839,7 @@ object CelebornConf extends Logging {
buildConf("celeborn.ha.master.ratis.raft.server.snapshot.auto.trigger.enabled")
.withAlternative("rss.ha.ratis.snapshot.auto.trigger.enabled")
.internal
- .categories("master")
+ .categories("ha")
.version("0.2.0")
.booleanConf
.createWithDefault(true)
@@ -1807,7 +1848,7 @@ object CelebornConf extends Logging {
buildConf("celeborn.ha.master.ratis.raft.server.snapshot.auto.trigger.threshold")
.withAlternative("rss.ha.ratis.snapshot.auto.trigger.threshold")
.internal
- .categories("master")
+ .categories("ha")
.version("0.2.0")
.longConf
.createWithDefault(200000L)
@@ -1816,7 +1857,7 @@ object CelebornConf extends Logging {
buildConf("celeborn.ha.master.ratis.raft.server.snapshot.retention.file.num")
.withAlternative("rss.ratis.snapshot.retention.file.num")
.internal
- .categories("master")
+ .categories("ha")
.version("0.2.0")
.intConf
.createWithDefault(3)
@@ -1839,6 +1880,7 @@ object CelebornConf extends Logging {
val WORKER_STORAGE_BASE_DIR_PREFIX: ConfigEntry[String] =
buildConf("celeborn.worker.storage.baseDir.prefix")
.withAlternative("rss.worker.base.dir.prefix")
+ .internal
.categories("worker")
.version("0.2.0")
.doc("Base directory for Celeborn worker to write if " +
@@ -1849,6 +1891,7 @@ object CelebornConf extends Logging {
val WORKER_STORAGE_BASE_DIR_COUNT: ConfigEntry[Int] =
buildConf("celeborn.worker.storage.baseDir.number")
.withAlternative("rss.worker.base.dir.number")
+ .internal
.categories("worker")
.version("0.2.0")
.doc(s"How many directories will be used if `${WORKER_STORAGE_DIRS.key}` is not set. " +
@@ -1933,8 +1976,9 @@ object CelebornConf extends Logging {
.timeConf(TimeUnit.MILLISECONDS)
.createWithDefaultString("180s")
- val WORKER_NON_EMPTY_DIR_EXPIRE_DURATION: ConfigEntry[Long] =
- buildConf("celeborn.worker.noneEmptyDirExpireDuration")
+ val WORKER_APPLICATION_DATA_KEEP_ALIVE_TIME: ConfigEntry[Long] =
+ buildConf("celeborn.worker.storage.applicationData.keepAliveTime")
+ .withAlternative("celeborn.worker.noneEmptyDirExpireDuration")
.withAlternative("rss.expire.nonEmptyDir.duration")
.categories("worker")
.doc("If a non-empty application shuffle data dir have not been operated during le duration time, will mark this application as expired.")
@@ -1943,7 +1987,8 @@ object CelebornConf extends Logging {
.createWithDefaultString("1d")
val WORKER_WORKING_DIR: ConfigEntry[String] =
- buildConf("celeborn.worker.workingDir")
+ buildConf("celeborn.worker.storage.workingDir")
+ .withAlternative("celeborn.worker.workingDir")
.withAlternative("rss.worker.workingDirName")
.categories("worker")
.doc("Worker's working dir path name.")
@@ -1981,7 +2026,8 @@ object CelebornConf extends Logging {
.createWithDefault(true)
val WORKER_DEVICE_STATUS_CHECK_TIMEOUT: ConfigEntry[Long] =
- buildConf("celeborn.worker.disk.check.timeout")
+ buildConf("celeborn.worker.monitor.disk.check.timeout")
+ .withAlternative("celeborn.worker.disk.check.timeout")
.withAlternative("rss.worker.status.check.timeout")
.categories("worker")
.doc("Timeout time for worker check device status.")
@@ -1990,7 +2036,8 @@ object CelebornConf extends Logging {
.createWithDefaultString("30s")
val WORKER_CHECK_FILE_CLEAN_MAX_RETRIES: ConfigEntry[Int] =
- buildConf("celeborn.worker.disk.checkFileClean.maxRetries")
+ buildConf("celeborn.worker.storage.checkDirsEmpty.maxRetries")
+ .withAlternative("celeborn.worker.disk.checkFileClean.maxRetries")
.withAlternative("rss.worker.checkFileCleanRetryTimes")
.categories("worker")
.doc("The number of retries for a worker to check if the working directory is cleaned up before registering with the master.")
@@ -1999,7 +2046,8 @@ object CelebornConf extends Logging {
.createWithDefault(3)
val WORKER_CHECK_FILE_CLEAN_TIMEOUT: ConfigEntry[Long] =
- buildConf("celeborn.worker.disk.checkFileClean.timeout")
+ buildConf("celeborn.worker.storage.checkDirsEmpty.timeout")
+ .withAlternative("celeborn.worker.disk.checkFileClean.timeout")
.withAlternative("rss.worker.checkFileCleanTimeoutMs")
.categories("worker")
.doc("The wait time per retry for a worker to check if the working directory is cleaned up before registering with the master.")
@@ -2008,9 +2056,10 @@ object CelebornConf extends Logging {
.createWithDefaultString("1000ms")
val WORKER_HEARTBEAT_TIMEOUT: ConfigEntry[Long] =
- buildConf("celeborn.worker.heartbeat.timeout")
+ buildConf("celeborn.master.heartbeat.worker.timeout")
+ .withAlternative("celeborn.worker.heartbeat.timeout")
.withAlternative("rss.worker.timeout")
- .categories("master", "worker")
+ .categories("master")
.version("0.2.0")
.doc("Worker heartbeat timeout.")
.timeConf(TimeUnit.MILLISECONDS)
@@ -2026,7 +2075,8 @@ object CelebornConf extends Logging {
.createWithDefault(64)
val WORKER_COMMIT_THREADS: ConfigEntry[Int] =
- buildConf("celeborn.worker.commit.threads")
+ buildConf("celeborn.worker.commitFiles.threads")
+ .withAlternative("celeborn.worker.commit.threads")
.withAlternative("rss.worker.asyncCommitFiles.numThreads")
.categories("worker")
.version("0.2.0")
@@ -2035,7 +2085,8 @@ object CelebornConf extends Logging {
.createWithDefault(32)
val WORKER_SHUFFLE_COMMIT_TIMEOUT: ConfigEntry[Long] =
- buildConf("celeborn.worker.shuffle.commit.timeout")
+ buildConf("celeborn.worker.commitFiles.timeout")
+ .withAlternative("celeborn.worker.shuffle.commit.timeout")
.withAlternative("rss.flush.timeout")
.categories("worker")
.doc("Timeout for a Celeborn worker to commit files of a shuffle.")
@@ -2043,7 +2094,8 @@ object CelebornConf extends Logging {
.fallbackConf(RPC_ASK_TIMEOUT)
val PARTITION_SORTER_SORT_TIMEOUT: ConfigEntry[Long] =
- buildConf("celeborn.worker.partitionSorter.sort.timeout")
+ buildConf("celeborn.worker.sortPartition.timeout")
+ .withAlternative("celeborn.worker.partitionSorter.sort.timeout")
.withAlternative("rss.partition.sort.timeout")
.categories("worker")
.doc("Timeout for a shuffle file to sort.")
@@ -2052,7 +2104,8 @@ object CelebornConf extends Logging {
.createWithDefaultString("220s")
val PARTITION_SORTER_PER_PARTITION_RESERVED_MEMORY: ConfigEntry[Long] =
- buildConf("celeborn.worker.partitionSorter.reservedMemoryPerPartition")
+ buildConf("celeborn.worker.sortPartition.reservedMemoryPerPartition")
+ .withAlternative("celeborn.worker.partitionSorter.reservedMemoryPerPartition")
.withAlternative("rss.worker.initialReserveSingleSortMemory")
.categories("worker")
.doc("Reserved memory when sorting a shuffle file off-heap.")
@@ -2061,7 +2114,8 @@ object CelebornConf extends Logging {
.createWithDefaultString("1mb")
val PARTITION_SORTER_THREADS: OptionalConfigEntry[Int] =
- buildConf("celeborn.worker.partitionSorter.threads")
+ buildConf("celeborn.worker.sortPartition.threads")
+ .withAlternative("celeborn.worker.partitionSorter.threads")
.categories("worker")
.doc("PartitionSorter's thread counts.")
.version("0.3.0")
@@ -2132,7 +2186,8 @@ object CelebornConf extends Logging {
.createWithDefaultString("3s")
val WORKER_DISK_RESERVE_SIZE: ConfigEntry[Long] =
- buildConf("celeborn.worker.disk.reserve.size")
+ buildConf("celeborn.worker.storage.disk.reserve.size")
+ .withAlternative("celeborn.worker.disk.reserve.size")
.withAlternative("rss.disk.minimum.reserve.size")
.categories("worker")
.doc("Celeborn worker reserved space for each disk.")
@@ -2141,7 +2196,7 @@ object CelebornConf extends Logging {
.createWithDefaultString("5G")
val WORKER_DISKTIME_SLIDINGWINDOW_SIZE: ConfigEntry[Int] =
- buildConf("celeborn.worker.diskTime.slidingWindow.size")
+ buildConf("celeborn.worker.flusher.diskTime.slidingWindow.size")
.withAlternative("celeborn.worker.flusher.avgFlushTime.slidingWindow.size")
.withAlternative("rss.flusher.avg.time.window")
.categories("worker")
@@ -2173,7 +2228,8 @@ object CelebornConf extends Logging {
.createWithDefault(100)
val SLOTS_ASSIGN_LOADAWARE_DISKGROUP_NUM: ConfigEntry[Int] =
- buildConf("celeborn.slots.assign.loadAware.numDiskGroups")
+ buildConf("celeborn.master.slot.assign.loadAware.numDiskGroups")
+ .withAlternative("celeborn.slots.assign.loadAware.numDiskGroups")
.withAlternative("rss.disk.groups")
.categories("master")
.doc("This configuration is a guidance for load-aware slot allocation algorithm. " +
@@ -2183,7 +2239,8 @@ object CelebornConf extends Logging {
.createWithDefault(5)
val SLOTS_ASSIGN_LOADAWARE_DISKGROUP_GRADIENT: ConfigEntry[Double] =
- buildConf("celeborn.slots.assign.loadAware.diskGroupGradient")
+ buildConf("celeborn.master.slot.assign.loadAware.diskGroupGradient")
+ .withAlternative("celeborn.slots.assign.loadAware.diskGroupGradient")
.withAlternative("rss.disk.groups.gradient")
.categories("master")
.doc("This value means how many more workload will be placed into a faster disk group " +
@@ -2193,7 +2250,8 @@ object CelebornConf extends Logging {
.createWithDefault(0.1)
val SLOTS_ASSIGN_LOADAWARE_FLUSHTIME_WEIGHT: ConfigEntry[Double] =
- buildConf("celeborn.slots.assign.loadAware.flushTimeWeight")
+ buildConf("celeborn.master.slot.assign.loadAware.flushTimeWeight")
+ .withAlternative("celeborn.slots.assign.loadAware.flushTimeWeight")
.categories("master")
.doc(
"Weight of average flush time when calculating ordering in load-aware assignment strategy")
@@ -2202,7 +2260,8 @@ object CelebornConf extends Logging {
.createWithDefault(0)
val SLOTS_ASSIGN_LOADAWARE_FETCHTIME_WEIGHT: ConfigEntry[Double] =
- buildConf("celeborn.slots.assign.loadAware.fetchTimeWeight")
+ buildConf("celeborn.master.slot.assign.loadAware.fetchTimeWeight")
+ .withAlternative("celeborn.slots.assign.loadAware.fetchTimeWeight")
.categories("master")
.doc(
"Weight of average fetch time when calculating ordering in load-aware assignment strategy")
@@ -2211,7 +2270,8 @@ object CelebornConf extends Logging {
.createWithDefault(1)
val SLOTS_ASSIGN_EXTRA_SLOTS: ConfigEntry[Int] =
- buildConf("celeborn.slots.assign.extraSlots")
+ buildConf("celeborn.master.slot.assign.extraSlots")
+ .withAlternative("celeborn.slots.assign.extraSlots")
.withAlternative("rss.offer.slots.extra.size")
.categories("master")
.version("0.2.0")
@@ -2220,7 +2280,8 @@ object CelebornConf extends Logging {
.createWithDefault(2)
val SLOTS_ASSIGN_POLICY: ConfigEntry[String] =
- buildConf("celeborn.slots.assign.policy")
+ buildConf("celeborn.master.slot.assign.policy")
+ .withAlternative("celeborn.slots.assign.policy")
.withAlternative("rss.offer.slots.algorithm")
.categories("master")
.version("0.2.0")
@@ -2230,8 +2291,9 @@ object CelebornConf extends Logging {
.checkValues(Set(SlotsAssignPolicy.ROUNDROBIN.name, SlotsAssignPolicy.LOADAWARE.name))
.createWithDefault(SlotsAssignPolicy.ROUNDROBIN.name)
- val SHUFFLE_INITIAL_ESRIMATED_PARTITION_SIZE: ConfigEntry[Long] =
- buildConf("celeborn.shuffle.initialEstimatedPartitionSize")
+ val SHUFFLE_INITIAL_ESTIMATED_PARTITION_SIZE: ConfigEntry[Long] =
+ buildConf("celeborn.master.estimatedPartitionSize.initialSize")
+ .withAlternative("celeborn.shuffle.initialEstimatedPartitionSize")
.withAlternative("rss.initial.partition.size")
.categories("master")
.doc("Initial partition size for estimation, it will change according to runtime stats.")
@@ -2239,8 +2301,9 @@ object CelebornConf extends Logging {
.bytesConf(ByteUnit.BYTE)
.createWithDefaultString("64mb")
- val SHUFFLE_MIN_PARTITION_SIZE_TO_ESTIMATE: ConfigEntry[Long] =
- buildConf("celeborn.shuffle.minPartitionSizeToEstimate")
+ val MIN_PARTITION_SIZE_TO_ESTIMATE: ConfigEntry[Long] =
+ buildConf("celeborn.master.estimatedPartitionSize.minSize")
+ .withAlternative("celeborn.shuffle.minPartitionSizeToEstimate")
.withAlternative("rss.minimum.estimate.partition.size")
.categories("worker")
.doc(
@@ -2249,8 +2312,9 @@ object CelebornConf extends Logging {
.bytesConf(ByteUnit.BYTE)
.createWithDefaultString("8mb")
- val SHUFFLE_ESTIMATED_PARTITION_SIZE_UPDATE_INITIAL_DELAY: ConfigEntry[Long] =
- buildConf("celeborn.shuffle.estimatedPartitionSize.update.initialDelay")
+ val ESTIMATED_PARTITION_SIZE_UPDATE_INITIAL_DELAY: ConfigEntry[Long] =
+ buildConf("celeborn.master.estimatedPartitionSize.update.initialDelay")
+ .withAlternative("celeborn.shuffle.estimatedPartitionSize.update.initialDelay")
.withAlternative("rss.partition.size.update.initial.delay")
.categories("master")
.doc("Initial delay time before start updating partition size for estimation.")
@@ -2258,8 +2322,9 @@ object CelebornConf extends Logging {
.timeConf(TimeUnit.MILLISECONDS)
.createWithDefaultString("5min")
- val SHUFFLE_ESTIMATED_PARTITION_SIZE_UPDATE_INTERVAL: ConfigEntry[Long] =
- buildConf("celeborn.shuffle.estimatedPartitionSize.update.interval")
+ val ESTIMATED_PARTITION_SIZE_UPDATE_INTERVAL: ConfigEntry[Long] =
+ buildConf("celeborn.master.estimatedPartitionSize.update.interval")
+ .withAlternative("celeborn.shuffle.estimatedPartitionSize.update.interval")
.withAlternative("rss.partition.size.update.interval")
.categories("master")
.doc("Interval of updating partition size for estimation.")
@@ -2270,7 +2335,7 @@ object CelebornConf extends Logging {
val METRICS_APP_TOP_DISK_USAGE_COUNT: ConfigEntry[Int] =
buildConf("celeborn.metrics.app.topDiskUsage.count")
.withAlternative("rss.metrics.app.topDiskUsage.count")
- .categories("master")
+ .categories("metrics")
.doc("Size for top items about top disk usage applications list.")
.version("0.2.0")
.intConf
@@ -2279,7 +2344,7 @@ object CelebornConf extends Logging {
val METRICS_APP_TOP_DISK_USAGE_WINDOW_SIZE: ConfigEntry[Int] =
buildConf("celeborn.metrics.app.topDiskUsage.windowSize")
.withAlternative("rss.metrics.app.topDiskUsage.windowSize")
- .categories("master")
+ .categories("metrics")
.doc("Window size about top disk usage application list.")
.version("0.2.0")
.intConf
@@ -2288,14 +2353,14 @@ object CelebornConf extends Logging {
val METRICS_APP_TOP_DISK_USAGE_INTERVAL: ConfigEntry[Long] =
buildConf("celeborn.metrics.app.topDiskUsage.interval")
.withAlternative("rss.metrics.app.topDiskUsage.interval")
- .categories("master")
+ .categories("metrics")
.doc("Time length for a window about top disk usage application list.")
.version("0.2.0")
.timeConf(TimeUnit.SECONDS)
.createWithDefaultString("10min")
- val METRICS_RESOURCE_CONSUMPTION_INTERVAL: ConfigEntry[Long] =
- buildConf("celeborn.metrics.userResourceConsumption.update.interval")
+ val MASTER_RESOURCE_CONSUMPTION_INTERVAL: ConfigEntry[Long] =
+ buildConf("celeborn.master.userResourceConsumption.update.interval")
.categories("master")
.doc("Time length for a window about compute user resource consumption.")
.version("0.3.0")
@@ -2303,7 +2368,8 @@ object CelebornConf extends Logging {
.createWithDefaultString("30s")
val PUSH_STAGE_END_TIMEOUT: ConfigEntry[Long] =
- buildConf("celeborn.push.stageEnd.timeout")
+ buildConf("celeborn.client.push.stageEnd.timeout")
+ .withAlternative("celeborn.push.stageEnd.timeout")
.withAlternative("rss.stage.end.timeout")
.categories("client")
.doc(s"Timeout for waiting StageEnd. " +
@@ -2314,7 +2380,7 @@ object CelebornConf extends Logging {
.fallbackConf(NETWORK_IO_CONNECTION_TIMEOUT)
val PUSH_LIMIT_STRATEGY: ConfigEntry[String] =
- buildConf("celeborn.push.limit.strategy")
+ buildConf("celeborn.client.push.limit.strategy")
.categories("client")
.doc("The strategy used to control the push speed. " +
"Valid strategies are SIMPLE and SLOWSTART. the SLOWSTART strategy is usually cooperate with " +
@@ -2326,7 +2392,7 @@ object CelebornConf extends Logging {
.createWithDefaultString("SIMPLE")
val PUSH_SLOW_START_INITIAL_SLEEP_TIME: ConfigEntry[Long] =
- buildConf("celeborn.push.slowStart.initialSleepTime")
+ buildConf("celeborn.client.push.slowStart.initialSleepTime")
.categories("client")
.version("0.3.0")
.doc(s"The initial sleep time if the current max in flight requests is 0")
@@ -2334,7 +2400,7 @@ object CelebornConf extends Logging {
.createWithDefaultString("500ms")
val PUSH_SLOW_START_MAX_SLEEP_TIME: ConfigEntry[Long] =
- buildConf("celeborn.push.slowStart.maxSleepTime")
+ buildConf("celeborn.client.push.slowStart.maxSleepTime")
.categories("client")
.version("0.3.0")
.doc(s"If ${PUSH_LIMIT_STRATEGY.key} is set to SLOWSTART, push side will " +
@@ -2344,35 +2410,38 @@ object CelebornConf extends Logging {
.createWithDefaultString("2s")
val PUSH_DATA_TIMEOUT: ConfigEntry[Long] =
- buildConf("celeborn.push.data.timeout")
+ buildConf("celeborn.client.push.data.timeout")
+ .withAlternative("celeborn.push.data.timeout")
.withAlternative("rss.push.data.rpc.timeout")
.categories("client")
.version("0.2.0")
.doc(s"Timeout for a task to push data rpc message. This value should better be more than twice of `${PUSH_TIMEOUT_CHECK_INTERVAL.key}`")
.timeConf(TimeUnit.MILLISECONDS)
- .checkValue(_ > 0, "celeborn.push.data.timeout must be positive!")
- .createWithDefaultString("30s")
+ .checkValue(_ > 0, "celeborn.client.push.data.timeout must be positive!")
+ .createWithDefaultString("120s")
val TEST_PUSH_MASTER_DATA_TIMEOUT: ConfigEntry[Boolean] =
- buildConf("celeborn.test.pushMasterDataTimeout")
+ buildConf("celeborn.test.worker.pushMasterDataTimeout")
+ .withAlternative("celeborn.test.pushMasterDataTimeout")
.internal
- .categories("worker", "test")
+ .categories("test", "worker")
.version("0.2.0")
.doc("Whether to test push master data timeout")
.booleanConf
.createWithDefault(false)
val TEST_PUSH_SLAVE_DATA_TIMEOUT: ConfigEntry[Boolean] =
- buildConf("celeborn.test.pushSlaveDataTimeout")
+ buildConf("celeborn.test.worker.pushSlaveDataTimeout")
.internal
- .categories("worker", "test")
+ .categories("test", "worker")
.version("0.3.0")
.doc("Whether to test push slave data timeout")
.booleanConf
.createWithDefault(false)
val PUSH_LIMIT_IN_FLIGHT_TIMEOUT: OptionalConfigEntry[Long] =
- buildConf("celeborn.push.limit.inFlight.timeout")
+ buildConf("celeborn.client.push.limit.inFlight.timeout")
+ .withAlternative("celeborn.push.limit.inFlight.timeout")
.withAlternative("rss.limit.inflight.timeout")
.categories("client")
.doc("Timeout for netty in-flight requests to be done." +
@@ -2382,7 +2451,8 @@ object CelebornConf extends Logging {
.createOptional
val PUSH_LIMIT_IN_FLIGHT_SLEEP_INTERVAL: ConfigEntry[Long] =
- buildConf("celeborn.push.limit.inFlight.sleepInterval")
+ buildConf("celeborn.client.push.limit.inFlight.sleepInterval")
+ .withAlternative("celeborn.push.limit.inFlight.sleepInterval")
.withAlternative("rss.limit.inflight.sleep.delta")
.categories("client")
.doc("Sleep interval when check netty in-flight requests to be done.")
@@ -2391,7 +2461,8 @@ object CelebornConf extends Logging {
.createWithDefaultString("50ms")
val PUSH_SORT_MEMORY_THRESHOLD: ConfigEntry[Long] =
- buildConf("celeborn.push.sortMemory.threshold")
+ buildConf("celeborn.client.push.sort.memory.threshold")
+ .withAlternative("celeborn.push.sortMemory.threshold")
.withAlternative("rss.sort.push.data.threshold")
.categories("client")
.doc("When SortBasedPusher use memory over the threshold, will trigger push data.")
@@ -2400,7 +2471,8 @@ object CelebornConf extends Logging {
.createWithDefaultString("64m")
val PUSH_SORT_PIPELINE_ENABLED: ConfigEntry[Boolean] =
- buildConf("celeborn.push.sort.pipeline.enabled")
+ buildConf("celeborn.client.push.sort.pipeline.enabled")
+ .withAlternative("celeborn.push.sort.pipeline.enabled")
.categories("client")
.doc("Whether to enable pipelining for sort based shuffle writer. If true, double buffering" +
" will be used to pipeline push")
@@ -2409,7 +2481,8 @@ object CelebornConf extends Logging {
.createWithDefault(false)
val PUSH_SORT_RANDOMIZE_PARITION_ENABLED: ConfigEntry[Boolean] =
- buildConf("celeborn.push.sort.randomizePartitionId.enabled")
+ buildConf("celeborn.client.push.sort.randomizePartitionId.enabled")
+ .withAlternative("celeborn.push.sort.randomizePartitionId.enabled")
.categories("client")
.doc(
"Whether to randomize partitionId in push sorter. If true, partitionId will be randomized " +
@@ -2419,7 +2492,8 @@ object CelebornConf extends Logging {
.createWithDefault(false)
val PUSH_RETRY_THREADS: ConfigEntry[Int] =
- buildConf("celeborn.push.retry.threads")
+ buildConf("celeborn.client.push.retry.threads")
+ .withAlternative("celeborn.push.retry.threads")
.withAlternative("rss.pushdata.retry.thread.num")
.categories("client")
.doc("Thread number to process shuffle re-send push data requests.")
@@ -2428,7 +2502,8 @@ object CelebornConf extends Logging {
.createWithDefault(8)
val PUSH_SPLIT_PARTITION_THREADS: ConfigEntry[Int] =
- buildConf("celeborn.push.splitPartition.threads")
+ buildConf("celeborn.client.push.splitPartition.threads")
+ .withAlternative("celeborn.push.splitPartition.threads")
.withAlternative("rss.client.split.pool.size")
.categories("client")
.doc("Thread number to process shuffle split request in shuffle client.")
@@ -2437,7 +2512,7 @@ object CelebornConf extends Logging {
.createWithDefault(8)
val PUSH_TAKE_TASK_WAIT_TIME: ConfigEntry[Long] =
- buildConf("celeborn.push.takeTaskWaitTime")
+ buildConf("celeborn.client.push.takeTaskWaitTime")
.categories("client")
.doc("Wait time if no task available to push to worker.")
.version("0.3.0")
@@ -2445,7 +2520,8 @@ object CelebornConf extends Logging {
.createWithDefaultString("50ms")
val PARTITION_SPLIT_THRESHOLD: ConfigEntry[Long] =
- buildConf("celeborn.shuffle.partitionSplit.threshold")
+ buildConf("celeborn.client.shuffle.partitionSplit.threshold")
+ .withAlternative("celeborn.shuffle.partitionSplit.threshold")
.withAlternative("rss.partition.split.threshold")
.categories("client")
.doc("Shuffle file size threshold, if file size exceeds this, trigger split.")
@@ -2454,7 +2530,8 @@ object CelebornConf extends Logging {
.createWithDefaultString("1G")
val PARTITION_SPLIT_MODE: ConfigEntry[String] =
- buildConf("celeborn.shuffle.partitionSplit.mode")
+ buildConf("celeborn.client.shuffle.partitionSplit.mode")
+ .withAlternative("celeborn.shuffle.partitionSplit.mode")
.withAlternative("rss.partition.split.mode")
.categories("client")
.doc("soft: the shuffle file size might be larger than split threshold. " +
@@ -2466,7 +2543,8 @@ object CelebornConf extends Logging {
.createWithDefault(PartitionSplitMode.SOFT.name)
val PARTITION_SPLIT_MIN: ConfigEntry[Long] =
- buildConf("celeborn.shuffle.partitionSplit.min")
+ buildConf("celeborn.worker.shuffle.partitionSplit.min")
+ .withAlternative("celeborn.shuffle.partitionSplit.min")
.withAlternative("rss.partition.split.minimum.size")
.categories("worker")
.doc("Min size for a partition to split")
@@ -2475,7 +2553,8 @@ object CelebornConf extends Logging {
.createWithDefaultString("1m")
val BATCH_HANDLE_CHANGE_PARTITION_ENABLED: ConfigEntry[Boolean] =
- buildConf("celeborn.shuffle.batchHandleChangePartition.enabled")
+ buildConf("celeborn.client.shuffle.batchHandleChangePartition.enabled")
+ .withAlternative("celeborn.shuffle.batchHandleChangePartition.enabled")
.withAlternative("rss.change.partition.batch.enabled")
.categories("client")
.doc("When true, LifecycleManager will handle change partition request in batch. " +
@@ -2485,7 +2564,8 @@ object CelebornConf extends Logging {
.createWithDefault(false)
val BATCH_HANDLE_CHANGE_PARTITION_THREADS: ConfigEntry[Int] =
- buildConf("celeborn.shuffle.batchHandleChangePartition.threads")
+ buildConf("celeborn.client.shuffle.batchHandleChangePartition.threads")
+ .withAlternative("celeborn.shuffle.batchHandleChangePartition.threads")
.withAlternative("rss.change.partition.numThreads")
.categories("client")
.doc("Threads number for LifecycleManager to handle change partition request in batch.")
@@ -2494,7 +2574,8 @@ object CelebornConf extends Logging {
.createWithDefault(8)
val BATCH_HANDLE_CHANGE_PARTITION_INTERVAL: ConfigEntry[Long] =
- buildConf("celeborn.shuffle.batchHandleChangePartition.interval")
+ buildConf("celeborn.client.shuffle.batchHandleChangePartition.interval")
+ .withAlternative("celeborn.shuffle.batchHandleChangePartition.interval")
.withAlternative("rss.change.partition.batchInterval")
.categories("client")
.doc("Interval for LifecycleManager to schedule handling change partition requests in batch.")
@@ -2503,7 +2584,8 @@ object CelebornConf extends Logging {
.createWithDefaultString("100ms")
val BATCH_HANDLE_COMMIT_PARTITION_ENABLED: ConfigEntry[Boolean] =
- buildConf("celeborn.shuffle.batchHandleCommitPartition.enabled")
+ buildConf("celeborn.client.shuffle.batchHandleCommitPartition.enabled")
+ .withAlternative("celeborn.shuffle.batchHandleCommitPartition.enabled")
.categories("client")
.doc("When true, LifecycleManager will handle commit partition request in batch. " +
"Otherwise, LifecycleManager won't commit partition before stage end")
@@ -2512,7 +2594,8 @@ object CelebornConf extends Logging {
.createWithDefault(false)
val BATCH_HANDLE_COMMIT_PARTITION_THREADS: ConfigEntry[Int] =
- buildConf("celeborn.shuffle.batchHandleCommitPartition.threads")
+ buildConf("celeborn.client.shuffle.batchHandleCommitPartition.threads")
+ .withAlternative("celeborn.shuffle.batchHandleCommitPartition.threads")
.categories("client")
.doc("Threads number for LifecycleManager to handle commit partition request in batch.")
.version("0.2.0")
@@ -2520,7 +2603,8 @@ object CelebornConf extends Logging {
.createWithDefault(8)
val BATCH_HANDLED_COMMIT_PARTITION_INTERVAL: ConfigEntry[Long] =
- buildConf("celeborn.shuffle.batchHandleCommitPartition.interval")
+ buildConf("celeborn.client.shuffle.batchHandleCommitPartition.interval")
+ .withAlternative("celeborn.shuffle.batchHandleCommitPartition.interval")
.categories("client")
.doc("Interval for LifecycleManager to schedule handling commit partition requests in batch.")
.version("0.2.0")
@@ -2528,7 +2612,7 @@ object CelebornConf extends Logging {
.createWithDefaultString("5s")
val BATCH_HANDLE_RELEASE_PARTITION_ENABLED: ConfigEntry[Boolean] =
- buildConf("celeborn.shuffle.batchHandleReleasePartition.enabled")
+ buildConf("celeborn.client.shuffle.batchHandleReleasePartition.enabled")
.categories("client")
.doc("When true, LifecycleManager will handle release partition request in batch. " +
"Otherwise, LifecycleManager will process release partition request immediately")
@@ -2537,7 +2621,7 @@ object CelebornConf extends Logging {
.createWithDefault(true)
val BATCH_HANDLE_RELEASE_PARTITION_THREADS: ConfigEntry[Int] =
- buildConf("celeborn.shuffle.batchHandleReleasePartition.threads")
+ buildConf("celeborn.client.shuffle.batchHandleReleasePartition.threads")
.categories("client")
.doc("Threads number for LifecycleManager to handle release partition request in batch.")
.version("0.3.0")
@@ -2545,7 +2629,7 @@ object CelebornConf extends Logging {
.createWithDefault(8)
val BATCH_HANDLED_RELEASE_PARTITION_INTERVAL: ConfigEntry[Long] =
- buildConf("celeborn.shuffle.batchHandleReleasePartition.interval")
+ buildConf("celeborn.client.shuffle.batchHandleReleasePartition.interval")
.categories("client")
.doc(
"Interval for LifecycleManager to schedule handling release partition requests in batch.")
@@ -2554,7 +2638,7 @@ object CelebornConf extends Logging {
.createWithDefaultString("5s")
val RESERVE_SLOTS_RACKAWARE_ENABLED: ConfigEntry[Boolean] =
- buildConf("celeborn.reserveSlots.rackware.enabled")
+ buildConf("celeborn.client.reserveSlots.rackware.enabled")
.categories("client")
.version("0.3.0")
.doc("Whether need to place different replicates on different racks when allocating slots.")
@@ -2562,14 +2646,15 @@ object CelebornConf extends Logging {
.createWithDefault(false)
val RESERVE_SLOTS_RPC_TIMEOUT: ConfigEntry[Long] =
- buildConf("celeborn.rpc.reserveSlots.askTimeout")
+ buildConf("celeborn.client.rpc.reserveSlots.askTimeout")
.categories("client")
.version("0.3.0")
.doc("Timeout for LifecycleManager request reserve slots.")
.fallbackConf(RPC_ASK_TIMEOUT)
val REGISTER_SHUFFLE_RPC_ASK_TIMEOUT: ConfigEntry[Long] =
- buildConf("celeborn.rpc.registerShuffle.askTimeout")
+ buildConf("celeborn.client.rpc.registerShuffle.askTimeout")
+ .withAlternative("celeborn.rpc.registerShuffle.askTimeout")
.categories("client")
.version("0.2.0")
.doc(s"Timeout for ask operations during register shuffle. " +
@@ -2581,7 +2666,7 @@ object CelebornConf extends Logging {
.fallbackConf(NETWORK_IO_CONNECTION_TIMEOUT)
val REQUEST_PARTITION_LOCATION_RPC_ASK_TIMEOUT: ConfigEntry[Long] =
- buildConf("celeborn.rpc.requestPartition.askTimeout")
+ buildConf("celeborn.client.rpc.requestPartition.askTimeout")
.categories("client")
.version("0.2.0")
.doc(s"Timeout for ask operations during requesting change partition location, such as reviving or spliting partition. " +
@@ -2591,7 +2676,7 @@ object CelebornConf extends Logging {
.fallbackConf(NETWORK_IO_CONNECTION_TIMEOUT)
val GET_REDUCER_FILE_GROUP_RPC_ASK_TIMEOUT: ConfigEntry[Long] =
- buildConf("celeborn.rpc.getReducerFileGroup.askTimeout")
+ buildConf("celeborn.client.rpc.getReducerFileGroup.askTimeout")
.categories("client")
.version("0.2.0")
.doc(s"Timeout for ask operations during getting reducer file group information. " +
@@ -2609,9 +2694,11 @@ object CelebornConf extends Logging {
.intConf
.createWithDefault(1)
- val CLIENT_MAX_RETRIES: ConfigEntry[Int] =
- buildConf("celeborn.client.maxRetries")
+ val MASTER_CLIENT_MAX_RETRIES: ConfigEntry[Int] =
+ buildConf("celeborn.masterClient.maxRetries")
+ .withAlternative("celeborn.client.maxRetries")
.withAlternative("rss.ha.client.maxTries")
+ .internal
.categories("client", "worker")
.doc("Max retry times for client to connect master endpoint")
.version("0.2.0")
@@ -2629,7 +2716,7 @@ object CelebornConf extends Logging {
val METRICS_ENABLED: ConfigEntry[Boolean] =
buildConf("celeborn.metrics.enabled")
.withAlternative("rss.metrics.system.enabled")
- .categories("master", "worker", "metrics")
+ .categories("metrics")
.doc("When true, enable metrics system.")
.version("0.2.0")
.booleanConf
@@ -2638,7 +2725,7 @@ object CelebornConf extends Logging {
val METRICS_SAMPLE_RATE: ConfigEntry[Double] =
buildConf("celeborn.metrics.sample.rate")
.withAlternative("rss.metrics.system.sample.rate")
- .categories("master", "worker", "metrics")
+ .categories("metrics")
.doc("It controls if Celeborn collect timer metrics for some operations. Its value should be in [0.0, 1.0].")
.version("0.2.0")
.doubleConf
@@ -2648,7 +2735,7 @@ object CelebornConf extends Logging {
val METRICS_SLIDING_WINDOW_SIZE: ConfigEntry[Int] =
buildConf("celeborn.metrics.timer.slidingWindow.size")
.withAlternative("rss.metrics.system.sliding.window.size")
- .categories("master", "worker", "metrics")
+ .categories("metrics")
.doc("The sliding window size of timer metric.")
.version("0.2.0")
.intConf
@@ -2657,7 +2744,7 @@ object CelebornConf extends Logging {
val METRICS_COLLECT_CRITICAL_ENABLED: ConfigEntry[Boolean] =
buildConf("celeborn.metrics.collectPerfCritical.enabled")
.withAlternative("rss.metrics.system.sample.perf.critical")
- .categories("master", "worker", "metrics")
+ .categories("metrics")
.doc("It controls whether to collect metrics which may affect performance. When enable, Celeborn collects them.")
.version("0.2.0")
.booleanConf
@@ -2666,25 +2753,27 @@ object CelebornConf extends Logging {
val METRICS_CAPACITY: ConfigEntry[Int] =
buildConf("celeborn.metrics.capacity")
.withAlternative("rss.inner.metrics.size")
- .categories("master", "worker", "metrics")
+ .categories("metrics")
.doc("The maximum number of metrics which a source can use to generate output strings.")
.version("0.2.0")
.intConf
.createWithDefault(4096)
val MASTER_PROMETHEUS_HOST: ConfigEntry[String] =
- buildConf("celeborn.master.metrics.prometheus.host")
+ buildConf("celeborn.metrics.master.prometheus.host")
+ .withAlternative("celeborn.master.metrics.prometheus.host")
.withAlternative("rss.master.prometheus.metric.host")
- .categories("master", "metrics")
+ .categories("metrics")
.doc("Master's Prometheus host.")
.version("0.2.0")
.stringConf
.createWithDefault("0.0.0.0")
val MASTER_PROMETHEUS_PORT: ConfigEntry[Int] =
- buildConf("celeborn.master.metrics.prometheus.port")
+ buildConf("celeborn.metrics.master.prometheus.port")
+ .withAlternative("celeborn.master.metrics.prometheus.port")
.withAlternative("rss.master.prometheus.metric.port")
- .categories("master", "metrics")
+ .categories("metrics")
.doc("Master's Prometheus port.")
.version("0.2.0")
.intConf
@@ -2692,18 +2781,20 @@ object CelebornConf extends Logging {
.createWithDefault(9098)
val WORKER_PROMETHEUS_HOST: ConfigEntry[String] =
- buildConf("celeborn.worker.metrics.prometheus.host")
+ buildConf("celeborn.metrics.worker.prometheus.host")
+ .withAlternative("celeborn.worker.metrics.prometheus.host")
.withAlternative("rss.worker.prometheus.metric.host")
- .categories("worker", "metrics")
+ .categories("metrics")
.doc("Worker's Prometheus host.")
.version("0.2.0")
.stringConf
.createWithDefault("0.0.0.0")
val WORKER_PROMETHEUS_PORT: ConfigEntry[Int] =
- buildConf("celeborn.worker.metrics.prometheus.port")
+ buildConf("celeborn.metrics.worker.prometheus.port")
+ .withAlternative("celeborn.worker.metrics.prometheus.port")
.withAlternative("rss.worker.prometheus.metric.port")
- .categories("worker", "metrics")
+ .categories("metrics")
.doc("Worker's Prometheus port.")
.version("0.2.0")
.intConf
@@ -2770,7 +2861,8 @@ object CelebornConf extends Logging {
.createOptional
val SHUFFLE_FORCE_FALLBACK_ENABLED: ConfigEntry[Boolean] =
- buildConf("celeborn.shuffle.forceFallback.enabled")
+ buildConf("celeborn.client.shuffle.forceFallback.enabled")
+ .withAlternative("celeborn.shuffle.forceFallback.enabled")
.withAlternative("rss.force.fallback")
.categories("client")
.version("0.2.0")
@@ -2779,7 +2871,8 @@ object CelebornConf extends Logging {
.createWithDefault(false)
val SHUFFLE_FORCE_FALLBACK_PARTITION_THRESHOLD: ConfigEntry[Long] =
- buildConf("celeborn.shuffle.forceFallback.numPartitionsThreshold")
+ buildConf("celeborn.client.shuffle.forceFallback.numPartitionsThreshold")
+ .withAlternative("celeborn.shuffle.forceFallback.numPartitionsThreshold")
.withAlternative("rss.max.partition.number")
.categories("client")
.version("0.2.0")
@@ -2789,7 +2882,8 @@ object CelebornConf extends Logging {
.createWithDefault(500000)
val SHUFFLE_MANAGER_PORT: ConfigEntry[Int] =
- buildConf("celeborn.shuffle.manager.port")
+ buildConf("celeborn.client.shuffle.manager.port")
+ .withAlternative("celeborn.shuffle.manager.port")
.withAlternative("rss.driver.metaService.port")
.categories("client")
.version("0.2.0")
@@ -2829,7 +2923,8 @@ object CelebornConf extends Logging {
.createWithDefaultString("readwrite,diskusage")
val WORKER_DISK_MONITOR_CHECK_INTERVAL: ConfigEntry[Long] =
- buildConf("celeborn.worker.monitor.disk.checkInterval")
+ buildConf("celeborn.worker.monitor.disk.check.interval")
+ .withAlternative("celeborn.worker.monitor.disk.checkInterval")
.withAlternative("rss.disk.check.interval")
.categories("worker")
.version("0.2.0")
@@ -2874,7 +2969,8 @@ object CelebornConf extends Logging {
.createWithDefault(3)
val SHUFFLE_PARTITION_TYPE: ConfigEntry[String] =
- buildConf("celeborn.shuffle.partition.type")
+ buildConf("celeborn.client.shuffle.partition.type")
+ .withAlternative("celeborn.shuffle.partition.type")
.withAlternative("rss.partition.type")
.categories("client")
.doc("Type of shuffle's partition.")
@@ -2888,7 +2984,8 @@ object CelebornConf extends Logging {
.createWithDefault(PartitionType.REDUCE.name)
val SHUFFLE_COMPRESSION_CODEC: ConfigEntry[String] =
- buildConf("celeborn.shuffle.compression.codec")
+ buildConf("celeborn.client.shuffle.compression.codec")
+ .withAlternative("celeborn.shuffle.compression.codec")
.withAlternative("rss.client.compression.codec")
.categories("client")
.doc("The codec used to compress shuffle data. By default, Celeborn provides two codecs: `lz4` and `zstd`.")
@@ -2899,7 +2996,8 @@ object CelebornConf extends Logging {
.createWithDefault(CompressionCodec.LZ4.name)
val SHUFFLE_COMPRESSION_ZSTD_LEVEL: ConfigEntry[Int] =
- buildConf("celeborn.shuffle.compression.zstd.level")
+ buildConf("celeborn.client.shuffle.compression.zstd.level")
+ .withAlternative("celeborn.shuffle.compression.zstd.level")
.withAlternative("rss.client.compression.zstd.level")
.categories("client")
.doc("Compression level for Zstd compression codec, its value should be an integer " +
@@ -2967,7 +3065,8 @@ object CelebornConf extends Logging {
.createWithDefault(0.5)
val WORKER_DIRECT_MEMORY_CHECK_INTERVAL: ConfigEntry[Long] =
- buildConf("celeborn.worker.memory.checkInterval")
+ buildConf("celeborn.worker.monitor.memory.check.interval")
+ .withAlternative("celeborn.worker.memory.checkInterval")
.withAlternative("rss.worker.memory.check.interval")
.categories("worker")
.doc("Interval of worker direct memory checking.")
@@ -2976,7 +3075,8 @@ object CelebornConf extends Logging {
.createWithDefaultString("10ms")
val WORKER_DIRECT_MEMORY_REPORT_INTERVAL: ConfigEntry[Long] =
- buildConf("celeborn.worker.memory.reportInterval")
+ buildConf("celeborn.worker.monitor.memory.report.interval")
+ .withAlternative("celeborn.worker.memory.reportInterval")
.withAlternative("rss.worker.memory.report.interval")
.categories("worker")
.doc("Interval of worker direct memory tracker reporting to log.")
@@ -2985,7 +3085,7 @@ object CelebornConf extends Logging {
.createWithDefaultString("10s")
val WORKER_DIRECT_MEMORY_TRIM_CHANNEL_WAIT_INTERVAL: ConfigEntry[Long] =
- buildConf("celeborn.worker.memory.trimChannelWaitInterval")
+ buildConf("celeborn.worker.monitor.memory.trimChannelWaitInterval")
.categories("worker")
.doc("Wait time after worker trigger channel to trim cache.")
.version("0.3.0")
@@ -2993,7 +3093,7 @@ object CelebornConf extends Logging {
.createWithDefaultString("1s")
val WORKER_DIRECT_MEMORY_TRIM_FLUSH_WAIT_INTERVAL: ConfigEntry[Long] =
- buildConf("celeborn.worker.memory.trimFlushWaitInterval")
+ buildConf("celeborn.worker.monitor.memory.trimFlushWaitInterval")
.categories("worker")
.doc("Wait time after worker trigger StorageManger to flush data.")
.version("0.3.0")
@@ -3084,7 +3184,7 @@ object CelebornConf extends Logging {
.timeConf(TimeUnit.MILLISECONDS)
.createWithDefaultString("480s")
- val WORKER_RECOVER_PATH: ConfigEntry[String] =
+ val WORKER_GRACEFUL_SHUTDOWN_RECOVER_PATH: ConfigEntry[String] =
buildConf("celeborn.worker.graceful.shutdown.recoverPath")
.withAlternative("rss.worker.recoverPath")
.categories("worker")
@@ -3105,16 +3205,18 @@ object CelebornConf extends Logging {
.createWithDefaultString("120s")
val HDFS_DIR: OptionalConfigEntry[String] =
- buildConf("celeborn.storage.hdfs.dir")
+ buildConf("celeborn.worker.storage.hdfs.dir")
+ .withAlternative("celeborn.storage.hdfs.dir")
.withAlternative("rss.worker.hdfs.dir")
- .categories("worker", "client")
+ .categories("worker")
.version("0.2.0")
.doc("HDFS dir configuration for Celeborn to access HDFS.")
.stringConf
.createOptional
val SHUFFLE_RANGE_READ_FILTER_ENABLED: ConfigEntry[Boolean] =
- buildConf("celeborn.shuffle.rangeReadFilter.enabled")
+ buildConf("celeborn.client.shuffle.rangeReadFilter.enabled")
+ .withAlternative("celeborn.shuffle.rangeReadFilter.enabled")
.withAlternative("rss.range.read.filter.enabled")
.categories("client")
.version("0.2.0")
@@ -3123,7 +3225,8 @@ object CelebornConf extends Logging {
.createWithDefault(false)
val COLUMNAR_SHUFFLE_ENABLED: ConfigEntry[Boolean] =
- buildConf("celeborn.columnar.shuffle.enabled")
+ buildConf("celeborn.columnarShuffle.enabled")
+ .withAlternative("celeborn.columnar.shuffle.enabled")
.categories("columnar-shuffle")
.version("0.2.0")
.doc("Whether to enable columnar-based shuffle.")
@@ -3131,7 +3234,8 @@ object CelebornConf extends Logging {
.createWithDefault(false)
val COLUMNAR_SHUFFLE_BATCH_SIZE: ConfigEntry[Int] =
- buildConf("celeborn.columnar.shuffle.batch.size")
+ buildConf("celeborn.columnarShuffle.batch.size")
+ .withAlternative("celeborn.columnar.shuffle.batch.size")
.categories("columnar-shuffle")
.version("0.2.0")
.doc("Vector batch size for columnar shuffle.")
@@ -3140,7 +3244,8 @@ object CelebornConf extends Logging {
.createWithDefault(10000)
val COLUMNAR_SHUFFLE_OFF_HEAP_ENABLED: ConfigEntry[Boolean] =
- buildConf("celeborn.columnar.offHeap.enabled")
+ buildConf("celeborn.columnarShuffle.offHeap.enabled")
+ .withAlternative("celeborn.columnar.offHeap.enabled")
.categories("columnar-shuffle")
.version("0.2.0")
.doc("Whether to use off heap columnar vector.")
@@ -3148,7 +3253,8 @@ object CelebornConf extends Logging {
.createWithDefault(false)
val COLUMNAR_SHUFFLE_DICTIONARY_ENCODING_ENABLED: ConfigEntry[Boolean] =
- buildConf("celeborn.columnar.shuffle.encoding.dictionary.enabled")
+ buildConf("celeborn.columnarShuffle.encoding.dictionary.enabled")
+ .withAlternative("celeborn.columnar.shuffle.encoding.dictionary.enabled")
.categories("columnar-shuffle")
.version("0.2.0")
.doc("Whether to use dictionary encoding for columnar-based shuffle data.")
@@ -3156,7 +3262,8 @@ object CelebornConf extends Logging {
.createWithDefault(false)
val COLUMNAR_SHUFFLE_DICTIONARY_ENCODING_MAX_FACTOR: ConfigEntry[Double] =
- buildConf("celeborn.columnar.shuffle.encoding.dictionary.maxFactor")
+ buildConf("celeborn.columnarShuffle.encoding.dictionary.maxFactor")
+ .withAlternative("celeborn.columnar.shuffle.encoding.dictionary.maxFactor")
.categories("columnar-shuffle")
.version("0.2.0")
.doc("Max factor for dictionary size. The max dictionary size is " +
@@ -3166,7 +3273,8 @@ object CelebornConf extends Logging {
.createWithDefault(0.3)
val COLUMNAR_SHUFFLE_CODEGEN_ENABLED: ConfigEntry[Boolean] =
- buildConf("celeborn.columnar.shuffle.codegen.enabled")
+ buildConf("celeborn.columnarShuffle.codegen.enabled")
+ .withAlternative("celeborn.columnar.shuffle.codegen.enabled")
.categories("columnar-shuffle")
.version("0.2.0")
.doc("Whether to use codegen for columnar-based shuffle.")
@@ -3174,7 +3282,8 @@ object CelebornConf extends Logging {
.createWithDefault(false)
val RPC_CACHE_SIZE: ConfigEntry[Int] =
- buildConf("celeborn.rpc.cache.size")
+ buildConf("celeborn.client.rpc.cache.size")
+ .withAlternative("celeborn.rpc.cache.size")
.categories("client")
.withAlternative("rss.rpc.cache.size")
.version("0.2.0")
@@ -3183,7 +3292,8 @@ object CelebornConf extends Logging {
.createWithDefault(256)
val RPC_CACHE_CONCURRENCY_LEVEL: ConfigEntry[Int] =
- buildConf("celeborn.rpc.cache.concurrencyLevel")
+ buildConf("celeborn.client.rpc.cache.concurrencyLevel")
+ .withAlternative("celeborn.rpc.cache.concurrencyLevel")
.categories("client")
.withAlternative("rss.rpc.cache.concurrent.level")
.version("0.2.0")
@@ -3192,9 +3302,10 @@ object CelebornConf extends Logging {
.createWithDefault(32)
val RPC_CACHE_EXPIRE_TIME: ConfigEntry[Long] =
- buildConf("celeborn.rpc.cache.expireTime")
- .categories("client")
+ buildConf("celeborn.client.rpc.cache.expireTime")
+ .withAlternative("celeborn.rpc.cache.expireTime")
.withAlternative("rss.rpc.cache.expire")
+ .categories("client")
.version("0.2.0")
.doc("The time before a cache item is removed.")
.timeConf(TimeUnit.MILLISECONDS)
@@ -3265,24 +3376,24 @@ object CelebornConf extends Logging {
.createWithDefault(32)
val MEMORY_ALLOCATOR_SHARE: ConfigEntry[Boolean] =
- buildConf("celeborn.memory.allocator.share")
- .categories("worker", "client")
+ buildConf("celeborn.network.memory.allocator.share")
+ .categories("network")
.version("0.3.0")
.doc("Whether to share memory allocator.")
.booleanConf
.createWithDefault(false)
val MEMORY_ALLOCATOR_ARENAS: OptionalConfigEntry[Int] =
- buildConf("celeborn.memory.allocator.numArenas")
- .categories("worker", "client")
+ buildConf("celeborn.network.memory.allocator.numArenas")
+ .categories("network")
.version("0.3.0")
.doc("Number of arenas for pooled memory allocator. Default value is Runtime.getRuntime.availableProcessors, min value is 2.")
.intConf
.createOptional
val MEMORY_ALLOCATOR_VERBOSE_METRIC: ConfigEntry[Boolean] =
- buildConf("celeborn.memory.allocator.verbose.metric")
- .categories("worker", "client")
+ buildConf("celeborn.network.memory.allocator.verbose.metric")
+ .categories("network")
.version("0.3.0")
.doc("Weather to enable verbose metric for pooled allocator.")
.booleanConf
@@ -3290,7 +3401,7 @@ object CelebornConf extends Logging {
val METRICS_EXTRA_LABELS: ConfigEntry[Seq[String]] =
buildConf("celeborn.metrics.extraLabels")
- .categories("master", "worker", "metrics")
+ .categories("metrics")
.doc("If default metric labels are not enough, extra metric labels can be customized. " +
"Labels' pattern is: `<label1_key>=<label1_value>[,<label2_key>=<label2_value>]*`; e.g. `env=prod,version=1`")
.version("0.3.0")
@@ -3317,11 +3428,16 @@ object CelebornConf extends Logging {
.booleanConf
.createWithDefault(true)
- val CLIENT_HEARTBEAT_INTERVAL: ConfigEntry[Long] =
- buildConf("celeborn.client.heartbeat.interval")
- .categories("client", "worker")
+ val CHANNEL_HEARTBEAT_INTERVAL: ConfigEntry[Long] =
+ buildConf("celeborn.<module>.heartbeat.interval")
+ .withAlternative("celeborn.client.heartbeat.interval")
+ .categories("network")
.version("0.3.0")
- .doc("the heartbeat interval between worker and client")
+ .doc("The heartbeat interval between worker and client. " +
+ s"If setting <module> to `${TransportModuleConstants.DATA_MODULE}`, " +
+ s"it works for shuffle client push and fetch data and should be configured in client side. " +
+ s"If setting <module> to `${TransportModuleConstants.REPLICATE_MODULE}`, " +
+ s"it works for worker replicate data to peer worker and should be configured in worker side.")
.timeConf(TimeUnit.MILLISECONDS)
.createWithDefaultString("60s")
diff --git a/common/src/test/java/org/apache/celeborn/common/haclient/RssHARetryClientSuiteJ.java b/common/src/test/java/org/apache/celeborn/common/haclient/RssHARetryClientSuiteJ.java
index 21dd10620..687c9a661 100644
--- a/common/src/test/java/org/apache/celeborn/common/haclient/RssHARetryClientSuiteJ.java
+++ b/common/src/test/java/org/apache/celeborn/common/haclient/RssHARetryClientSuiteJ.java
@@ -361,14 +361,14 @@ public class RssHARetryClientSuiteJ {
private CelebornConf prepareForCelebornConfWithoutHA() {
return conf.clone()
- .set("celeborn.ha.enabled", "false")
- .set("celeborn.master.endpoints", masterHost + ":" + masterPort);
+ .set(CelebornConf.HA_ENABLED().key(), "false")
+ .set(CelebornConf.MASTER_ENDPOINTS().key(), masterHost + ":" + masterPort);
}
private CelebornConf prepareForCelebornConfWithHA() {
return conf.clone()
- .set("celeborn.ha.enabled", "true")
- .set("celeborn.master.endpoints", "host1:9097,host2:9097,host3:9097")
- .set("celeborn.ha.client.maxRetries", "5");
+ .set(CelebornConf.HA_ENABLED().key(), "true")
+ .set(CelebornConf.MASTER_ENDPOINTS().key(), "host1:9097,host2:9097,host3:9097")
+ .set(CelebornConf.MASTER_CLIENT_MAX_RETRIES().key(), "5");
}
}
diff --git a/common/src/test/java/org/apache/celeborn/common/write/SlowStartPushStrategyTest.java b/common/src/test/java/org/apache/celeborn/common/write/SlowStartPushStrategyTest.java
index fdabd71b2..0c9ecceb1 100644
--- a/common/src/test/java/org/apache/celeborn/common/write/SlowStartPushStrategyTest.java
+++ b/common/src/test/java/org/apache/celeborn/common/write/SlowStartPushStrategyTest.java
@@ -28,9 +28,9 @@ public class SlowStartPushStrategyTest {
@Test
public void testSleepTime() {
- conf.set("celeborn.push.maxReqsInFlight", "32");
- conf.set("celeborn.push.limit.strategy", "slowstart");
- conf.set("celeborn.push.slowStart.maxSleepTime", "3s");
+ conf.set(CelebornConf.PUSH_MAX_REQS_IN_FLIGHT().key(), "32");
+ conf.set(CelebornConf.PUSH_LIMIT_STRATEGY().key(), "slowstart");
+ conf.set(CelebornConf.PUSH_SLOW_START_MAX_SLEEP_TIME().key(), "3s");
SlowStartPushStrategy strategy = (SlowStartPushStrategy) PushStrategy.getStrategy(conf);
String dummyHostPort = "test:9087";
SlowStartPushStrategy.CongestControlContext context =
@@ -84,9 +84,9 @@ public class SlowStartPushStrategyTest {
@Test
public void testCongestStrategy() {
- conf.set("celeborn.push.maxReqsInFlight", "5");
- conf.set("celeborn.push.limit.strategy", "slowstart");
- conf.set("celeborn.push.slowStart.maxSleepTime", "4s");
+ conf.set(CelebornConf.PUSH_MAX_REQS_IN_FLIGHT().key(), "5");
+ conf.set(CelebornConf.PUSH_LIMIT_STRATEGY().key(), "slowstart");
+ conf.set(CelebornConf.PUSH_SLOW_START_MAX_SLEEP_TIME().key(), "4s");
SlowStartPushStrategy strategy = (SlowStartPushStrategy) PushStrategy.getStrategy(conf);
String dummyHostPort = "test:9087";
// Slow start, should exponentially increase the currentReq
@@ -122,9 +122,9 @@ public class SlowStartPushStrategyTest {
@Test
public void testMultiHosts() {
- conf.set("celeborn.push.maxReqsInFlight", "3");
- conf.set("celeborn.push.limit.strategy", "slowstart");
- conf.set("celeborn.push.slowStart.maxSleepTime", "3s");
+ conf.set(CelebornConf.PUSH_MAX_REQS_IN_FLIGHT().key(), "3");
+ conf.set(CelebornConf.PUSH_LIMIT_STRATEGY().key(), "slowstart");
+ conf.set(CelebornConf.PUSH_SLOW_START_MAX_SLEEP_TIME().key(), "3s");
SlowStartPushStrategy strategy = (SlowStartPushStrategy) PushStrategy.getStrategy(conf);
String dummyHostPort1 = "test1:9087";
String dummyHostPort2 = "test2:9087";
diff --git a/common/src/test/scala/org/apache/celeborn/ConfigurationSuite.scala b/common/src/test/scala/org/apache/celeborn/ConfigurationSuite.scala
index b53588991..59f22671b 100644
--- a/common/src/test/scala/org/apache/celeborn/ConfigurationSuite.scala
+++ b/common/src/test/scala/org/apache/celeborn/ConfigurationSuite.scala
@@ -82,6 +82,10 @@ class ConfigurationSuite extends AnyFunSuite {
generateConfigurationMarkdown("metrics")
}
+ test("docs - configuration/ha.md") {
+ generateConfigurationMarkdown("ha")
+ }
+
def generateConfigurationMarkdown(category: String): Unit = {
val markdown = confMarkdownDir.resolve(s"$category.md")
diff --git a/common/src/test/scala/org/apache/celeborn/common/CelebornConfSuite.scala b/common/src/test/scala/org/apache/celeborn/common/CelebornConfSuite.scala
index befd561dc..1f9918f9e 100644
--- a/common/src/test/scala/org/apache/celeborn/common/CelebornConfSuite.scala
+++ b/common/src/test/scala/org/apache/celeborn/common/CelebornConfSuite.scala
@@ -18,13 +18,14 @@
package org.apache.celeborn.common
import org.apache.celeborn.CelebornFunSuite
+import org.apache.celeborn.common.CelebornConf._
import org.apache.celeborn.common.util.Utils
class CelebornConfSuite extends CelebornFunSuite {
test("celeborn.master.endpoints support multi nodes") {
val conf = new CelebornConf()
- .set("celeborn.master.endpoints", "localhost1:9097,localhost2:9097")
+ .set(CelebornConf.MASTER_ENDPOINTS.key, "localhost1:9097,localhost2:9097")
val masterEndpoints = conf.masterEndpoints
assert(masterEndpoints.length == 2)
assert(masterEndpoints(0) == "localhost1:9097")
@@ -34,7 +35,7 @@ class CelebornConfSuite extends CelebornFunSuite {
test("storage test") {
val conf = new CelebornConf()
val defaultMaxUsableSpace = 1024L * 1024 * 1024 * 1024 * 1024
- conf.set("celeborn.worker.storage.dirs", "/mnt/disk1")
+ conf.set(CelebornConf.WORKER_STORAGE_DIRS.key, "/mnt/disk1")
val workerBaseDirs = conf.workerBaseDirs
assert(workerBaseDirs.size == 1)
assert(workerBaseDirs.head._3 == 2)
@@ -43,8 +44,7 @@ class CelebornConfSuite extends CelebornFunSuite {
test("storage test2") {
val conf = new CelebornConf()
- val defaultMaxUsableSpace = 1024L * 1024 * 1024 * 1024 * 1024
- conf.set("celeborn.worker.storage.dirs", "/mnt/disk1:disktype=SSD:capacity=10g")
+ conf.set(CelebornConf.WORKER_STORAGE_DIRS.key, "/mnt/disk1:disktype=SSD:capacity=10g")
val workerBaseDirs = conf.workerBaseDirs
assert(workerBaseDirs.size == 1)
assert(workerBaseDirs.head._3 == 8)
@@ -53,7 +53,9 @@ class CelebornConfSuite extends CelebornFunSuite {
test("storage test3") {
val conf = new CelebornConf()
- conf.set("celeborn.worker.storage.dirs", "/mnt/disk1:disktype=SSD:capacity=10g:flushthread=3")
+ conf.set(
+ CelebornConf.WORKER_STORAGE_DIRS.key,
+ "/mnt/disk1:disktype=SSD:capacity=10g:flushthread=3")
val workerBaseDirs = conf.workerBaseDirs
assert(workerBaseDirs.size == 1)
assert(workerBaseDirs.head._3 == 3)
@@ -63,7 +65,7 @@ class CelebornConfSuite extends CelebornFunSuite {
test("storage test4") {
val conf = new CelebornConf()
conf.set(
- "celeborn.worker.storage.dirs",
+ CelebornConf.WORKER_STORAGE_DIRS.key,
"/mnt/disk1:disktype=SSD:capacity=10g:flushthread=3," +
"/mnt/disk2:disktype=HDD:capacity=15g:flushthread=7")
val workerBaseDirs = conf.workerBaseDirs
@@ -79,39 +81,39 @@ class CelebornConfSuite extends CelebornFunSuite {
test("storage test5") {
val conf = new CelebornConf()
- conf.set("celeborn.worker.storage.dirs", "/mnt/disk1")
+ conf.set(CelebornConf.WORKER_STORAGE_DIRS.key, "/mnt/disk1")
val workerBaseDirs = conf.workerBaseDirs
assert(workerBaseDirs.head._3 == 2)
}
test("storage test6") {
val conf = new CelebornConf()
- conf.set("celeborn.worker.flusher.threads", "4")
- .set("celeborn.worker.storage.dirs", "/mnt/disk1")
+ conf.set(CelebornConf.WORKER_FLUSHER_THREADS.key, "4")
+ .set(CelebornConf.WORKER_STORAGE_DIRS.key, "/mnt/disk1")
val workerBaseDirs = conf.workerBaseDirs
assert(workerBaseDirs.head._3 == 4)
}
test("storage test7") {
val conf = new CelebornConf()
- conf.set("celeborn.worker.flusher.threads", "4")
- .set("celeborn.worker.storage.dirs", "/mnt/disk1:flushthread=8")
+ conf.set(CelebornConf.WORKER_FLUSHER_THREADS.key, "4")
+ .set(CelebornConf.WORKER_STORAGE_DIRS.key, "/mnt/disk1:flushthread=8")
val workerBaseDirs = conf.workerBaseDirs
assert(workerBaseDirs.head._3 == 8)
}
test("storage test8") {
val conf = new CelebornConf()
- conf.set("celeborn.worker.flusher.threads", "4")
- .set("celeborn.worker.storage.dirs", "/mnt/disk1:disktype=SSD")
+ conf.set(CelebornConf.WORKER_FLUSHER_THREADS.key, "4")
+ .set(CelebornConf.WORKER_STORAGE_DIRS.key, "/mnt/disk1:disktype=SSD")
val workerBaseDirs = conf.workerBaseDirs
assert(workerBaseDirs.head._3 == 8)
}
test("storage test9") {
val conf = new CelebornConf()
- conf.set("celeborn.worker.flusher.threads", "4")
- .set("celeborn.worker.storage.dirs", "/mnt/disk1:flushthread=9:disktype=HDD")
+ conf.set(CelebornConf.WORKER_FLUSHER_THREADS.key, "4")
+ .set(CelebornConf.WORKER_STORAGE_DIRS.key, "/mnt/disk1:flushthread=9:disktype=HDD")
val workerBaseDirs = conf.workerBaseDirs
assert(workerBaseDirs.head._3 == 9)
}
@@ -119,22 +121,22 @@ class CelebornConfSuite extends CelebornFunSuite {
test("zstd level") {
val conf = new CelebornConf()
val error1 = intercept[IllegalArgumentException] {
- conf.set("celeborn.shuffle.compression.zstd.level", "-100")
+ conf.set(CelebornConf.SHUFFLE_COMPRESSION_ZSTD_LEVEL.key, "-100")
assert(conf.shuffleCompressionZstdCompressLevel == -100)
}.getMessage
- assert(error1.contains("'-100' in celeborn.shuffle.compression.zstd.level is invalid. " +
+ assert(error1.contains(s"'-100' in ${SHUFFLE_COMPRESSION_ZSTD_LEVEL.key} is invalid. " +
"Compression level for Zstd compression codec should be an integer between -5 and 22."))
- conf.set("celeborn.shuffle.compression.zstd.level", "-5")
+ conf.set(CelebornConf.SHUFFLE_COMPRESSION_ZSTD_LEVEL.key, "-5")
assert(conf.shuffleCompressionZstdCompressLevel == -5)
- conf.set("celeborn.shuffle.compression.zstd.level", "0")
+ conf.set(CelebornConf.SHUFFLE_COMPRESSION_ZSTD_LEVEL.key, "0")
assert(conf.shuffleCompressionZstdCompressLevel == 0)
- conf.set("celeborn.shuffle.compression.zstd.level", "22")
+ conf.set(CelebornConf.SHUFFLE_COMPRESSION_ZSTD_LEVEL.key, "22")
assert(conf.shuffleCompressionZstdCompressLevel == 22)
val error2 = intercept[IllegalArgumentException] {
- conf.set("celeborn.shuffle.compression.zstd.level", "100")
+ conf.set(CelebornConf.SHUFFLE_COMPRESSION_ZSTD_LEVEL.key, "100")
assert(conf.shuffleCompressionZstdCompressLevel == 100)
}.getMessage
- assert(error2.contains("'100' in celeborn.shuffle.compression.zstd.level is invalid. " +
+ assert(error2.contains(s"'100' in ${SHUFFLE_COMPRESSION_ZSTD_LEVEL.key} is invalid. " +
"Compression level for Zstd compression codec should be an integer between -5 and 22."))
}
@@ -166,7 +168,7 @@ class CelebornConfSuite extends CelebornFunSuite {
.set(CelebornConf.NETWORK_CONNECT_TIMEOUT, 2000L)
assert(conf.rpcAskTimeout.duration.toMillis == 1000L)
- assert(conf.haClientRpcAskTimeout.duration.toMillis == 1000L)
+ assert(conf.masterClientRpcAskTimeout.duration.toMillis == 1000L)
assert(conf.reserveSlotsRpcTimeout.duration.toMillis == 1000L)
assert(conf.networkTimeout.duration.toMillis == 20000L)
assert(conf.networkIoConnectionTimeoutMs("data") == 20000L)
diff --git a/conf/celeborn-defaults.conf.template b/conf/celeborn-defaults.conf.template
index e33293f6b..9f3bbdbc1 100644
--- a/conf/celeborn-defaults.conf.template
+++ b/conf/celeborn-defaults.conf.template
@@ -24,11 +24,9 @@ celeborn.rpc.io.numConnectionsPerPeer 2
celeborn.push.io.numConnectionsPerPeer 2
celeborn.worker.push.io.threads 8
-celeborn.push.replicate.enabled true
-
celeborn.metrics.enabled true
-celeborn.master.metrics.prometheus.port 9098
-celeborn.worker.metrics.prometheus.port 9096
+celeborn.metrics.master.prometheus.port 9098
+celeborn.metrics.worker.prometheus.port 9096
celeborn.worker.storage.dirs /mnt/disk1,/mnt/disk2,/mnt/disk3,/mnt/disk4,/mnt/disk5,/mnt/disk6,/mnt/disk7,/mnt/disk8,/mnt/disk9,/mnt/disk10
diff --git a/docs/configuration/client.md b/docs/configuration/client.md
index 37e72c54a..d8365b6eb 100644
--- a/docs/configuration/client.md
+++ b/docs/configuration/client.md
@@ -19,75 +19,67 @@ license: |
<!--begin-include-->
| Key | Default | Description | Since |
| --- | ------- | ----------- | ----- |
-| celeborn.application.heartbeatInterval | 10s | Interval for client to send heartbeat message to master. | 0.2.0 |
+| celeborn.client.application.heartbeatInterval | 10s | Interval for client to send heartbeat message to master. | 0.2.0 |
| celeborn.client.blacklistSlave.enabled | true | When true, Celeborn will add partition's peer worker into blacklist when push data to slave failed. | 0.3.0 |
| celeborn.client.closeIdleConnections | true | Whether client will close idle connections. | 0.3.0 |
-| celeborn.client.heartbeat.interval | 60s | the heartbeat interval between worker and client | 0.3.0 |
-| celeborn.client.maxRetries | 15 | Max retry times for client to connect master endpoint | 0.2.0 |
+| celeborn.client.excludedWorker.expireTimeout | 600s | Timeout time for LifecycleManager to clear reserved excluded worker. | 0.2.0 |
+| celeborn.client.fetch.maxReqsInFlight | 3 | Amount of in-flight chunk fetch request. | 0.2.0 |
+| celeborn.client.fetch.maxRetriesForEachReplica | 3 | Max retry times of fetch chunk on each replica | 0.2.0 |
+| celeborn.client.fetch.timeout | 30s | Timeout for a task to fetch chunk. | 0.2.0 |
| celeborn.client.push.blacklist.enabled | false | Whether to enable shuffle client-side push blacklist of workers. | 0.3.0 |
-| celeborn.fetch.maxReqsInFlight | 3 | Amount of in-flight chunk fetch request. | 0.2.0 |
-| celeborn.fetch.maxRetriesForEachReplica | 3 | Max retry times of fetch chunk on each replica | 0.2.0 |
-| celeborn.fetch.timeout | 30s | Timeout for a task to fetch chunk. | 0.2.0 |
+| celeborn.client.push.buffer.initial.size | 8k | | 0.2.0 |
+| celeborn.client.push.buffer.max.size | 64k | Max size of reducer partition buffer memory for shuffle hash writer. The pushed data will be buffered in memory before sending to Celeborn worker. For performance consideration keep this buffer size higher than 32K. Example: If reducer amount is 2000, buffer size is 64K, then each task will consume up to `64KiB * 2000 = 125MiB` heap memory. | 0.2.0 |
+| celeborn.client.push.data.timeout | 120s | Timeout for a task to push data rpc message. This value should better be more than twice of `celeborn.<module>.push.timeoutCheck.interval` | 0.2.0 |
+| celeborn.client.push.limit.inFlight.sleepInterval | 50ms | Sleep interval when check netty in-flight requests to be done. | 0.2.0 |
+| celeborn.client.push.limit.inFlight.timeout | <undefined> | Timeout for netty in-flight requests to be done.Default value should be `celeborn.client.push.data.timeout * 2`. | 0.2.0 |
+| celeborn.client.push.limit.strategy | SIMPLE | The strategy used to control the push speed. Valid strategies are SIMPLE and SLOWSTART. the SLOWSTART strategy is usually cooperate with congest control mechanism in the worker side. | 0.3.0 |
+| celeborn.client.push.maxReqsInFlight | 4 | Amount of Netty in-flight requests per worker. The maximum memory is `celeborn.push.maxReqsInFlight` * `celeborn.push.buffer.max.size` * compression ratio(1 in worst case), default: 64Kib * 32 = 2Mib | 0.2.0 |
+| celeborn.client.push.queue.capacity | 512 | Push buffer queue size for a task. The maximum memory is `celeborn.push.buffer.max.size` * `celeborn.push.queue.capacity`, default: 64KiB * 512 = 32MiB | 0.2.0 |
+| celeborn.client.push.replicate.enabled | false | When true, Celeborn worker will replicate shuffle data to another Celeborn worker asynchronously to ensure the pushed shuffle data won't be lost after the node failure. | 0.2.0 |
+| celeborn.client.push.retry.threads | 8 | Thread number to process shuffle re-send push data requests. | 0.2.0 |
+| celeborn.client.push.revive.maxRetries | 5 | Max retry times for reviving when celeborn push data failed. | 0.3.0 |
+| celeborn.client.push.slowStart.initialSleepTime | 500ms | The initial sleep time if the current max in flight requests is 0 | 0.3.0 |
+| celeborn.client.push.slowStart.maxSleepTime | 2s | If celeborn.client.push.limit.strategy is set to SLOWSTART, push side will take a sleep strategy for each batch of requests, this controls the max sleep time if the max in flight requests limit is 1 for a long time | 0.3.0 |
+| celeborn.client.push.sort.memory.threshold | 64m | When SortBasedPusher use memory over the threshold, will trigger push data. | 0.2.0 |
+| celeborn.client.push.sort.pipeline.enabled | false | Whether to enable pipelining for sort based shuffle writer. If true, double buffering will be used to pipeline push | 0.2.1 |
+| celeborn.client.push.sort.randomizePartitionId.enabled | false | Whether to randomize partitionId in push sorter. If true, partitionId will be randomized when sort data to avoid skew when push to worker | 0.2.1 |
+| celeborn.client.push.splitPartition.threads | 8 | Thread number to process shuffle split request in shuffle client. | 0.2.0 |
+| celeborn.client.push.stageEnd.timeout | <value of celeborn.<module>.io.connectionTimeout> | Timeout for waiting StageEnd. During this process, there are `celeborn.client.rpc.requestCommitFiles.maxRetries` times for retry opportunities for committing filesand 1 times for releasing slots request. User can customize this value according to your setting. By default, the value is the max timeout value `celeborn.<module>.io.connectionTimeout`. | 0.2.0 |
+| celeborn.client.push.takeTaskWaitTime | 50ms | Wait time if no task available to push to worker. | 0.3.0 |
+| celeborn.client.registerShuffle.maxRetries | 3 | Max retry times for client to register shuffle. | 0.2.0 |
+| celeborn.client.registerShuffle.retryWait | 3s | Wait time before next retry if register shuffle failed. | 0.2.0 |
+| celeborn.client.reserveSlots.maxRetries | 3 | Max retry times for client to reserve slots. | 0.2.0 |
+| celeborn.client.reserveSlots.rackware.enabled | false | Whether need to place different replicates on different racks when allocating slots. | 0.3.0 |
+| celeborn.client.reserveSlots.retryWait | 3s | Wait time before next retry if reserve slots failed. | 0.2.0 |
+| celeborn.client.rpc.cache.concurrencyLevel | 32 | The number of write locks to update rpc cache. | 0.2.0 |
+| celeborn.client.rpc.cache.expireTime | 15s | The time before a cache item is removed. | 0.2.0 |
+| celeborn.client.rpc.cache.size | 256 | The max cache items count for rpc cache. | 0.2.0 |
+| celeborn.client.rpc.getReducerFileGroup.askTimeout | <value of celeborn.<module>.io.connectionTimeout> | Timeout for ask operations during getting reducer file group information. During this process, there are `celeborn.client.rpc.requestCommitFiles.maxRetries` times for retry opportunities for committing filesand 1 times for releasing slots request. User can customize this value according to your setting. By default, the value is the max timeout value `celeborn.<module>.io [...]
+| celeborn.client.rpc.maxParallelism | 1024 | Max parallelism of client on sending RPC requests. | 0.2.0 |
+| celeborn.client.rpc.registerShuffle.askTimeout | <value of celeborn.<module>.io.connectionTimeout> | Timeout for ask operations during register shuffle. During this process, there are two times for retry opportunities for requesting slots, one request for establishing a connection with Worker and `celeborn.client.reserveSlots.maxRetries` times for retry opportunities for reserving slots. User can customize this value according to your setting. By default, the value is the m [...]
+| celeborn.client.rpc.requestCommitFiles.maxRetries | 2 | Max retry times for requestCommitFiles RPC. | 0.3.0 |
+| celeborn.client.rpc.requestPartition.askTimeout | <value of celeborn.<module>.io.connectionTimeout> | Timeout for ask operations during requesting change partition location, such as reviving or spliting partition. During this process, there are `celeborn.client.reserveSlots.maxRetries` times for retry opportunities for reserving slots. User can customize this value according to your setting. By default, the value is the max timeout value `celeborn.<module>.io.connectionTime [...]
+| celeborn.client.rpc.reserveSlots.askTimeout | <value of celeborn.rpc.askTimeout> | Timeout for LifecycleManager request reserve slots. | 0.3.0 |
+| celeborn.client.shuffle.batchHandleChangePartition.enabled | false | When true, LifecycleManager will handle change partition request in batch. Otherwise, LifecycleManager will process the requests one by one | 0.2.0 |
+| celeborn.client.shuffle.batchHandleChangePartition.interval | 100ms | Interval for LifecycleManager to schedule handling change partition requests in batch. | 0.2.0 |
+| celeborn.client.shuffle.batchHandleChangePartition.threads | 8 | Threads number for LifecycleManager to handle change partition request in batch. | 0.2.0 |
+| celeborn.client.shuffle.batchHandleCommitPartition.enabled | false | When true, LifecycleManager will handle commit partition request in batch. Otherwise, LifecycleManager won't commit partition before stage end | 0.2.0 |
+| celeborn.client.shuffle.batchHandleCommitPartition.interval | 5s | Interval for LifecycleManager to schedule handling commit partition requests in batch. | 0.2.0 |
+| celeborn.client.shuffle.batchHandleCommitPartition.threads | 8 | Threads number for LifecycleManager to handle commit partition request in batch. | 0.2.0 |
+| celeborn.client.shuffle.batchHandleReleasePartition.enabled | true | When true, LifecycleManager will handle release partition request in batch. Otherwise, LifecycleManager will process release partition request immediately | 0.3.0 |
+| celeborn.client.shuffle.batchHandleReleasePartition.interval | 5s | Interval for LifecycleManager to schedule handling release partition requests in batch. | 0.3.0 |
+| celeborn.client.shuffle.batchHandleReleasePartition.threads | 8 | Threads number for LifecycleManager to handle release partition request in batch. | 0.3.0 |
+| celeborn.client.shuffle.compression.codec | LZ4 | The codec used to compress shuffle data. By default, Celeborn provides two codecs: `lz4` and `zstd`. | 0.2.0 |
+| celeborn.client.shuffle.compression.zstd.level | 1 | Compression level for Zstd compression codec, its value should be an integer between -5 and 22. Increasing the compression level will result in better compression at the expense of more CPU and memory. | 0.2.0 |
+| celeborn.client.shuffle.expired.checkInterval | 60s | Interval for client to check expired shuffles. | 0.2.0 |
+| celeborn.client.shuffle.forceFallback.enabled | false | Whether force fallback shuffle to Spark's default. | 0.2.0 |
+| celeborn.client.shuffle.forceFallback.numPartitionsThreshold | 500000 | Celeborn will only accept shuffle of partition number lower than this configuration value. | 0.2.0 |
+| celeborn.client.shuffle.manager.port | 0 | Port used by the LifecycleManager on the Driver. | 0.2.0 |
+| celeborn.client.shuffle.partition.type | REDUCE | Type of shuffle's partition. | 0.2.0 |
+| celeborn.client.shuffle.partitionSplit.mode | SOFT | soft: the shuffle file size might be larger than split threshold. hard: the shuffle file size will be limited to split threshold. | 0.2.0 |
+| celeborn.client.shuffle.partitionSplit.threshold | 1G | Shuffle file size threshold, if file size exceeds this, trigger split. | 0.2.0 |
+| celeborn.client.shuffle.rangeReadFilter.enabled | false | If a spark application have skewed partition, this value can set to true to improve performance. | 0.2.0 |
+| celeborn.client.shuffle.writer | HASH | Celeborn supports the following kind of shuffle writers. 1. hash: hash-based shuffle writer works fine when shuffle partition count is normal; 2. sort: sort-based shuffle writer works fine when memory pressure is high or shuffle partition count is huge. | 0.2.0 |
| celeborn.master.endpoints | <localhost>:9097 | Endpoints of master nodes for celeborn client to connect, allowed pattern is: `<host1>:<port1>[,<host2>:<port2>]*`, e.g. `clb1:9097,clb2:9098,clb3:9099`. If the port is omitted, 9097 will be used. | 0.2.0 |
-| celeborn.memory.allocator.numArenas | <undefined> | Number of arenas for pooled memory allocator. Default value is Runtime.getRuntime.availableProcessors, min value is 2. | 0.3.0 |
-| celeborn.memory.allocator.share | false | Whether to share memory allocator. | 0.3.0 |
-| celeborn.memory.allocator.verbose.metric | false | Weather to enable verbose metric for pooled allocator. | 0.3.0 |
-| celeborn.push.buffer.initial.size | 8k | | 0.2.0 |
-| celeborn.push.buffer.max.size | 64k | Max size of reducer partition buffer memory for shuffle hash writer. The pushed data will be buffered in memory before sending to Celeborn worker. For performance consideration keep this buffer size higher than 32K. Example: If reducer amount is 2000, buffer size is 64K, then each task will consume up to `64KiB * 2000 = 125MiB` heap memory. | 0.2.0 |
-| celeborn.push.data.timeout | 30s | Timeout for a task to push data rpc message. This value should better be more than twice of `celeborn.push.timeoutCheck.interval` | 0.2.0 |
-| celeborn.push.limit.inFlight.sleepInterval | 50ms | Sleep interval when check netty in-flight requests to be done. | 0.2.0 |
-| celeborn.push.limit.inFlight.timeout | <undefined> | Timeout for netty in-flight requests to be done.Default value should be `celeborn.push.data.timeout * 2`. | 0.2.0 |
-| celeborn.push.limit.strategy | SIMPLE | The strategy used to control the push speed. Valid strategies are SIMPLE and SLOWSTART. the SLOWSTART strategy is usually cooperate with congest control mechanism in the worker side. | 0.3.0 |
-| celeborn.push.maxReqsInFlight | 4 | Amount of Netty in-flight requests per worker. The maximum memory is `celeborn.push.maxReqsInFlight` * `celeborn.push.buffer.max.size` * compression ratio(1 in worst case), default: 64Kib * 32 = 2Mib | 0.2.0 |
-| celeborn.push.queue.capacity | 512 | Push buffer queue size for a task. The maximum memory is `celeborn.push.buffer.max.size` * `celeborn.push.queue.capacity`, default: 64KiB * 512 = 32MiB | 0.2.0 |
-| celeborn.push.replicate.enabled | false | When true, Celeborn worker will replicate shuffle data to another Celeborn worker asynchronously to ensure the pushed shuffle data won't be lost after the node failure. | 0.2.0 |
-| celeborn.push.retry.threads | 8 | Thread number to process shuffle re-send push data requests. | 0.2.0 |
-| celeborn.push.revive.maxRetries | 5 | Max retry times for reviving when celeborn push data failed. | 0.3.0 |
-| celeborn.push.slowStart.initialSleepTime | 500ms | The initial sleep time if the current max in flight requests is 0 | 0.3.0 |
-| celeborn.push.slowStart.maxSleepTime | 2s | If celeborn.push.limit.strategy is set to SLOWSTART, push side will take a sleep strategy for each batch of requests, this controls the max sleep time if the max in flight requests limit is 1 for a long time | 0.3.0 |
-| celeborn.push.sort.pipeline.enabled | false | Whether to enable pipelining for sort based shuffle writer. If true, double buffering will be used to pipeline push | 0.2.1 |
-| celeborn.push.sort.randomizePartitionId.enabled | false | Whether to randomize partitionId in push sorter. If true, partitionId will be randomized when sort data to avoid skew when push to worker | 0.2.1 |
-| celeborn.push.sortMemory.threshold | 64m | When SortBasedPusher use memory over the threshold, will trigger push data. | 0.2.0 |
-| celeborn.push.splitPartition.threads | 8 | Thread number to process shuffle split request in shuffle client. | 0.2.0 |
-| celeborn.push.stageEnd.timeout | <value of celeborn.<module>.io.connectionTimeout> | Timeout for waiting StageEnd. During this process, there are `celeborn.rpc.requestCommitFiles.maxRetries` times for retry opportunities for committing filesand 1 times for releasing slots request. User can customize this value according to your setting. By default, the value is the max timeout value `celeborn.<module>.io.connectionTimeout`. | 0.2.0 |
-| celeborn.push.takeTaskWaitTime | 50ms | Wait time if no task available to push to worker. | 0.3.0 |
-| celeborn.push.timeoutCheck.interval | 5s | Interval for checking push data timeout. | 0.3.0 |
-| celeborn.push.timeoutCheck.threads | 16 | Threads num for checking push data timeout. | 0.3.0 |
-| celeborn.reserveSlots.rackware.enabled | false | Whether need to place different replicates on different racks when allocating slots. | 0.3.0 |
-| celeborn.rpc.cache.concurrencyLevel | 32 | The number of write locks to update rpc cache. | 0.2.0 |
-| celeborn.rpc.cache.expireTime | 15s | The time before a cache item is removed. | 0.2.0 |
-| celeborn.rpc.cache.size | 256 | The max cache items count for rpc cache. | 0.2.0 |
-| celeborn.rpc.getReducerFileGroup.askTimeout | <value of celeborn.<module>.io.connectionTimeout> | Timeout for ask operations during getting reducer file group information. During this process, there are `celeborn.rpc.requestCommitFiles.maxRetries` times for retry opportunities for committing filesand 1 times for releasing slots request. User can customize this value according to your setting. By default, the value is the max timeout value `celeborn.<module>.io.connectionTim [...]
-| celeborn.rpc.maxParallelism | 1024 | Max parallelism of client on sending RPC requests. | 0.2.0 |
-| celeborn.rpc.registerShuffle.askTimeout | <value of celeborn.<module>.io.connectionTimeout> | Timeout for ask operations during register shuffle. During this process, there are two times for retry opportunities for requesting slots, one request for establishing a connection with Worker and `celeborn.slots.reserve.maxRetries` times for retry opportunities for reserving slots. User can customize this value according to your setting. By default, the value is the max timeout va [...]
-| celeborn.rpc.requestCommitFiles.maxRetries | 2 | Max retry times for requestCommitFiles RPC. | 1.0.0 |
-| celeborn.rpc.requestPartition.askTimeout | <value of celeborn.<module>.io.connectionTimeout> | Timeout for ask operations during requesting change partition location, such as reviving or spliting partition. During this process, there are `celeborn.slots.reserve.maxRetries` times for retry opportunities for reserving slots. User can customize this value according to your setting. By default, the value is the max timeout value `celeborn.<module>.io.connectionTimeout`. | 0.2.0 |
-| celeborn.rpc.reserveSlots.askTimeout | <value of celeborn.rpc.askTimeout> | Timeout for LifecycleManager request reserve slots. | 0.3.0 |
-| celeborn.shuffle.batchHandleChangePartition.enabled | false | When true, LifecycleManager will handle change partition request in batch. Otherwise, LifecycleManager will process the requests one by one | 0.2.0 |
-| celeborn.shuffle.batchHandleChangePartition.interval | 100ms | Interval for LifecycleManager to schedule handling change partition requests in batch. | 0.2.0 |
-| celeborn.shuffle.batchHandleChangePartition.threads | 8 | Threads number for LifecycleManager to handle change partition request in batch. | 0.2.0 |
-| celeborn.shuffle.batchHandleCommitPartition.enabled | false | When true, LifecycleManager will handle commit partition request in batch. Otherwise, LifecycleManager won't commit partition before stage end | 0.2.0 |
-| celeborn.shuffle.batchHandleCommitPartition.interval | 5s | Interval for LifecycleManager to schedule handling commit partition requests in batch. | 0.2.0 |
-| celeborn.shuffle.batchHandleCommitPartition.threads | 8 | Threads number for LifecycleManager to handle commit partition request in batch. | 0.2.0 |
-| celeborn.shuffle.batchHandleReleasePartition.enabled | true | When true, LifecycleManager will handle release partition request in batch. Otherwise, LifecycleManager will process release partition request immediately | 0.3.0 |
-| celeborn.shuffle.batchHandleReleasePartition.interval | 5s | Interval for LifecycleManager to schedule handling release partition requests in batch. | 0.3.0 |
-| celeborn.shuffle.batchHandleReleasePartition.threads | 8 | Threads number for LifecycleManager to handle release partition request in batch. | 0.3.0 |
| celeborn.shuffle.chunk.size | 8m | Max chunk size of reducer's merged shuffle data. For example, if a reducer's shuffle data is 128M and the data will need 16 fetch chunk requests to fetch. | 0.2.0 |
-| celeborn.shuffle.compression.codec | LZ4 | The codec used to compress shuffle data. By default, Celeborn provides two codecs: `lz4` and `zstd`. | 0.2.0 |
-| celeborn.shuffle.compression.zstd.level | 1 | Compression level for Zstd compression codec, its value should be an integer between -5 and 22. Increasing the compression level will result in better compression at the expense of more CPU and memory. | 0.2.0 |
-| celeborn.shuffle.expired.checkInterval | 60s | Interval for client to check expired shuffles. | 0.2.0 |
-| celeborn.shuffle.forceFallback.enabled | false | Whether force fallback shuffle to Spark's default. | 0.2.0 |
-| celeborn.shuffle.forceFallback.numPartitionsThreshold | 500000 | Celeborn will only accept shuffle of partition number lower than this configuration value. | 0.2.0 |
-| celeborn.shuffle.manager.port | 0 | Port used by the LifecycleManager on the Driver. | 0.2.0 |
-| celeborn.shuffle.partition.type | REDUCE | Type of shuffle's partition. | 0.2.0 |
-| celeborn.shuffle.partitionSplit.mode | SOFT | soft: the shuffle file size might be larger than split threshold. hard: the shuffle file size will be limited to split threshold. | 0.2.0 |
-| celeborn.shuffle.partitionSplit.threshold | 1G | Shuffle file size threshold, if file size exceeds this, trigger split. | 0.2.0 |
-| celeborn.shuffle.rangeReadFilter.enabled | false | If a spark application have skewed partition, this value can set to true to improve performance. | 0.2.0 |
-| celeborn.shuffle.register.maxRetries | 3 | Max retry times for client to register shuffle. | 0.2.0 |
-| celeborn.shuffle.register.retryWait | 3s | Wait time before next retry if register shuffle failed. | 0.2.0 |
-| celeborn.shuffle.writer | HASH | Celeborn supports the following kind of shuffle writers. 1. hash: hash-based shuffle writer works fine when shuffle partition count is normal; 2. sort: sort-based shuffle writer works fine when memory pressure is high or shuffle partition count is huge. | 0.2.0 |
-| celeborn.slots.reserve.maxRetries | 3 | Max retry times for client to reserve slots. | 0.2.0 |
-| celeborn.slots.reserve.retryWait | 3s | Wait time before next retry if reserve slots failed. | 0.2.0 |
-| celeborn.storage.hdfs.dir | <undefined> | HDFS dir configuration for Celeborn to access HDFS. | 0.2.0 |
-| celeborn.worker.excluded.expireTimeout | 600s | Timeout time for LifecycleManager to clear reserved excluded worker. | 0.2.0 |
<!--end-include-->
diff --git a/docs/configuration/columnar-shuffle.md b/docs/configuration/columnar-shuffle.md
index f27d0f556..4fd8779fd 100644
--- a/docs/configuration/columnar-shuffle.md
+++ b/docs/configuration/columnar-shuffle.md
@@ -19,10 +19,10 @@ license: |
<!--begin-include-->
| Key | Default | Description | Since |
| --- | ------- | ----------- | ----- |
-| celeborn.columnar.offHeap.enabled | false | Whether to use off heap columnar vector. | 0.2.0 |
-| celeborn.columnar.shuffle.batch.size | 10000 | Vector batch size for columnar shuffle. | 0.2.0 |
-| celeborn.columnar.shuffle.codegen.enabled | false | Whether to use codegen for columnar-based shuffle. | 0.2.0 |
-| celeborn.columnar.shuffle.enabled | false | Whether to enable columnar-based shuffle. | 0.2.0 |
-| celeborn.columnar.shuffle.encoding.dictionary.enabled | false | Whether to use dictionary encoding for columnar-based shuffle data. | 0.2.0 |
-| celeborn.columnar.shuffle.encoding.dictionary.maxFactor | 0.3 | Max factor for dictionary size. The max dictionary size is `min(32.0 KB, celeborn.columnar.shuffle.batch.size * celeborn.columnar.shuffle.encoding.dictionary.maxFactor)`. | 0.2.0 |
+| celeborn.columnarShuffle.batch.size | 10000 | Vector batch size for columnar shuffle. | 0.2.0 |
+| celeborn.columnarShuffle.codegen.enabled | false | Whether to use codegen for columnar-based shuffle. | 0.2.0 |
+| celeborn.columnarShuffle.enabled | false | Whether to enable columnar-based shuffle. | 0.2.0 |
+| celeborn.columnarShuffle.encoding.dictionary.enabled | false | Whether to use dictionary encoding for columnar-based shuffle data. | 0.2.0 |
+| celeborn.columnarShuffle.encoding.dictionary.maxFactor | 0.3 | Max factor for dictionary size. The max dictionary size is `min(32.0 KB, celeborn.columnarShuffle.batch.size * celeborn.columnar.shuffle.encoding.dictionary.maxFactor)`. | 0.2.0 |
+| celeborn.columnarShuffle.offHeap.enabled | false | Whether to use off heap columnar vector. | 0.2.0 |
<!--end-include-->
diff --git a/docs/configuration/columnar-shuffle.md b/docs/configuration/ha.md
similarity index 53%
copy from docs/configuration/columnar-shuffle.md
copy to docs/configuration/ha.md
index f27d0f556..d3b4f74c0 100644
--- a/docs/configuration/columnar-shuffle.md
+++ b/docs/configuration/ha.md
@@ -19,10 +19,10 @@ license: |
<!--begin-include-->
| Key | Default | Description | Since |
| --- | ------- | ----------- | ----- |
-| celeborn.columnar.offHeap.enabled | false | Whether to use off heap columnar vector. | 0.2.0 |
-| celeborn.columnar.shuffle.batch.size | 10000 | Vector batch size for columnar shuffle. | 0.2.0 |
-| celeborn.columnar.shuffle.codegen.enabled | false | Whether to use codegen for columnar-based shuffle. | 0.2.0 |
-| celeborn.columnar.shuffle.enabled | false | Whether to enable columnar-based shuffle. | 0.2.0 |
-| celeborn.columnar.shuffle.encoding.dictionary.enabled | false | Whether to use dictionary encoding for columnar-based shuffle data. | 0.2.0 |
-| celeborn.columnar.shuffle.encoding.dictionary.maxFactor | 0.3 | Max factor for dictionary size. The max dictionary size is `min(32.0 KB, celeborn.columnar.shuffle.batch.size * celeborn.columnar.shuffle.encoding.dictionary.maxFactor)`. | 0.2.0 |
+| celeborn.ha.enabled | false | When true, master nodes run as Raft cluster mode. | 0.2.0 |
+| celeborn.ha.master.node.<id>.host | <required> | Host to bind of master node <id> in HA mode. | 0.2.0 |
+| celeborn.ha.master.node.<id>.port | 9097 | Port to bind of master node <id> in HA mode. | 0.2.0 |
+| celeborn.ha.master.node.<id>.ratis.port | 9872 | Ratis port to bind of master node <id> in HA mode. | 0.2.0 |
+| celeborn.ha.master.ratis.raft.rpc.type | netty | RPC type for Ratis, available options: netty, grpc. | 0.2.0 |
+| celeborn.ha.master.ratis.raft.server.storage.dir | /tmp/ratis | | 0.2.0 |
<!--end-include-->
diff --git a/docs/configuration/index.md b/docs/configuration/index.md
index 1d4b5db4e..40d43a070 100644
--- a/docs/configuration/index.md
+++ b/docs/configuration/index.md
@@ -44,38 +44,41 @@ memory. Empirically, Celeborn worker off-heap memory should be set to `(numDirs
## All Configurations
-### Client
+### Master
{!
-include-markdown "./client.md"
+include-markdown "./master.md"
start="<!--begin-include-->"
end="<!--end-include-->"
!}
-### Columnar Shuffle
+Apart from these, the following properties are also available for enable master HA:
+### Master HA
{!
-include-markdown "./columnar-shuffle.md"
+include-markdown "./ha.md"
start="<!--begin-include-->"
end="<!--end-include-->"
!}
-### Master
+### Worker
{!
-include-markdown "./master.md"
+include-markdown "./worker.md"
start="<!--begin-include-->"
end="<!--end-include-->"
!}
-### Worker
+
+### Client
{!
-include-markdown "./worker.md"
+include-markdown "./client.md"
start="<!--begin-include-->"
end="<!--end-include-->"
!}
+
### Quota
{!
@@ -92,8 +95,19 @@ start="<!--begin-include-->"
end="<!--end-include-->"
!}
+
+### Columnar Shuffle
+
+{!
+include-markdown "./columnar-shuffle.md"
+start="<!--begin-include-->"
+end="<!--end-include-->"
+!}
+
### Metrics
+Below metrics configuration both work for master and worker.
+
{!
include-markdown "./metrics.md"
start="<!--begin-include-->"
diff --git a/docs/configuration/master.md b/docs/configuration/master.md
index 5868420a3..314b1254e 100644
--- a/docs/configuration/master.md
+++ b/docs/configuration/master.md
@@ -19,35 +19,18 @@ license: |
<!--begin-include-->
| Key | Default | Description | Since |
| --- | ------- | ----------- | ----- |
-| celeborn.application.heartbeat.timeout | 300s | Application heartbeat timeout. | 0.2.0 |
-| celeborn.ha.enabled | false | When true, master nodes run as Raft cluster mode. | 0.2.0 |
-| celeborn.ha.master.node.<id>.host | <required> | Host to bind of master node <id> in HA mode. | 0.2.0 |
-| celeborn.ha.master.node.<id>.port | 9097 | Port to bind of master node <id> in HA mode. | 0.2.0 |
-| celeborn.ha.master.node.<id>.ratis.port | 9872 | Ratis port to bind of master node <id> in HA mode. | 0.2.0 |
-| celeborn.ha.master.ratis.raft.rpc.type | netty | RPC type for Ratis, available options: netty, grpc. | 0.2.0 |
-| celeborn.ha.master.ratis.raft.server.storage.dir | /tmp/ratis | | 0.2.0 |
+| celeborn.master.estimatedPartitionSize.initialSize | 64mb | Initial partition size for estimation, it will change according to runtime stats. | 0.2.0 |
+| celeborn.master.estimatedPartitionSize.update.initialDelay | 5min | Initial delay time before start updating partition size for estimation. | 0.2.0 |
+| celeborn.master.estimatedPartitionSize.update.interval | 10min | Interval of updating partition size for estimation. | 0.2.0 |
+| celeborn.master.heartbeat.application.timeout | 300s | Application heartbeat timeout. | 0.2.0 |
+| celeborn.master.heartbeat.worker.timeout | 120s | Worker heartbeat timeout. | 0.2.0 |
| celeborn.master.host | <localhost> | Hostname for master to bind. | 0.2.0 |
-| celeborn.master.metrics.prometheus.host | 0.0.0.0 | Master's Prometheus host. | 0.2.0 |
-| celeborn.master.metrics.prometheus.port | 9098 | Master's Prometheus port. | 0.2.0 |
| celeborn.master.port | 9097 | Port for master to bind. | 0.2.0 |
-| celeborn.metrics.app.topDiskUsage.count | 50 | Size for top items about top disk usage applications list. | 0.2.0 |
-| celeborn.metrics.app.topDiskUsage.interval | 10min | Time length for a window about top disk usage application list. | 0.2.0 |
-| celeborn.metrics.app.topDiskUsage.windowSize | 24 | Window size about top disk usage application list. | 0.2.0 |
-| celeborn.metrics.capacity | 4096 | The maximum number of metrics which a source can use to generate output strings. | 0.2.0 |
-| celeborn.metrics.collectPerfCritical.enabled | false | It controls whether to collect metrics which may affect performance. When enable, Celeborn collects them. | 0.2.0 |
-| celeborn.metrics.enabled | true | When true, enable metrics system. | 0.2.0 |
-| celeborn.metrics.extraLabels | | If default metric labels are not enough, extra metric labels can be customized. Labels' pattern is: `<label1_key>=<label1_value>[,<label2_key>=<label2_value>]*`; e.g. `env=prod,version=1` | 0.3.0 |
-| celeborn.metrics.sample.rate | 1.0 | It controls if Celeborn collect timer metrics for some operations. Its value should be in [0.0, 1.0]. | 0.2.0 |
-| celeborn.metrics.timer.slidingWindow.size | 4096 | The sliding window size of timer metric. | 0.2.0 |
-| celeborn.metrics.userResourceConsumption.update.interval | 30s | Time length for a window about compute user resource consumption. | 0.3.0 |
-| celeborn.shuffle.estimatedPartitionSize.update.initialDelay | 5min | Initial delay time before start updating partition size for estimation. | 0.2.0 |
-| celeborn.shuffle.estimatedPartitionSize.update.interval | 10min | Interval of updating partition size for estimation. | 0.2.0 |
-| celeborn.shuffle.initialEstimatedPartitionSize | 64mb | Initial partition size for estimation, it will change according to runtime stats. | 0.2.0 |
-| celeborn.slots.assign.extraSlots | 2 | Extra slots number when master assign slots. | 0.2.0 |
-| celeborn.slots.assign.loadAware.diskGroupGradient | 0.1 | This value means how many more workload will be placed into a faster disk group than a slower group. | 0.2.0 |
-| celeborn.slots.assign.loadAware.fetchTimeWeight | 1.0 | Weight of average fetch time when calculating ordering in load-aware assignment strategy | 0.2.1 |
-| celeborn.slots.assign.loadAware.flushTimeWeight | 0.0 | Weight of average flush time when calculating ordering in load-aware assignment strategy | 0.2.1 |
-| celeborn.slots.assign.loadAware.numDiskGroups | 5 | This configuration is a guidance for load-aware slot allocation algorithm. This value is control how many disk groups will be created. | 0.2.0 |
-| celeborn.slots.assign.policy | ROUNDROBIN | Policy for master to assign slots, Celeborn supports two types of policy: roundrobin and loadaware. | 0.2.0 |
-| celeborn.worker.heartbeat.timeout | 120s | Worker heartbeat timeout. | 0.2.0 |
+| celeborn.master.slot.assign.extraSlots | 2 | Extra slots number when master assign slots. | 0.2.0 |
+| celeborn.master.slot.assign.loadAware.diskGroupGradient | 0.1 | This value means how many more workload will be placed into a faster disk group than a slower group. | 0.2.0 |
+| celeborn.master.slot.assign.loadAware.fetchTimeWeight | 1.0 | Weight of average fetch time when calculating ordering in load-aware assignment strategy | 0.2.1 |
+| celeborn.master.slot.assign.loadAware.flushTimeWeight | 0.0 | Weight of average flush time when calculating ordering in load-aware assignment strategy | 0.2.1 |
+| celeborn.master.slot.assign.loadAware.numDiskGroups | 5 | This configuration is a guidance for load-aware slot allocation algorithm. This value is control how many disk groups will be created. | 0.2.0 |
+| celeborn.master.slot.assign.policy | ROUNDROBIN | Policy for master to assign slots, Celeborn supports two types of policy: roundrobin and loadaware. | 0.2.0 |
+| celeborn.master.userResourceConsumption.update.interval | 30s | Time length for a window about compute user resource consumption. | 0.3.0 |
<!--end-include-->
diff --git a/docs/configuration/metrics.md b/docs/configuration/metrics.md
index 6283b5e66..25aa9f7f0 100644
--- a/docs/configuration/metrics.md
+++ b/docs/configuration/metrics.md
@@ -19,14 +19,17 @@ license: |
<!--begin-include-->
| Key | Default | Description | Since |
| --- | ------- | ----------- | ----- |
-| celeborn.master.metrics.prometheus.host | 0.0.0.0 | Master's Prometheus host. | 0.2.0 |
-| celeborn.master.metrics.prometheus.port | 9098 | Master's Prometheus port. | 0.2.0 |
+| celeborn.metrics.app.topDiskUsage.count | 50 | Size for top items about top disk usage applications list. | 0.2.0 |
+| celeborn.metrics.app.topDiskUsage.interval | 10min | Time length for a window about top disk usage application list. | 0.2.0 |
+| celeborn.metrics.app.topDiskUsage.windowSize | 24 | Window size about top disk usage application list. | 0.2.0 |
| celeborn.metrics.capacity | 4096 | The maximum number of metrics which a source can use to generate output strings. | 0.2.0 |
| celeborn.metrics.collectPerfCritical.enabled | false | It controls whether to collect metrics which may affect performance. When enable, Celeborn collects them. | 0.2.0 |
| celeborn.metrics.enabled | true | When true, enable metrics system. | 0.2.0 |
| celeborn.metrics.extraLabels | | If default metric labels are not enough, extra metric labels can be customized. Labels' pattern is: `<label1_key>=<label1_value>[,<label2_key>=<label2_value>]*`; e.g. `env=prod,version=1` | 0.3.0 |
+| celeborn.metrics.master.prometheus.host | 0.0.0.0 | Master's Prometheus host. | 0.2.0 |
+| celeborn.metrics.master.prometheus.port | 9098 | Master's Prometheus port. | 0.2.0 |
| celeborn.metrics.sample.rate | 1.0 | It controls if Celeborn collect timer metrics for some operations. Its value should be in [0.0, 1.0]. | 0.2.0 |
| celeborn.metrics.timer.slidingWindow.size | 4096 | The sliding window size of timer metric. | 0.2.0 |
-| celeborn.worker.metrics.prometheus.host | 0.0.0.0 | Worker's Prometheus host. | 0.2.0 |
-| celeborn.worker.metrics.prometheus.port | 9096 | Worker's Prometheus port. | 0.2.0 |
+| celeborn.metrics.worker.prometheus.host | 0.0.0.0 | Worker's Prometheus host. | 0.2.0 |
+| celeborn.metrics.worker.prometheus.port | 9096 | Worker's Prometheus port. | 0.2.0 |
<!--end-include-->
diff --git a/docs/configuration/network.md b/docs/configuration/network.md
index 0df0f2f83..a113127bb 100644
--- a/docs/configuration/network.md
+++ b/docs/configuration/network.md
@@ -19,6 +19,7 @@ license: |
<!--begin-include-->
| Key | Default | Description | Since |
| --- | ------- | ----------- | ----- |
+| celeborn.<module>.heartbeat.interval | 60s | The heartbeat interval between worker and client. If setting <module> to `data`, it works for shuffle client push and fetch data and should be configured in client side. If setting <module> to `replicate`, it works for worker replicate data to peer worker and should be configured in worker side. | 0.3.0 |
| celeborn.<module>.io.backLog | 0 | Requested maximum length of the queue of incoming connections. Default 0 for no backlog. | |
| celeborn.<module>.io.clientThreads | 0 | Number of threads used in the client thread pool. Default to 0, which is 2x#cores. | |
| celeborn.<module>.io.connectTimeout | <value of celeborn.network.connect.timeout> | Socket connect timeout. | |
@@ -33,13 +34,17 @@ license: |
| celeborn.<module>.io.retryWait | 5s | Time that we will wait in order to perform a retry after an IOException. Only relevant if maxIORetries > 0. | 0.2.0 |
| celeborn.<module>.io.sendBuffer | 0b | Send buffer size (SO_SNDBUF). | 0.2.0 |
| celeborn.<module>.io.serverThreads | 0 | Number of threads used in the server thread pool. Default to 0, which is 2x#cores. | |
+| celeborn.<module>.push.timeoutCheck.interval | 5s | Interval for checking push data timeout. If setting <module> to `data`, it works for shuffle client push data and should be configured in client side. If setting <module> to `replicate`, it works for worker replicate data to peer worker and should be configured in worker side. | 0.3.0 |
+| celeborn.<module>.push.timeoutCheck.threads | 16 | Threads num for checking push data timeout. If setting <module> to `data`, it works for shuffle client push data and should be configured in client side. If setting <module> to `replicate`, it works for worker replicate data to peer worker and should be configured in worker side. | 0.3.0 |
| celeborn.network.connect.timeout | 10s | Default socket connect timeout. | 0.2.0 |
+| celeborn.network.memory.allocator.numArenas | <undefined> | Number of arenas for pooled memory allocator. Default value is Runtime.getRuntime.availableProcessors, min value is 2. | 0.3.0 |
+| celeborn.network.memory.allocator.share | false | Whether to share memory allocator. | 0.3.0 |
+| celeborn.network.memory.allocator.verbose.metric | false | Weather to enable verbose metric for pooled allocator. | 0.3.0 |
| celeborn.network.timeout | 240s | Default timeout for network operations. | 0.2.0 |
| celeborn.port.maxRetries | 1 | When port is occupied, we will retry for max retry times. | 0.2.0 |
| celeborn.rpc.askTimeout | 30s | Timeout for RPC ask operations. | 0.2.0 |
| celeborn.rpc.connect.threads | 64 | | 0.2.0 |
-| celeborn.rpc.haClient.askTimeout | <value of celeborn.rpc.askTimeout> | Timeout for HA client RPC ask operations. | 0.2.0 |
| celeborn.rpc.io.threads | <undefined> | Netty IO thread number of NettyRpcEnv to handle RPC request. The default threads number is the number of runtime available processors. | 0.2.0 |
| celeborn.rpc.lookupTimeout | 30s | Timeout for RPC lookup operations. | 0.2.0 |
-| celeborn.shuffle.maxChunksBeingTransferred | 9223372036854775807 | The max number of chunks allowed to be transferred at the same time on shuffle service. Note that new incoming connections will be closed when the max number is hit. The client will retry according to the shuffle retry configs (see `celeborn.shuffle.io.maxRetries` and `celeborn.shuffle.io.retryWait`), if those limits are reached the task will fail with fetch failure. | 0.2.0 |
+| celeborn.shuffle.io.maxChunksBeingTransferred | 9223372036854775807 | The max number of chunks allowed to be transferred at the same time on shuffle service. Note that new incoming connections will be closed when the max number is hit. The client will retry according to the shuffle retry configs (see `celeborn.<module>.io.maxRetries` and `celeborn.<module>.io.retryWait`), if those limits are reached the task will fail with fetch failure. | 0.2.0 |
<!--end-include-->
diff --git a/docs/configuration/worker.md b/docs/configuration/worker.md
index 039b69bdf..19bf9b6e0 100644
--- a/docs/configuration/worker.md
+++ b/docs/configuration/worker.md
@@ -19,27 +19,13 @@ license: |
<!--begin-include-->
| Key | Default | Description | Since |
| --- | ------- | ----------- | ----- |
-| celeborn.client.heartbeat.interval | 60s | the heartbeat interval between worker and client | 0.3.0 |
-| celeborn.client.maxRetries | 15 | Max retry times for client to connect master endpoint | 0.2.0 |
| celeborn.master.endpoints | <localhost>:9097 | Endpoints of master nodes for celeborn client to connect, allowed pattern is: `<host1>:<port1>[,<host2>:<port2>]*`, e.g. `clb1:9097,clb2:9098,clb3:9099`. If the port is omitted, 9097 will be used. | 0.2.0 |
-| celeborn.memory.allocator.numArenas | <undefined> | Number of arenas for pooled memory allocator. Default value is Runtime.getRuntime.availableProcessors, min value is 2. | 0.3.0 |
-| celeborn.memory.allocator.share | false | Whether to share memory allocator. | 0.3.0 |
-| celeborn.memory.allocator.verbose.metric | false | Weather to enable verbose metric for pooled allocator. | 0.3.0 |
-| celeborn.metrics.capacity | 4096 | The maximum number of metrics which a source can use to generate output strings. | 0.2.0 |
-| celeborn.metrics.collectPerfCritical.enabled | false | It controls whether to collect metrics which may affect performance. When enable, Celeborn collects them. | 0.2.0 |
-| celeborn.metrics.enabled | true | When true, enable metrics system. | 0.2.0 |
-| celeborn.metrics.extraLabels | | If default metric labels are not enough, extra metric labels can be customized. Labels' pattern is: `<label1_key>=<label1_value>[,<label2_key>=<label2_value>]*`; e.g. `env=prod,version=1` | 0.3.0 |
-| celeborn.metrics.sample.rate | 1.0 | It controls if Celeborn collect timer metrics for some operations. Its value should be in [0.0, 1.0]. | 0.2.0 |
-| celeborn.metrics.timer.slidingWindow.size | 4096 | The sliding window size of timer metric. | 0.2.0 |
-| celeborn.push.timeoutCheck.interval | 5s | Interval for checking push data timeout. | 0.3.0 |
-| celeborn.push.timeoutCheck.threads | 16 | Threads num for checking push data timeout. | 0.3.0 |
+| celeborn.master.estimatedPartitionSize.minSize | 8mb | Ignore partition size smaller than this configuration of partition size for estimation. | 0.2.0 |
| celeborn.shuffle.chunk.size | 8m | Max chunk size of reducer's merged shuffle data. For example, if a reducer's shuffle data is 128M and the data will need 16 fetch chunk requests to fetch. | 0.2.0 |
-| celeborn.shuffle.minPartitionSizeToEstimate | 8mb | Ignore partition size smaller than this configuration of partition size for estimation. | 0.2.0 |
-| celeborn.shuffle.partitionSplit.min | 1m | Min size for a partition to split | 0.2.0 |
-| celeborn.storage.hdfs.dir | <undefined> | HDFS dir configuration for Celeborn to access HDFS. | 0.2.0 |
| celeborn.worker.bufferStream.threadsPerMountpoint | 8 | Threads count for read buffer per mount point. | 0.3.0 |
| celeborn.worker.closeIdleConnections | false | Whether worker will close idle connections. | 0.2.0 |
-| celeborn.worker.commit.threads | 32 | Thread number of worker to commit shuffle data files asynchronously. | 0.2.0 |
+| celeborn.worker.commitFiles.threads | 32 | Thread number of worker to commit shuffle data files asynchronously. | 0.2.0 |
+| celeborn.worker.commitFiles.timeout | <value of celeborn.rpc.askTimeout> | Timeout for a Celeborn worker to commit files of a shuffle. | 0.2.0 |
| celeborn.worker.congestionControl.enabled | false | Whether to enable congestion control or not. | 0.3.0 |
| celeborn.worker.congestionControl.high.watermark | <undefined> | If the total bytes in disk buffer exceeds this configure, will start to congestusers whose produce rate is higher than the potential average consume rate. The congestion will stop if the produce rate is lower or equal to the average consume rate, or the total pending bytes lower than celeborn.worker.congestionControl.low.watermark | 0.3.0 |
| celeborn.worker.congestionControl.low.watermark | <undefined> | Will stop congest users if the total pending bytes of disk buffer is lower than this configuration | 0.3.0 |
@@ -50,15 +36,11 @@ license: |
| celeborn.worker.directMemoryRatioToPauseReceive | 0.85 | If direct memory usage reaches this limit, the worker will stop to receive data from Celeborn shuffle clients. | 0.2.0 |
| celeborn.worker.directMemoryRatioToPauseReplicate | 0.95 | If direct memory usage reaches this limit, the worker will stop to receive replication data from other workers. | 0.2.0 |
| celeborn.worker.directMemoryRatioToResume | 0.5 | If direct memory usage is less than this limit, worker will resume. | 0.2.0 |
-| celeborn.worker.disk.check.timeout | 30s | Timeout time for worker check device status. | 0.2.0 |
-| celeborn.worker.disk.checkFileClean.maxRetries | 3 | The number of retries for a worker to check if the working directory is cleaned up before registering with the master. | 0.2.0 |
-| celeborn.worker.disk.checkFileClean.timeout | 1000ms | The wait time per retry for a worker to check if the working directory is cleaned up before registering with the master. | 0.2.0 |
-| celeborn.worker.disk.reserve.size | 5G | Celeborn worker reserved space for each disk. | 0.2.0 |
-| celeborn.worker.diskTime.slidingWindow.size | 20 | The size of sliding windows used to calculate statistics about flushed time and count. | 0.2.1 |
| celeborn.worker.fetch.heartbeat.enabled | true | enable the heartbeat from worker to client when fetching data | 0.3.0 |
| celeborn.worker.fetch.io.threads | <undefined> | Netty IO thread number of worker to handle client fetch data. The default threads number is the number of flush thread. | 0.2.0 |
| celeborn.worker.fetch.port | 0 | Server port for Worker to receive fetch data request from ShuffleClient. | 0.2.0 |
| celeborn.worker.flusher.buffer.size | 256k | Size of buffer used by a single flusher. | 0.2.0 |
+| celeborn.worker.flusher.diskTime.slidingWindow.size | 20 | The size of sliding windows used to calculate statistics about flushed time and count. | 0.2.1 |
| celeborn.worker.flusher.hdd.threads | 1 | Flusher's thread count per disk used for write data to HDD disks. | 0.2.0 |
| celeborn.worker.flusher.hdfs.threads | 4 | Flusher's thread count used for write data to HDFS. | 0.2.0 |
| celeborn.worker.flusher.shutdownTimeout | 3s | Timeout for a flusher to shutdown. | 0.2.0 |
@@ -70,27 +52,21 @@ license: |
| celeborn.worker.graceful.shutdown.partitionSorter.shutdownTimeout | 120s | The wait time of waiting for sorting partition files during worker graceful shutdown. | 0.2.0 |
| celeborn.worker.graceful.shutdown.recoverPath | <tmp>/recover | The path to store levelDB. | 0.2.0 |
| celeborn.worker.graceful.shutdown.timeout | 600s | The worker's graceful shutdown timeout time. | 0.2.0 |
-| celeborn.worker.heartbeat.timeout | 120s | Worker heartbeat timeout. | 0.2.0 |
-| celeborn.worker.memory.checkInterval | 10ms | Interval of worker direct memory checking. | 0.2.0 |
-| celeborn.worker.memory.reportInterval | 10s | Interval of worker direct memory tracker reporting to log. | 0.2.0 |
-| celeborn.worker.memory.trimChannelWaitInterval | 1s | Wait time after worker trigger channel to trim cache. | 0.3.0 |
-| celeborn.worker.memory.trimFlushWaitInterval | 1s | Wait time after worker trigger StorageManger to flush data. | 0.3.0 |
-| celeborn.worker.metrics.prometheus.host | 0.0.0.0 | Worker's Prometheus host. | 0.2.0 |
-| celeborn.worker.metrics.prometheus.port | 9096 | Worker's Prometheus port. | 0.2.0 |
-| celeborn.worker.monitor.disk.checkInterval | 60s | Intervals between device monitor to check disk. | 0.2.0 |
+| celeborn.worker.monitor.disk.check.interval | 60s | Intervals between device monitor to check disk. | 0.2.0 |
+| celeborn.worker.monitor.disk.check.timeout | 30s | Timeout time for worker check device status. | 0.2.0 |
| celeborn.worker.monitor.disk.checklist | readwrite,diskusage | Monitor type for disk, available items are: iohang, readwrite and diskusage. | 0.2.0 |
| celeborn.worker.monitor.disk.enabled | true | When true, worker will monitor device and report to master. | 0.2.0 |
| celeborn.worker.monitor.disk.notifyError.expireTimeout | 10m | The expire timeout of non-critical device error. Only notify critical error when the number of non-critical errors for a period of time exceeds threshold. | 0.3.0 |
| celeborn.worker.monitor.disk.notifyError.threshold | 64 | Device monitor will only notify critical error once the accumulated valid non-critical error number exceeding this threshold. | 0.3.0 |
| celeborn.worker.monitor.disk.sys.block.dir | /sys/block | The directory where linux file block information is stored. | 0.2.0 |
-| celeborn.worker.noneEmptyDirExpireDuration | 1d | If a non-empty application shuffle data dir have not been operated during le duration time, will mark this application as expired. | 0.2.0 |
+| celeborn.worker.monitor.memory.check.interval | 10ms | Interval of worker direct memory checking. | 0.2.0 |
+| celeborn.worker.monitor.memory.report.interval | 10s | Interval of worker direct memory tracker reporting to log. | 0.2.0 |
+| celeborn.worker.monitor.memory.trimChannelWaitInterval | 1s | Wait time after worker trigger channel to trim cache. | 0.3.0 |
+| celeborn.worker.monitor.memory.trimFlushWaitInterval | 1s | Wait time after worker trigger StorageManger to flush data. | 0.3.0 |
| celeborn.worker.partition.initial.readBuffersMax | 1024 | Max number of initial read buffers | 0.3.0 |
| celeborn.worker.partition.initial.readBuffersMin | 1 | Min number of initial read buffers | 0.3.0 |
| celeborn.worker.partition.split.enabled | true | enable the partition split in worker side | 0.3.0 |
| celeborn.worker.partitionSorter.directMemoryRatioThreshold | 0.1 | Max ratio of partition sorter's memory for sorting, when reserved memory is higher than max partition sorter memory, partition sorter will stop sorting. | 0.2.0 |
-| celeborn.worker.partitionSorter.reservedMemoryPerPartition | 1mb | Reserved memory when sorting a shuffle file off-heap. | 0.2.0 |
-| celeborn.worker.partitionSorter.sort.timeout | 220s | Timeout for a shuffle file to sort. | 0.2.0 |
-| celeborn.worker.partitionSorter.threads | <undefined> | PartitionSorter's thread counts. | 0.3.0 |
| celeborn.worker.push.heartbeat.enabled | true | enable the heartbeat from worker to client when pushing data | 0.3.0 |
| celeborn.worker.push.io.threads | <undefined> | Netty IO thread number of worker to handle client push data. The default threads number is the number of flush thread. | 0.2.0 |
| celeborn.worker.push.port | 0 | Server port for Worker to receive push data request from ShuffleClient. | 0.2.0 |
@@ -106,11 +82,17 @@ license: |
| celeborn.worker.replicate.randomConnection.enabled | true | Whether worker will create random connection to peer when replicate data. When false, worker tend to reuse the same cached TransportClient to a specific replicate worker; when true, worker tend to use different cached TransportClient. Netty will use the same thread to serve the same connection, so with more connections replicate server can leverage more netty threads | 0.2.1 |
| celeborn.worker.replicate.threads | 64 | Thread number of worker to replicate shuffle data. | 0.2.0 |
| celeborn.worker.rpc.port | 0 | Server port for Worker to receive RPC request. | 0.2.0 |
-| celeborn.worker.shuffle.commit.timeout | <value of celeborn.rpc.askTimeout> | Timeout for a Celeborn worker to commit files of a shuffle. | 0.2.0 |
-| celeborn.worker.storage.baseDir.number | 16 | How many directories will be used if `celeborn.worker.storage.dirs` is not set. The directory name is a combination of `celeborn.worker.storage.baseDir.prefix` and from one(inclusive) to `celeborn.worker.storage.baseDir.number`(inclusive) step by one. | 0.2.0 |
-| celeborn.worker.storage.baseDir.prefix | /mnt/disk | Base directory for Celeborn worker to write if `celeborn.worker.storage.dirs` is not set. | 0.2.0 |
+| celeborn.worker.shuffle.partitionSplit.min | 1m | Min size for a partition to split | 0.2.0 |
+| celeborn.worker.sortPartition.reservedMemoryPerPartition | 1mb | Reserved memory when sorting a shuffle file off-heap. | 0.2.0 |
+| celeborn.worker.sortPartition.threads | <undefined> | PartitionSorter's thread counts. | 0.3.0 |
+| celeborn.worker.sortPartition.timeout | 220s | Timeout for a shuffle file to sort. | 0.2.0 |
+| celeborn.worker.storage.applicationData.keepAliveTime | 1d | If a non-empty application shuffle data dir have not been operated during le duration time, will mark this application as expired. | 0.2.0 |
+| celeborn.worker.storage.checkDirsEmpty.maxRetries | 3 | The number of retries for a worker to check if the working directory is cleaned up before registering with the master. | 0.2.0 |
+| celeborn.worker.storage.checkDirsEmpty.timeout | 1000ms | The wait time per retry for a worker to check if the working directory is cleaned up before registering with the master. | 0.2.0 |
| celeborn.worker.storage.dirs | <undefined> | Directory list to store shuffle data. It's recommended to configure one directory on each disk. Storage size limit can be set for each directory. For the sake of performance, there should be no more than 2 flush threads on the same disk partition if you are using HDD, and should be 8 or more flush threads on the same disk partition if you are using SSD. For example: `dir1[:capacity=][:disktype=][:flushthread=],dir2[:capacity=][:disktyp [...]
-| celeborn.worker.workingDir | rss-worker/shuffle_data | Worker's working dir path name. | 0.2.0 |
+| celeborn.worker.storage.disk.reserve.size | 5G | Celeborn worker reserved space for each disk. | 0.2.0 |
+| celeborn.worker.storage.hdfs.dir | <undefined> | HDFS dir configuration for Celeborn to access HDFS. | 0.2.0 |
+| celeborn.worker.storage.workingDir | rss-worker/shuffle_data | Worker's working dir path name. | 0.2.0 |
| celeborn.worker.writer.close.timeout | 120s | Timeout for a file writer to close | 0.2.0 |
| celeborn.worker.writer.create.maxAttempts | 3 | Retry count for a file writer to create if its creation was failed. | 0.2.0 |
<!--end-include-->
diff --git a/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala b/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
index cbb703e37..4056b8739 100644
--- a/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
+++ b/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
@@ -98,7 +98,7 @@ private[celeborn] class Master(
private val appHeartbeatTimeoutMs = conf.appHeartbeatTimeoutMs
private val quotaManager = QuotaManager.instantiate(conf)
- private val metricsResourceConsumptionInterval = conf.metricsResourceConsumptionInterval
+ private val masterResourceConsumptionInterval = conf.masterResourceConsumptionInterval
private val userResourceConsumptions =
JavaUtils.newConcurrentHashMap[UserIdentifier, (ResourceConsumption, Long)]()
@@ -676,7 +676,7 @@ private[celeborn] class Master(
val current = System.currentTimeMillis()
if (userResourceConsumptions.containsKey(userIdentifier)) {
val resourceConsumptionAndUpdateTime = userResourceConsumptions.get(userIdentifier)
- if (current - resourceConsumptionAndUpdateTime._2 > metricsResourceConsumptionInterval) {
+ if (current - resourceConsumptionAndUpdateTime._2 > masterResourceConsumptionInterval) {
val newResourceConsumption = statusSystem.workers.asScala.flatMap { workerInfo =>
workerInfo.userResourceConsumption.asScala.get(userIdentifier)
}.foldRight(ResourceConsumption(0, 0, 0, 0))(_ add _)
diff --git a/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/RatisBaseSuiteJ.java b/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/RatisBaseSuiteJ.java
index 2dc8028f5..a4466372f 100644
--- a/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/RatisBaseSuiteJ.java
+++ b/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/RatisBaseSuiteJ.java
@@ -37,7 +37,7 @@ public class RatisBaseSuiteJ {
File tmpDir1 = File.createTempFile("rss-ratis-tmp", "for-test-only");
tmpDir1.delete();
tmpDir1.mkdirs();
- conf.set("celeborn.ha.master.ratis.raft.server.storage.dir", tmpDir1.getAbsolutePath());
+ conf.set(CelebornConf.HA_MASTER_RATIS_STORAGE_DIR().key(), tmpDir1.getAbsolutePath());
String id = UUID.randomUUID().toString();
int ratisPort = 9999;
MasterNode masterNode =
diff --git a/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/RatisMasterStatusSystemSuiteJ.java b/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/RatisMasterStatusSystemSuiteJ.java
index fe5f4271d..c16462b4e 100644
--- a/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/RatisMasterStatusSystemSuiteJ.java
+++ b/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/RatisMasterStatusSystemSuiteJ.java
@@ -91,19 +91,19 @@ public class RatisMasterStatusSystemSuiteJ {
File tmpDir1 = File.createTempFile("rss-ratis1", "for-test-only");
tmpDir1.delete();
tmpDir1.mkdirs();
- conf1.set("celeborn.ha.master.ratis.raft.server.storage.dir", tmpDir1.getAbsolutePath());
+ conf1.set(CelebornConf.HA_MASTER_RATIS_STORAGE_DIR().key(), tmpDir1.getAbsolutePath());
CelebornConf conf2 = new CelebornConf();
File tmpDir2 = File.createTempFile("rss-ratis2", "for-test-only");
tmpDir2.delete();
tmpDir2.mkdirs();
- conf2.set("celeborn.ha.master.ratis.raft.server.storage.dir", tmpDir2.getAbsolutePath());
+ conf2.set(CelebornConf.HA_MASTER_RATIS_STORAGE_DIR().key(), tmpDir2.getAbsolutePath());
CelebornConf conf3 = new CelebornConf();
File tmpDir3 = File.createTempFile("rss-ratis3", "for-test-only");
tmpDir3.delete();
tmpDir3.mkdirs();
- conf3.set("celeborn.ha.master.ratis.raft.server.storage.dir", tmpDir3.getAbsolutePath());
+ conf3.set(CelebornConf.HA_MASTER_RATIS_STORAGE_DIR().key(), tmpDir3.getAbsolutePath());
String id1 = UUID.randomUUID().toString();
String id2 = UUID.randomUUID().toString();
diff --git a/master/src/test/scala/org/apache/celeborn/service/deploy/master/AppDiskUsageMetricSuite.scala b/master/src/test/scala/org/apache/celeborn/service/deploy/master/AppDiskUsageMetricSuite.scala
index 62b7fd5a3..00f600bb7 100644
--- a/master/src/test/scala/org/apache/celeborn/service/deploy/master/AppDiskUsageMetricSuite.scala
+++ b/master/src/test/scala/org/apache/celeborn/service/deploy/master/AppDiskUsageMetricSuite.scala
@@ -62,8 +62,8 @@ class AppDiskUsageMetricSuite extends AnyFunSuite
Thread.sleep(5000)
val conf = new CelebornConf()
- conf.set("celeborn.metrics.app.topDiskUsage.windowSize", "5")
- conf.set("celeborn.metrics.app.topDiskUsage.interval", "2s")
+ conf.set(CelebornConf.METRICS_APP_TOP_DISK_USAGE_WINDOW_SIZE.key, "5")
+ conf.set(CelebornConf.METRICS_APP_TOP_DISK_USAGE_INTERVAL.key, "2s")
val usageMetric = new AppDiskUsageMetric(conf)
val map1 = new util.HashMap[String, java.lang.Long]()
diff --git a/master/src/test/scala/org/apache/celeborn/service/deploy/master/MasterSuite.scala b/master/src/test/scala/org/apache/celeborn/service/deploy/master/MasterSuite.scala
index 519127772..5fceab10b 100644
--- a/master/src/test/scala/org/apache/celeborn/service/deploy/master/MasterSuite.scala
+++ b/master/src/test/scala/org/apache/celeborn/service/deploy/master/MasterSuite.scala
@@ -36,12 +36,12 @@ class MasterSuite extends AnyFunSuite
test("test single node startup functionality") {
val conf = new CelebornConf()
- conf.set("celeborn.ha.enabled", "false")
- conf.set("celeborn.ha.master.ratis.raft.server.storage.dir", getTmpDir())
- conf.set("celeborn.worker.storage.dirs", getTmpDir())
- conf.set("celeborn.metrics.enabled", "true")
- conf.set("celeborn.master.metrics.prometheus.host", "127.0.0.1")
- conf.set("celeborn.master.metrics.prometheus.port", "11112")
+ conf.set(CelebornConf.HA_ENABLED.key, "false")
+ conf.set(CelebornConf.HA_MASTER_RATIS_STORAGE_DIR.key, getTmpDir())
+ conf.set(CelebornConf.WORKER_STORAGE_DIRS.key, getTmpDir())
+ conf.set(CelebornConf.METRICS_ENABLED.key, "true")
+ conf.set(CelebornConf.MASTER_PROMETHEUS_HOST.key, "127.0.0.1")
+ conf.set(CelebornConf.MASTER_PROMETHEUS_PORT.key, "11112")
val args = Array("-h", "localhost", "-p", "9097")
diff --git a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/client/ShuffleClientSuite.scala b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/client/ShuffleClientSuite.scala
index d02293380..15d2f22f3 100644
--- a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/client/ShuffleClientSuite.scala
+++ b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/client/ShuffleClientSuite.scala
@@ -26,9 +26,9 @@ import org.apache.celeborn.service.deploy.MiniClusterFeature
class ShuffleClientSuite extends WithShuffleClientSuite with MiniClusterFeature {
private val masterPort = 19097
- celebornConf.set("celeborn.master.endpoints", s"localhost:$masterPort")
- .set("celeborn.push.replicate.enabled", "true")
- .set("celeborn.push.buffer.max.size", "256K")
+ celebornConf.set(CelebornConf.MASTER_ENDPOINTS.key, s"localhost:$masterPort")
+ .set(CelebornConf.PUSH_REPLICATE_ENABLED.key, "true")
+ .set(CelebornConf.PUSH_BUFFER_MAX_SIZE.key, "256K")
override def beforeAll(): Unit = {
super.beforeAll()
@@ -42,8 +42,8 @@ class ShuffleClientSuite extends WithShuffleClientSuite with MiniClusterFeature
test("test register when master not available") {
val celebornConf: CelebornConf = new CelebornConf()
- celebornConf.set("celeborn.master.endpoints", "localhost:19098")
- celebornConf.set("celeborn.client.maxRetries", "0")
+ celebornConf.set(CelebornConf.MASTER_ENDPOINTS.key, "localhost:19098")
+ celebornConf.set(CelebornConf.MASTER_CLIENT_MAX_RETRIES.key, "0")
val lifecycleManager: LifecycleManager = new LifecycleManager(APP, celebornConf)
val shuffleClient: ShuffleClientImpl = {
diff --git a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/PushDataTimeoutTest.scala b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/PushDataTimeoutTest.scala
index bc18a91ca..a45db2322 100644
--- a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/PushDataTimeoutTest.scala
+++ b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/PushDataTimeoutTest.scala
@@ -23,6 +23,7 @@ import org.scalatest.BeforeAndAfterEach
import org.scalatest.funsuite.AnyFunSuite
import org.apache.celeborn.client.ShuffleClient
+import org.apache.celeborn.common.CelebornConf
class PushDataTimeoutTest extends AnyFunSuite
with SparkTestBase
@@ -48,10 +49,10 @@ class PushDataTimeoutTest extends AnyFunSuite
test("celeborn spark integration test - pushdata timeout") {
Seq("false", "true").foreach { enabled =>
val sparkConf = new SparkConf().setAppName("rss-demo").setMaster("local[2]")
- .set("spark.celeborn.push.data.timeout", "5s")
- .set("spark.celeborn.push.timeoutCheck.interval", "2s")
- .set("spark.celeborn.push.replicate.enabled", enabled)
- .set("spark.celeborn.client.blacklistSlave.enabled", "false")
+ .set(s"spark.${CelebornConf.PUSH_DATA_TIMEOUT.key}", "5s")
+ .set(s"spark.celeborn.data.push.timeoutCheck.interval", "2s")
+ .set(s"spark.${CelebornConf.PUSH_REPLICATE_ENABLED.key}", enabled)
+ .set(s"spark.${CelebornConf.BLACKLIST_SLAVE_ENABLED.key}", "false")
val sparkSession = SparkSession.builder().config(sparkConf).getOrCreate()
val combineResult = combine(sparkSession)
val groupbyResult = groupBy(sparkSession)
@@ -81,9 +82,9 @@ class PushDataTimeoutTest extends AnyFunSuite
test("celeborn spark integration test - pushdata timeout will add to balcklist") {
val sparkConf = new SparkConf().setAppName("rss-demo").setMaster("local[2]")
- .set("spark.celeborn.push.data.timeout", "5s")
- .set("spark.celeborn.push.replicate.enabled", "true")
- .set("spark.celeborn.client.blacklistSlave.enabled", "true")
+ .set(s"spark.${CelebornConf.PUSH_DATA_TIMEOUT.key}", "5s")
+ .set(s"spark.celeborn.data.push.timeoutCheck.interval", "2s")
+ .set(s"spark.${CelebornConf.BLACKLIST_SLAVE_ENABLED.key}", "false")
val rssSparkSession = SparkSession.builder()
.config(updateSparkConf(sparkConf, false)).getOrCreate()
try {
diff --git a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/RetryCommitFilesTest.scala b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/RetryCommitFilesTest.scala
index 9c4d71b9a..4a4483c11 100644
--- a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/RetryCommitFilesTest.scala
+++ b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/RetryCommitFilesTest.scala
@@ -23,6 +23,7 @@ import org.scalatest.BeforeAndAfterEach
import org.scalatest.funsuite.AnyFunSuite
import org.apache.celeborn.client.ShuffleClient
+import org.apache.celeborn.common.CelebornConf
class RetryCommitFilesTest extends AnyFunSuite
with SparkTestBase
@@ -45,7 +46,7 @@ class RetryCommitFilesTest extends AnyFunSuite
test("celeborn spark integration test - retry commit files") {
val sparkConf = new SparkConf()
- .set("spark.celeborn.test.retryCommitFiles", "true")
+ .set(s"spark.${CelebornConf.TEST_RETRY_COMMIT_FILE.key}", "true")
.setAppName("rss-demo").setMaster("local[2]")
val ss = SparkSession.builder().config(updateSparkConf(sparkConf, false)).getOrCreate()
ss.sparkContext.parallelize(1 to 1000, 2)
diff --git a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/RetryReviveTest.scala b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/RetryReviveTest.scala
index 145247d9f..1f825533b 100644
--- a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/RetryReviveTest.scala
+++ b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/RetryReviveTest.scala
@@ -23,6 +23,7 @@ import org.scalatest.BeforeAndAfterEach
import org.scalatest.funsuite.AnyFunSuite
import org.apache.celeborn.client.ShuffleClient
+import org.apache.celeborn.common.CelebornConf
class RetryReviveTest extends AnyFunSuite
with SparkTestBase
@@ -43,7 +44,7 @@ class RetryReviveTest extends AnyFunSuite
test("celeborn spark integration test - retry revive as configured times") {
val sparkConf = new SparkConf()
- .set("spark.celeborn.test.retryRevive", "true")
+ .set(s"spark.${CelebornConf.TEST_RETRY_REVIVE.key}", "true")
.setAppName("rss-demo").setMaster("local[2]")
val ss = SparkSession.builder().config(updateSparkConf(sparkConf, false)).getOrCreate()
ss.sparkContext.parallelize(1 to 1000, 2)
diff --git a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/RssNonPipelineSortSuite.scala b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/RssNonPipelineSortSuite.scala
index 1e24d968c..a07d0a53e 100644
--- a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/RssNonPipelineSortSuite.scala
+++ b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/RssNonPipelineSortSuite.scala
@@ -23,6 +23,7 @@ import org.scalatest.BeforeAndAfterEach
import org.scalatest.funsuite.AnyFunSuite
import org.apache.celeborn.client.ShuffleClient
+import org.apache.celeborn.common.CelebornConf
class RssNonPipelineSortSuite extends AnyFunSuite
with SparkTestBase
@@ -38,8 +39,8 @@ class RssNonPipelineSortSuite extends AnyFunSuite
test("celeborn spark integration test - non pipeline sort") {
val sparkConf = new SparkConf().setAppName("rss-demo").setMaster("local[2]")
- .set("spark.celeborn.push.sort.pipeline.enabled", "false")
- .set("spark.celeborn.push.sort.randomizePartitionId.enabled", "false")
+ .set(s"spark.${CelebornConf.PUSH_SORT_PIPELINE_ENABLED.key}", "false")
+ .set(s"spark.${CelebornConf.PUSH_SORT_RANDOMIZE_PARITION_ENABLED.key}", "false")
val sparkSession = SparkSession.builder().config(sparkConf).getOrCreate()
val combineResult = combine(sparkSession)
val groupbyResult = groupBy(sparkSession)
diff --git a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/RssPipelineSortSuite.scala b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/RssPipelineSortSuite.scala
index 59d909271..71cd729a0 100644
--- a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/RssPipelineSortSuite.scala
+++ b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/RssPipelineSortSuite.scala
@@ -23,6 +23,7 @@ import org.scalatest.BeforeAndAfterEach
import org.scalatest.funsuite.AnyFunSuite
import org.apache.celeborn.client.ShuffleClient
+import org.apache.celeborn.common.CelebornConf
class RssPipelineSortSuite extends AnyFunSuite
with SparkTestBase
@@ -38,8 +39,8 @@ class RssPipelineSortSuite extends AnyFunSuite
test("celeborn spark integration test - pipeline sort") {
val sparkConf = new SparkConf().setAppName("rss-demo").setMaster("local[2]")
- .set("spark.celeborn.push.sort.pipeline.enabled", "true")
- .set("spark.celeborn.push.sort.randomizePartitionId.enabled", "true")
+ .set(s"spark.${CelebornConf.PUSH_SORT_PIPELINE_ENABLED.key}", "true")
+ .set(s"spark.${CelebornConf.PUSH_SORT_RANDOMIZE_PARITION_ENABLED.key}", "true")
val ss = SparkSession.builder().config(updateSparkConf(sparkConf, true)).getOrCreate()
val value = Range(1, 10000).mkString(",")
val tuples = ss.sparkContext.parallelize(1 to 10000, 2)
diff --git a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/ShuffleFallbackSuite.scala b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/ShuffleFallbackSuite.scala
index fd3227d57..b1158d8b2 100644
--- a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/ShuffleFallbackSuite.scala
+++ b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/ShuffleFallbackSuite.scala
@@ -25,6 +25,7 @@ import org.scalatest.BeforeAndAfterEach
import org.scalatest.funsuite.AnyFunSuite
import org.apache.celeborn.client.ShuffleClient
+import org.apache.celeborn.common.CelebornConf
import org.apache.celeborn.common.protocol.CompressionCodec
class ShuffleFallbackSuite extends AnyFunSuite
@@ -41,13 +42,13 @@ class ShuffleFallbackSuite extends AnyFunSuite
private def enableRss(conf: SparkConf) = {
conf.set("spark.shuffle.manager", "org.apache.spark.shuffle.celeborn.RssShuffleManager")
- .set("spark.celeborn.master.endpoints", masterInfo._1.rpcEnv.address.toString)
+ .set(s"spark.${CelebornConf.MASTER_ENDPOINTS.key}", masterInfo._1.rpcEnv.address.toString)
}
test(s"celeborn spark integration test - fallback") {
val sparkConf = new SparkConf().setAppName("rss-demo")
.setMaster("local[2]")
- .set("spark.celeborn.shuffle.forceFallback.enabled", "true")
+ .set(s"spark.${CelebornConf.SHUFFLE_FORCE_FALLBACK_ENABLED.key}", "true")
enableRss(sparkConf)
diff --git a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/SkewJoinSuite.scala b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/SkewJoinSuite.scala
index 0be380dd9..437b71179 100644
--- a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/SkewJoinSuite.scala
+++ b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/SkewJoinSuite.scala
@@ -25,6 +25,7 @@ import org.scalatest.BeforeAndAfterEach
import org.scalatest.funsuite.AnyFunSuite
import org.apache.celeborn.client.ShuffleClient
+import org.apache.celeborn.common.CelebornConf
import org.apache.celeborn.common.protocol.CompressionCodec
class SkewJoinSuite extends AnyFunSuite
@@ -41,8 +42,8 @@ class SkewJoinSuite extends AnyFunSuite
private def enableRss(conf: SparkConf) = {
conf.set("spark.shuffle.manager", "org.apache.spark.shuffle.celeborn.RssShuffleManager")
- .set("spark.celeborn.master.endpoints", masterInfo._1.rpcEnv.address.toString)
- .set("spark.celeborn.shuffle.partitionSplit.threshold", "10MB")
+ .set(s"spark.${CelebornConf.MASTER_ENDPOINTS.key}", masterInfo._1.rpcEnv.address.toString)
+ .set(s"spark.${CelebornConf.PARTITION_SPLIT_THRESHOLD.key}", "10MB")
}
CompressionCodec.values.foreach { codec =>
@@ -57,8 +58,8 @@ class SkewJoinSuite extends AnyFunSuite
.set("spark.sql.adaptive.autoBroadcastJoinThreshold", "-1")
.set("spark.sql.autoBroadcastJoinThreshold", "-1")
.set("spark.sql.parquet.compression.codec", "gzip")
- .set("spark.celeborn.shuffle.compression.codec", codec.name)
- .set("spark.celeborn.shuffle.rangeReadFilter.enabled", "true")
+ .set(s"spark.${CelebornConf.SHUFFLE_COMPRESSION_CODEC.key}", codec.name)
+ .set(s"spark.${CelebornConf.SHUFFLE_RANGE_READ_FILTER_ENABLED.key}", "true")
enableRss(sparkConf)
diff --git a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/SparkTestBase.scala b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/SparkTestBase.scala
index 2c1cc3a51..f627fdf13 100644
--- a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/SparkTestBase.scala
+++ b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/SparkTestBase.scala
@@ -24,6 +24,7 @@ import org.apache.spark.sql.SparkSession
import org.scalatest.BeforeAndAfterAll
import org.scalatest.funsuite.AnyFunSuite
+import org.apache.celeborn.common.CelebornConf
import org.apache.celeborn.common.internal.Logging
import org.apache.celeborn.service.deploy.MiniClusterFeature
@@ -52,9 +53,11 @@ trait SparkTestBase extends AnyFunSuite
sparkConf.set("spark.shuffle.service.enabled", "false")
sparkConf.set("spark.sql.adaptive.skewJoin.enabled", "false")
sparkConf.set("spark.sql.adaptive.localShuffleReader.enabled", "false")
- sparkConf.set("spark.celeborn.master.endpoints", masterInfo._1.rpcEnv.address.toString)
+ sparkConf.set(
+ s"spark${CelebornConf.MASTER_ENDPOINTS.key}",
+ masterInfo._1.rpcEnv.address.toString)
if (sort) {
- sparkConf.set("spark.celeborn.shuffle.writer", "sort")
+ sparkConf.set(s"spark.${CelebornConf.SHUFFLE_WRITER_MODE.key}", "sort")
}
sparkConf
}
diff --git a/toolkit/scripts/genConfs.py b/toolkit/scripts/genConfs.py
index 8ab51a820..b1d693033 100644
--- a/toolkit/scripts/genConfs.py
+++ b/toolkit/scripts/genConfs.py
@@ -100,9 +100,9 @@ def set_rss_confs(conf, replicate=False):
conf["spark.shuffle.service.enabled"] = "false"
if replicate:
- conf["spark.celeborn.push.replicate.enabled"] = "true"
+ conf["spark.celeborn.client.push.replicate.enabled"] = "true"
else:
- conf["spark.celeborn.push.replicate.enabled"] = "false"
+ conf["spark.celeborn.client.push.replicate.enabled"] = "false"
def save_ess_conf(dir):
@@ -173,7 +173,7 @@ def update_hibench_confs(hiben_conf_dir):
normal_hibench_spark_conf_file_path = hibench_spark_conf_file + ".rss"
save_confs(open(normal_hibench_spark_conf_file_path, "w"), new_hibench_spark_conf)
splits_hibench_spark_conf_file_path = hibench_spark_conf_file + ".split"
- new_hibench_spark_conf["spark.celeborn.shuffle.partitionSplit.threshold"] = "16m"
+ new_hibench_spark_conf["spark.celeborn.client.shuffle.partitionSplit.threshold"] = "16m"
save_confs(open(splits_hibench_spark_conf_file_path, "w"), new_hibench_spark_conf)
diff --git a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/FetchHandler.scala b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/FetchHandler.scala
index 689c76f95..a7370d252 100644
--- a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/FetchHandler.scala
+++ b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/FetchHandler.scala
@@ -39,12 +39,13 @@ import org.apache.celeborn.common.util.ExceptionUtils
import org.apache.celeborn.service.deploy.worker.storage.{ChunkStreamManager, CreditStreamManager, PartitionFilesSorter, StorageManager}
class FetchHandler(val conf: TransportConf) extends BaseMessageHandler with Logging {
+ var celebornConf = conf.getCelebornConf
var chunkStreamManager = new ChunkStreamManager()
val creditStreamManager = new CreditStreamManager(
- conf.getCelebornConf.partitionReadBuffersMin,
- conf.getCelebornConf.partitionReadBuffersMax,
- conf.getCelebornConf.creditStreamThreadsPerMountpoint,
- conf.getCelebornConf.readBuffersToTriggerReadMin)
+ celebornConf.partitionReadBuffersMin,
+ celebornConf.partitionReadBuffersMax,
+ celebornConf.creditStreamThreadsPerMountpoint,
+ celebornConf.readBuffersToTriggerReadMin)
var workerSource: WorkerSource = _
var storageManager: StorageManager = _
var partitionsSorter: PartitionFilesSorter = _
@@ -214,10 +215,10 @@ class FetchHandler(val conf: TransportConf) extends BaseMessageHandler with Logg
s" to fetch block ${req.streamChunkSlice}")
val chunksBeingTransferred = chunkStreamManager.chunksBeingTransferred
- if (chunksBeingTransferred > conf.maxChunksBeingTransferred) {
+ if (chunksBeingTransferred > celebornConf.shuffleIoMaxChunksBeingTransferred) {
val message = "Worker is too busy. The number of chunks being transferred " +
s"$chunksBeingTransferred exceeds celeborn.shuffle.maxChunksBeingTransferred " +
- s"${conf.maxChunksBeingTransferred}."
+ s"${celebornConf.shuffleIoMaxChunksBeingTransferred}."
logError(message)
client.getChannel.writeAndFlush(
new ChunkFetchFailure(req.streamChunkSlice, message))
diff --git a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/DeviceMonitor.scala b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/DeviceMonitor.scala
index 7041a84ff..4dcd2c04c 100644
--- a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/DeviceMonitor.scala
+++ b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/DeviceMonitor.scala
@@ -256,7 +256,7 @@ object DeviceMonitor {
highDiskUsage
})(false)(
deviceCheckThreadPool,
- conf.workerDeviceStatusCheckTimeout,
+ conf.diskMonitorStatusCheckTimeout,
s"Disk: ${diskInfo.mountPoint} Usage Check Timeout")
}
@@ -302,7 +302,7 @@ object DeviceMonitor {
}
})(false)(
deviceCheckThreadPool,
- conf.workerDeviceStatusCheckTimeout,
+ conf.diskMonitorStatusCheckTimeout,
s"Disk: $dataDir Read_Write Check Timeout")
}
diff --git a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
index 24840bc7a..ef6c7c5f7 100644
--- a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
+++ b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
@@ -197,7 +197,7 @@ final private[worker] class StorageManager(conf: CelebornConf, workerSource: Abs
logWarning(
"Worker still has residual files in the working directory before registering with Master, " +
"please refer to the configuration document to increase celeborn.worker.disk.checkFileClean.maxRetries or " +
- "celeborn.worker.disk.checkFileClean.timeout .")
+ s"${CelebornConf.WORKER_CHECK_FILE_CLEAN_TIMEOUT.key}.")
} else {
logInfo("Successfully remove all files under working directory.")
}
@@ -488,7 +488,7 @@ final private[worker] class StorageManager(conf: CelebornConf, workerSource: Abs
}
}
- private val noneEmptyDirExpireDurationMs = conf.workerNonEmptyDirExpireDuration
+ private val applicationDataKeepAliveTime = conf.workerApplicationDataKeepAliveTime
private val storageScheduler =
ThreadUtils.newDaemonSingleThreadScheduledExecutor("storage-scheduler")
@@ -498,7 +498,7 @@ final private[worker] class StorageManager(conf: CelebornConf, workerSource: Abs
try {
// Clean up dirs which has not been modified
// in the past {{noneEmptyExpireDurationsMs}}.
- cleanupExpiredAppDirs(System.currentTimeMillis() - noneEmptyDirExpireDurationMs)
+ cleanupExpiredAppDirs(System.currentTimeMillis() - applicationDataKeepAliveTime)
} catch {
case exception: Exception =>
logWarning(s"Cleanup expired shuffle data exception: ${exception.getMessage}")
diff --git a/worker/src/test/java/org/apache/celeborn/service/deploy/worker/storage/CreditStreamManagerSuiteJ.java b/worker/src/test/java/org/apache/celeborn/service/deploy/worker/storage/CreditStreamManagerSuiteJ.java
index ac2e220f0..c9d7ba68a 100644
--- a/worker/src/test/java/org/apache/celeborn/service/deploy/worker/storage/CreditStreamManagerSuiteJ.java
+++ b/worker/src/test/java/org/apache/celeborn/service/deploy/worker/storage/CreditStreamManagerSuiteJ.java
@@ -49,15 +49,15 @@ public class CreditStreamManagerSuiteJ {
@BeforeClass
public static void beforeAll() {
CelebornConf conf = new CelebornConf();
- conf.set("celeborn.worker.directMemoryRatioToPauseReceive", "0.8");
- conf.set("celeborn.worker.directMemoryRatioToPauseReplicate", "0.9");
- conf.set("celeborn.worker.directMemoryRatioToResume", "0.5");
- conf.set("celeborn.worker.partitionSorter.directMemoryRatioThreshold", "0.6");
- conf.set("celeborn.worker.directMemoryRatioForReadBuffer", "0.1");
- conf.set("celeborn.worker.directMemoryRatioForMemoryShuffleStorage", "0.1");
- conf.set("celeborn.worker.memory.checkInterval", "10");
- conf.set("celeborn.worker.memory.reportInterval", "10");
- conf.set("celeborn.worker.readBuffer.allocationWait", "10ms");
+ conf.set(CelebornConf.WORKER_DIRECT_MEMORY_RATIO_PAUSE_RECEIVE().key(), "0.8");
+ conf.set(CelebornConf.WORKER_DIRECT_MEMORY_RATIO_PAUSE_REPLICATE().key(), "0.9");
+ conf.set(CelebornConf.WORKER_DIRECT_MEMORY_RATIO_RESUME().key(), "0.5");
+ conf.set(CelebornConf.PARTITION_SORTER_DIRECT_MEMORY_RATIO_THRESHOLD().key(), "0.6");
+ conf.set(CelebornConf.WORKER_DIRECT_MEMORY_RATIO_FOR_READ_BUFFER().key(), "0.1");
+ conf.set(CelebornConf.WORKER_DIRECT_MEMORY_RATIO_FOR_SHUFFLE_STORAGE().key(), "0.1");
+ conf.set(CelebornConf.WORKER_DIRECT_MEMORY_CHECK_INTERVAL().key(), "10");
+ conf.set(CelebornConf.WORKER_DIRECT_MEMORY_REPORT_INTERVAL().key(), "10");
+ conf.set(CelebornConf.WORKER_READBUFFER_ALLOCATIONWAIT().key(), "10ms");
MemoryManager.initialize(conf);
}
diff --git a/worker/src/test/java/org/apache/celeborn/service/deploy/worker/storage/FileWriterSuiteJ.java b/worker/src/test/java/org/apache/celeborn/service/deploy/worker/storage/FileWriterSuiteJ.java
index a285c2463..9de94a593 100644
--- a/worker/src/test/java/org/apache/celeborn/service/deploy/worker/storage/FileWriterSuiteJ.java
+++ b/worker/src/test/java/org/apache/celeborn/service/deploy/worker/storage/FileWriterSuiteJ.java
@@ -101,7 +101,7 @@ public class FileWriterSuiteJ {
@BeforeClass
public static void beforeAll() {
tempDir = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "celeborn");
- CONF.set("celeborn.shuffle.chunk.size", "1k");
+ CONF.set(CelebornConf.SHUFFLE_CHUNK_SIZE().key(), "1k");
source = Mockito.mock(WorkerSource.class);
Mockito.doAnswer(
@@ -119,15 +119,15 @@ public class FileWriterSuiteJ {
source, DeviceMonitor$.MODULE$.EmptyMonitor(), 1, "disk1", StorageInfo.Type.HDD, null);
CelebornConf conf = new CelebornConf();
- conf.set("celeborn.worker.directMemoryRatioToPauseReceive", "0.8");
- conf.set("celeborn.worker.directMemoryRatioToPauseReplicate", "0.9");
- conf.set("celeborn.worker.directMemoryRatioToResume", "0.5");
- conf.set("celeborn.worker.partitionSorter.directMemoryRatioThreshold", "0.6");
- conf.set("celeborn.worker.directMemoryRatioForReadBuffer", "0.1");
- conf.set("celeborn.worker.directMemoryRatioForMemoryShuffleStorage", "0.1");
- conf.set("celeborn.worker.memory.checkInterval", "10");
- conf.set("celeborn.worker.memory.reportInterval", "10");
- conf.set("celeborn.worker.readBuffer.allocationWait", "10ms");
+ conf.set(CelebornConf.WORKER_DIRECT_MEMORY_RATIO_PAUSE_RECEIVE().key(), "0.8");
+ conf.set(CelebornConf.WORKER_DIRECT_MEMORY_RATIO_PAUSE_REPLICATE().key(), "0.9");
+ conf.set(CelebornConf.WORKER_DIRECT_MEMORY_RATIO_RESUME().key(), "0.5");
+ conf.set(CelebornConf.PARTITION_SORTER_DIRECT_MEMORY_RATIO_THRESHOLD().key(), "0.6");
+ conf.set(CelebornConf.WORKER_DIRECT_MEMORY_RATIO_FOR_READ_BUFFER().key(), "0.1");
+ conf.set(CelebornConf.WORKER_DIRECT_MEMORY_RATIO_FOR_SHUFFLE_STORAGE().key(), "0.1");
+ conf.set(CelebornConf.WORKER_DIRECT_MEMORY_CHECK_INTERVAL().key(), "10");
+ conf.set(CelebornConf.WORKER_DIRECT_MEMORY_REPORT_INTERVAL().key(), "10");
+ conf.set(CelebornConf.WORKER_READBUFFER_ALLOCATIONWAIT().key(), "10ms");
MemoryManager.initialize(conf);
}
diff --git a/worker/src/test/java/org/apache/celeborn/service/deploy/worker/storage/MapPartitionFileWriterSuiteJ.java b/worker/src/test/java/org/apache/celeborn/service/deploy/worker/storage/MapPartitionFileWriterSuiteJ.java
index 43fe130d1..ca2b67b31 100644
--- a/worker/src/test/java/org/apache/celeborn/service/deploy/worker/storage/MapPartitionFileWriterSuiteJ.java
+++ b/worker/src/test/java/org/apache/celeborn/service/deploy/worker/storage/MapPartitionFileWriterSuiteJ.java
@@ -84,15 +84,15 @@ public class MapPartitionFileWriterSuiteJ {
source, DeviceMonitor$.MODULE$.EmptyMonitor(), 1, "disk1", StorageInfo.Type.HDD, null);
CelebornConf conf = new CelebornConf();
- conf.set("celeborn.worker.directMemoryRatioToPauseReceive", "0.8");
- conf.set("celeborn.worker.directMemoryRatioToPauseReplicate", "0.9");
- conf.set("celeborn.worker.directMemoryRatioToResume", "0.5");
- conf.set("celeborn.worker.partitionSorter.directMemoryRatioThreshold", "0.6");
- conf.set("celeborn.worker.directMemoryRatioForReadBuffer", "0.1");
- conf.set("celeborn.worker.directMemoryRatioForMemoryShuffleStorage", "0.1");
- conf.set("celeborn.worker.memory.checkInterval", "10");
- conf.set("celeborn.worker.memory.reportInterval", "10");
- conf.set("celeborn.worker.readBuffer.allocationWait", "10ms");
+ conf.set(CelebornConf.WORKER_DIRECT_MEMORY_RATIO_PAUSE_RECEIVE().key(), "0.8");
+ conf.set(CelebornConf.WORKER_DIRECT_MEMORY_RATIO_PAUSE_REPLICATE().key(), "0.9");
+ conf.set(CelebornConf.WORKER_DIRECT_MEMORY_RATIO_RESUME().key(), "0.5");
+ conf.set(CelebornConf.PARTITION_SORTER_DIRECT_MEMORY_RATIO_THRESHOLD().key(), "0.6");
+ conf.set(CelebornConf.WORKER_DIRECT_MEMORY_RATIO_FOR_READ_BUFFER().key(), "0.1");
+ conf.set(CelebornConf.WORKER_DIRECT_MEMORY_RATIO_FOR_SHUFFLE_STORAGE().key(), "0.1");
+ conf.set(CelebornConf.WORKER_DIRECT_MEMORY_CHECK_INTERVAL().key(), "10");
+ conf.set(CelebornConf.WORKER_DIRECT_MEMORY_REPORT_INTERVAL().key(), "10");
+ conf.set(CelebornConf.WORKER_READBUFFER_ALLOCATIONWAIT().key(), "10ms");
MemoryManager.initialize(conf);
}
diff --git a/worker/src/test/java/org/apache/celeborn/service/deploy/worker/storage/PartitionFilesSorterSuiteJ.java b/worker/src/test/java/org/apache/celeborn/service/deploy/worker/storage/PartitionFilesSorterSuiteJ.java
index d08ed9f96..9d83324c2 100644
--- a/worker/src/test/java/org/apache/celeborn/service/deploy/worker/storage/PartitionFilesSorterSuiteJ.java
+++ b/worker/src/test/java/org/apache/celeborn/service/deploy/worker/storage/PartitionFilesSorterSuiteJ.java
@@ -111,15 +111,15 @@ public class PartitionFilesSorterSuiteJ {
+ "MB");
CelebornConf conf = new CelebornConf();
- conf.set("celeborn.worker.directMemoryRatioToPauseReceive", "0.8");
- conf.set("celeborn.worker.directMemoryRatioToPauseReplicate", "0.9");
- conf.set("celeborn.worker.directMemoryRatioToResume", "0.5");
- conf.set("celeborn.worker.partitionSorter.directMemoryRatioThreshold", "0.6");
- conf.set("celeborn.worker.directMemoryRatioForReadBuffer", "0.1");
- conf.set("celeborn.worker.directMemoryRatioForMemoryShuffleStorage", "0.1");
- conf.set("celeborn.worker.memory.checkInterval", "10");
- conf.set("celeborn.worker.memory.reportInterval", "10");
- conf.set("celeborn.worker.readBuffer.allocationWait", "10ms");
+ conf.set(CelebornConf.WORKER_DIRECT_MEMORY_RATIO_PAUSE_RECEIVE().key(), "0.8");
+ conf.set(CelebornConf.WORKER_DIRECT_MEMORY_RATIO_PAUSE_REPLICATE().key(), "0.9");
+ conf.set(CelebornConf.WORKER_DIRECT_MEMORY_RATIO_RESUME().key(), "0.5");
+ conf.set(CelebornConf.PARTITION_SORTER_DIRECT_MEMORY_RATIO_THRESHOLD().key(), "0.6");
+ conf.set(CelebornConf.WORKER_DIRECT_MEMORY_RATIO_FOR_READ_BUFFER().key(), "0.1");
+ conf.set(CelebornConf.WORKER_DIRECT_MEMORY_RATIO_FOR_SHUFFLE_STORAGE().key(), "0.1");
+ conf.set(CelebornConf.WORKER_DIRECT_MEMORY_CHECK_INTERVAL().key(), "10");
+ conf.set(CelebornConf.WORKER_DIRECT_MEMORY_REPORT_INTERVAL().key(), "10");
+ conf.set(CelebornConf.WORKER_READBUFFER_ALLOCATIONWAIT().key(), "10ms");
MemoryManager.initialize(conf);
fileWriter = Mockito.mock(FileWriter.class);
when(fileWriter.getFile()).thenAnswer(i -> shuffleFile);
@@ -169,8 +169,8 @@ public class PartitionFilesSorterSuiteJ {
}
File recoverPath = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "recover_path");
CelebornConf conf = new CelebornConf();
- conf.set("celeborn.worker.graceful.shutdown.enabled", "true");
- conf.set("celeborn.worker.graceful.shutdown.recoverPath", recoverPath.getPath());
+ conf.set(CelebornConf.WORKER_GRACEFUL_SHUTDOWN_ENABLED().key(), "true");
+ conf.set(CelebornConf.WORKER_GRACEFUL_SHUTDOWN_RECOVER_PATH(), recoverPath.getPath());
PartitionFilesSorter partitionFilesSorter =
new PartitionFilesSorter(MemoryManager.instance(), conf, new WorkerSource(conf));
partitionFilesSorter.initSortedShuffleFiles("application-1-1");
diff --git a/worker/src/test/scala/org/apache/celeborn/service/deploy/HeartbeatFeature.scala b/worker/src/test/scala/org/apache/celeborn/service/deploy/HeartbeatFeature.scala
index 7120362fd..13f409422 100644
--- a/worker/src/test/scala/org/apache/celeborn/service/deploy/HeartbeatFeature.scala
+++ b/worker/src/test/scala/org/apache/celeborn/service/deploy/HeartbeatFeature.scala
@@ -33,8 +33,8 @@ trait HeartbeatFeature extends MiniClusterFeature {
assertFunc: (TransportClient, TransportClient) => Unit): Unit = {
logInfo("test initialized , setup rss mini cluster")
val masterConf = Map(
- "celeborn.master.host" -> "localhost",
- "celeborn.master.port" -> "9097")
+ CelebornConf.MASTER_HOST.key -> "localhost",
+ CelebornConf.MASTER_PORT.key -> "9097")
var master: Master = null
var workers: collection.Set[Worker] = null
try {
@@ -65,13 +65,15 @@ trait HeartbeatFeature extends MiniClusterFeature {
def getTestHeartbeatFromWorker2ClientConf(): (Map[String, String], CelebornConf) = {
val workerConf = Map(
- "celeborn.master.endpoints" -> "localhost:9097",
- "celeborn.client.heartbeat.interval" -> "4s")
+ CelebornConf.MASTER_ENDPOINTS.key -> "localhost:9097",
+ "celeborn.push.heartbeat.interval" -> "4s",
+ "celeborn.fetch.heartbeat.interval" -> "4s")
val clientConf = new CelebornConf()
clientConf.set("celeborn.data.io.connectionTimeout", "6s")
- clientConf.set("celeborn.client.heartbeat.interval", "3s")
+ clientConf.set("celeborn.data.heartbeat.interval", "3s")
(workerConf, clientConf)
}
+
def testHeartbeatFromWorker2Client(dataClientFactory: TransportClientFactory): Unit = {
val (workerConf, _) = getTestHeartbeatFromWorker2ClientConf()
// client <- worker:default client do not send heartbeat to worker, and worker sends hearbeat to client
@@ -91,7 +93,8 @@ trait HeartbeatFeature extends MiniClusterFeature {
: (Map[String, String], CelebornConf) = {
val workerConf = Map(
"celeborn.master.endpoints" -> "localhost:9097",
- "celeborn.client.heartbeat.interval" -> "4s",
+ "celeborn.push.heartbeat.interval" -> "4s",
+ "celeborn.fetch.heartbeat.interval" -> "4s",
"celeborn.worker.push.heartbeat.enabled" -> "false",
"celeborn.worker.fetch.heartbeat.enabled" -> "false")
val clientConf = new CelebornConf()
@@ -119,11 +122,12 @@ trait HeartbeatFeature extends MiniClusterFeature {
def getTestHeartbeatFromWorker2ClientWithCloseChannelConf()
: (Map[String, String], CelebornConf) = {
val workerConf = Map(
- "celeborn.master.endpoints" -> "localhost:9097",
+ CelebornConf.MASTER_ENDPOINTS.key -> "localhost:9097",
"celeborn.fetch.io.connectionTimeout" -> "9s",
"celeborn.push.io.connectionTimeout" -> "9s",
- "celeborn.client.heartbeat.interval" -> "4s",
- "celeborn.worker.closeIdleConnections" -> "true")
+ "celeborn.push.heartbeat.interval" -> "4s",
+ "celeborn.fetch.heartbeat.interval" -> "4s",
+ CelebornConf.WORKER_CLOSE_IDLE_CONNECTIONS.key -> "true")
val clientConf = new CelebornConf()
clientConf.set("celeborn.data.io.connectionTimeout", "6s")
(workerConf, clientConf)
diff --git a/worker/src/test/scala/org/apache/celeborn/service/deploy/MiniClusterFeature.scala b/worker/src/test/scala/org/apache/celeborn/service/deploy/MiniClusterFeature.scala
index 40f5760b4..67449638d 100644
--- a/worker/src/test/scala/org/apache/celeborn/service/deploy/MiniClusterFeature.scala
+++ b/worker/src/test/scala/org/apache/celeborn/service/deploy/MiniClusterFeature.scala
@@ -50,9 +50,9 @@ trait MiniClusterFeature extends Logging {
private def createMaster(map: Map[String, String] = null): Master = {
val conf = new CelebornConf()
- conf.set("celeborn.metrics.enabled", "false")
+ conf.set(CelebornConf.METRICS_ENABLED.key, "false")
val prometheusPort = masterPrometheusPort.getAndIncrement()
- conf.set("celeborn.master.metrics.prometheus.port", s"$prometheusPort")
+ conf.set(CelebornConf.MASTER_PROMETHEUS_PORT.key, s"$prometheusPort")
logInfo(s"set celeborn.master.metrics.prometheus.port to $prometheusPort")
if (map != null) {
map.foreach(m => conf.set(m._1, m._2))
@@ -69,12 +69,10 @@ trait MiniClusterFeature extends Logging {
private def createWorker(map: Map[String, String] = null): Worker = {
logInfo("start create worker for mini cluster")
val conf = new CelebornConf()
- conf.set("celeborn.worker.storage.dirs", createTmpDir())
- conf.set("celeborn.worker.monitor.disk.enabled", "false")
- conf.set("celeborn.push.buffer.max.size", "256K")
- conf.set(
- "celeborn.worker.metrics.prometheus.port",
- s"${workerPrometheusPort.incrementAndGet()}")
+ conf.set(CelebornConf.WORKER_STORAGE_DIRS.key, createTmpDir())
+ conf.set(CelebornConf.WORKER_DISK_MONITOR_ENABLED.key, "false")
+ conf.set(CelebornConf.PUSH_BUFFER_MAX_SIZE.key, "256K")
+ conf.set(CelebornConf.WORKER_PROMETHEUS_PORT.key, s"${workerPrometheusPort.incrementAndGet()}")
conf.set("celeborn.fetch.io.threads", "4")
conf.set("celeborn.push.io.threads", "4")
if (map != null) {
diff --git a/worker/src/test/scala/org/apache/celeborn/service/deploy/cluster/ReadWriteTestBase.scala b/worker/src/test/scala/org/apache/celeborn/service/deploy/cluster/ReadWriteTestBase.scala
index a3207e647..c60c032b5 100644
--- a/worker/src/test/scala/org/apache/celeborn/service/deploy/cluster/ReadWriteTestBase.scala
+++ b/worker/src/test/scala/org/apache/celeborn/service/deploy/cluster/ReadWriteTestBase.scala
@@ -55,10 +55,10 @@ trait ReadWriteTestBase extends AnyFunSuite
val APP = "app-1"
val clientConf = new CelebornConf()
- .set("celeborn.master.endpoints", s"localhost:$masterPort")
- .set("celeborn.shuffle.compression.codec", codec.name)
- .set("celeborn.push.replicate.enabled", "true")
- .set("celeborn.push.buffer.max.size", "256K")
+ .set(CelebornConf.MASTER_ENDPOINTS.key, s"localhost:$masterPort")
+ .set(CelebornConf.SHUFFLE_COMPRESSION_CODEC.key, codec.name)
+ .set(CelebornConf.PUSH_REPLICATE_ENABLED.key, "true")
+ .set(CelebornConf.PUSH_BUFFER_MAX_SIZE.key, "256K")
.set("celeborn.data.io.numConnectionsPerPeer", "1")
val lifecycleManager = new LifecycleManager(APP, clientConf)
val shuffleClient = new ShuffleClientImpl(clientConf, UserIdentifier("mock", "mock"))
diff --git a/worker/src/test/scala/org/apache/celeborn/service/deploy/worker/storage/WorkerSuite.scala b/worker/src/test/scala/org/apache/celeborn/service/deploy/worker/storage/WorkerSuite.scala
index 7bfc51a29..7eb992aff 100644
--- a/worker/src/test/scala/org/apache/celeborn/service/deploy/worker/storage/WorkerSuite.scala
+++ b/worker/src/test/scala/org/apache/celeborn/service/deploy/worker/storage/WorkerSuite.scala
@@ -37,7 +37,7 @@ class WorkerSuite extends AnyFunSuite {
val conf = new CelebornConf()
val workerArgs = new WorkerArguments(Array(), conf)
test("clean up") {
- conf.set("celeborn.worker.storage.dirs", "/tmp")
+ conf.set(CelebornConf.WORKER_STORAGE_DIRS.key, "/tmp")
val worker = new Worker(conf, workerArgs)
val pl1 = new PartitionLocation(0, 0, "12", 0, 0, 0, 0, PartitionLocation.Mode.MASTER)
@@ -73,7 +73,7 @@ class WorkerSuite extends AnyFunSuite {
}
test("flush filewriters") {
- conf.set("celeborn.worker.storage.dirs", "/tmp")
+ conf.set(CelebornConf.WORKER_STORAGE_DIRS.key, "/tmp")
val worker = new Worker(conf, workerArgs)
val dir = new File("/tmp")
val allWriters = new util.HashSet[FileWriter]()