You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by an...@apache.org on 2024/02/26 03:33:29 UTC
(incubator-celeborn) branch main updated: [CELEBORN-1277] Add celeborn.quota.enabled at Master and Client side to enable checking quota
This is an automated email from the ASF dual-hosted git repository.
angerszhuuuu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 7b0211e34 [CELEBORN-1277] Add celeborn.quota.enabled at Master and Client side to enable checking quota
7b0211e34 is described below
commit 7b0211e345751edb1e66e2760d9cae4d852adffc
Author: Angerszhuuuu <an...@gmail.com>
AuthorDate: Mon Feb 26 11:33:14 2024 +0800
[CELEBORN-1277] Add celeborn.quota.enabled at Master and Client side to enable checking quota
### What changes were proposed in this pull request?
Add `celeborn.quota.enabled` at Master and Client side to enable checking quota
### Why are the changes needed?
`celeborn.quota.enabled` should be added in Master and Client side to enable quota check for Celeborn Master and Client.
### Does this PR introduce _any_ user-facing change?
Add categories of `celeborn.quota,enabled` with `master` and `client`.
### How was this patch tested?
No.
Closes #2318 from AngersZhuuuu/CELEBORN-1277.
Authored-by: Angerszhuuuu <an...@gmail.com>
Signed-off-by: Angerszhuuuu <an...@gmail.com>
---
.../org/apache/celeborn/common/CelebornConf.scala | 19 +++++++++++--------
docs/configuration/client.md | 4 ++++
docs/configuration/master.md | 2 ++
docs/configuration/quota.md | 2 +-
.../celeborn/service/deploy/master/Master.scala | 10 +++++++---
5 files changed, 25 insertions(+), 12 deletions(-)
diff --git a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
index 8b0d9ef1d..0588f23fa 100644
--- a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
+++ b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
@@ -4147,17 +4147,20 @@ object CelebornConf extends Logging {
val QUOTA_ENABLED: ConfigEntry[Boolean] =
buildConf("celeborn.quota.enabled")
- .categories("quota")
- .doc("When true, before registering shuffle, LifecycleManager should check " +
- "if current user have enough quota space, if cluster don't have enough " +
- "quota space for current user, fallback to Spark's default shuffle")
+ .categories("quota", "master", "client")
+ .doc(
+ "When Master side sets to true, the master will enable to check the quota via QuotaManager. " +
+ "When Client side sets to true, LifecycleManager will request Master side to check " +
+ "whether the current user has enough quota before registration of shuffle. " +
+ "Fallback to the default shuffle service of Spark when Master side checks that " +
+ "there is no enough quota for current user.")
.version("0.2.0")
.booleanConf
.createWithDefault(true)
val QUOTA_IDENTITY_PROVIDER: ConfigEntry[String] =
buildConf("celeborn.quota.identity.provider")
- .categories("quota")
+ .categories("quota", "client")
.doc(s"IdentityProvider class name. Default class is " +
s"`${classOf[DefaultIdentityProvider].getName}`. " +
s"Optional values: " +
@@ -4169,7 +4172,7 @@ object CelebornConf extends Logging {
val QUOTA_USER_SPECIFIC_TENANT: ConfigEntry[String] =
buildConf("celeborn.quota.identity.user-specific.tenant")
- .categories("quota")
+ .categories("quota", "client")
.doc(s"Tenant id if celeborn.quota.identity.provider is org.apache.celeborn.common.identity.DefaultIdentityProvider.")
.version("0.3.0")
.stringConf
@@ -4177,7 +4180,7 @@ object CelebornConf extends Logging {
val QUOTA_USER_SPECIFIC_USERNAME: ConfigEntry[String] =
buildConf("celeborn.quota.identity.user-specific.userName")
- .categories("quota")
+ .categories("quota", "client")
.doc(s"User name if celeborn.quota.identity.provider is org.apache.celeborn.common.identity.DefaultIdentityProvider.")
.version("0.3.0")
.stringConf
@@ -4185,7 +4188,7 @@ object CelebornConf extends Logging {
val QUOTA_CONFIGURATION_PATH: OptionalConfigEntry[String] =
buildConf("celeborn.quota.configuration.path")
- .categories("quota")
+ .categories("quota", "master")
.doc("Quota configuration file path. The file format should be yaml. Quota configuration file template can be " +
"found under conf directory.")
.version("0.2.0")
diff --git a/docs/configuration/client.md b/docs/configuration/client.md
index b3de107cc..acf3522ec 100644
--- a/docs/configuration/client.md
+++ b/docs/configuration/client.md
@@ -111,6 +111,10 @@ license: |
| celeborn.client.spark.shuffle.forceFallback.numPartitionsThreshold | 2147483647 | false | Celeborn will only accept shuffle of partition number lower than this configuration value. | 0.3.0 | celeborn.shuffle.forceFallback.numPartitionsThreshold |
| celeborn.client.spark.shuffle.writer | HASH | false | Celeborn supports the following kind of shuffle writers. 1. hash: hash-based shuffle writer works fine when shuffle partition count is normal; 2. sort: sort-based shuffle writer works fine when memory pressure is high or shuffle partition count is huge. This configuration only takes effect when celeborn.client.spark.push.dynamicWriteMode.enabled is false. | 0.3.0 | celeborn.shuffle.writer |
| celeborn.master.endpoints | <localhost>:9097 | false | Endpoints of master nodes for celeborn client to connect, allowed pattern is: `<host1>:<port1>[,<host2>:<port2>]*`, e.g. `clb1:9097,clb2:9098,clb3:9099`. If the port is omitted, 9097 will be used. | 0.2.0 | |
+| celeborn.quota.enabled | true | false | When Master side sets to true, the master will enable to check the quota via QuotaManager. When Client side sets to true, LifecycleManager will request Master side to check whether the current user has enough quota before registration of shuffle. Fallback to the default shuffle service of Spark when Master side checks that there is no enough quota for current user. | 0.2.0 | |
+| celeborn.quota.identity.provider | org.apache.celeborn.common.identity.DefaultIdentityProvider | false | IdentityProvider class name. Default class is `org.apache.celeborn.common.identity.DefaultIdentityProvider`. Optional values: org.apache.celeborn.common.identity.HadoopBasedIdentityProvider user name will be obtained by UserGroupInformation.getUserName; org.apache.celeborn.common.identity.DefaultIdentityProvider user name and tenant id are default values or user-specific values. | 0 [...]
+| celeborn.quota.identity.user-specific.tenant | default | false | Tenant id if celeborn.quota.identity.provider is org.apache.celeborn.common.identity.DefaultIdentityProvider. | 0.3.0 | |
+| celeborn.quota.identity.user-specific.userName | default | false | User name if celeborn.quota.identity.provider is org.apache.celeborn.common.identity.DefaultIdentityProvider. | 0.3.0 | |
| celeborn.storage.availableTypes | HDD | false | Enabled storages. Available options: MEMORY,HDD,SSD,HDFS. Note: HDD and SSD would be treated as identical. | 0.3.0 | celeborn.storage.activeTypes |
| celeborn.storage.hdfs.dir | <undefined> | false | HDFS base directory for Celeborn to store shuffle data. | 0.2.0 | |
<!--end-include-->
diff --git a/docs/configuration/master.md b/docs/configuration/master.md
index 9cc78f5e7..ad2aaedec 100644
--- a/docs/configuration/master.md
+++ b/docs/configuration/master.md
@@ -53,6 +53,8 @@ license: |
| celeborn.master.slot.assign.policy | ROUNDROBIN | false | Policy for master to assign slots, Celeborn supports two types of policy: roundrobin and loadaware. Loadaware policy will be ignored when `HDFS` is enabled in `celeborn.storage.activeTypes` | 0.3.0 | celeborn.slots.assign.policy |
| celeborn.master.userResourceConsumption.update.interval | 30s | false | Time length for a window about compute user resource consumption. | 0.3.0 | |
| celeborn.master.workerUnavailableInfo.expireTimeout | 1800s | false | Worker unavailable info would be cleared when the retention period is expired | 0.3.1 | |
+| celeborn.quota.configuration.path | <undefined> | false | Quota configuration file path. The file format should be yaml. Quota configuration file template can be found under conf directory. | 0.2.0 | |
+| celeborn.quota.enabled | true | false | When Master side sets to true, the master will enable to check the quota via QuotaManager. When Client side sets to true, LifecycleManager will request Master side to check whether the current user has enough quota before registration of shuffle. Fallback to the default shuffle service of Spark when Master side checks that there is no enough quota for current user. | 0.2.0 | |
| celeborn.storage.availableTypes | HDD | false | Enabled storages. Available options: MEMORY,HDD,SSD,HDFS. Note: HDD and SSD would be treated as identical. | 0.3.0 | celeborn.storage.activeTypes |
| celeborn.storage.hdfs.dir | <undefined> | false | HDFS base directory for Celeborn to store shuffle data. | 0.2.0 | |
| celeborn.storage.hdfs.kerberos.keytab | <undefined> | false | Kerberos keytab file path for HDFS storage connection. | 0.3.2 | |
diff --git a/docs/configuration/quota.md b/docs/configuration/quota.md
index ab729846b..9d36f3c02 100644
--- a/docs/configuration/quota.md
+++ b/docs/configuration/quota.md
@@ -20,7 +20,7 @@ license: |
| Key | Default | isDynamic | Description | Since | Deprecated |
| --- | ------- | --------- | ----------- | ----- | ---------- |
| celeborn.quota.configuration.path | <undefined> | false | Quota configuration file path. The file format should be yaml. Quota configuration file template can be found under conf directory. | 0.2.0 | |
-| celeborn.quota.enabled | true | false | When true, before registering shuffle, LifecycleManager should check if current user have enough quota space, if cluster don't have enough quota space for current user, fallback to Spark's default shuffle | 0.2.0 | |
+| celeborn.quota.enabled | true | false | When Master side sets to true, the master will enable to check the quota via QuotaManager. When Client side sets to true, LifecycleManager will request Master side to check whether the current user has enough quota before registration of shuffle. Fallback to the default shuffle service of Spark when Master side checks that there is no enough quota for current user. | 0.2.0 | |
| celeborn.quota.identity.provider | org.apache.celeborn.common.identity.DefaultIdentityProvider | false | IdentityProvider class name. Default class is `org.apache.celeborn.common.identity.DefaultIdentityProvider`. Optional values: org.apache.celeborn.common.identity.HadoopBasedIdentityProvider user name will be obtained by UserGroupInformation.getUserName; org.apache.celeborn.common.identity.DefaultIdentityProvider user name and tenant id are default values or user-specific values. | 0 [...]
| celeborn.quota.identity.user-specific.tenant | default | false | Tenant id if celeborn.quota.identity.provider is org.apache.celeborn.common.identity.DefaultIdentityProvider. | 0.3.0 | |
| celeborn.quota.identity.user-specific.userName | default | false | User name if celeborn.quota.identity.provider is org.apache.celeborn.common.identity.DefaultIdentityProvider. | 0.3.0 | |
diff --git a/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala b/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
index c309dd216..6ea45c741 100644
--- a/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
+++ b/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
@@ -1037,9 +1037,13 @@ private[celeborn] class Master(
userIdentifier: UserIdentifier,
context: RpcCallContext): Unit = {
val userResourceConsumption = handleResourceConsumption(userIdentifier)
- val (isAvailable, reason) =
- quotaManager.checkQuotaSpaceAvailable(userIdentifier, userResourceConsumption)
- context.reply(CheckQuotaResponse(isAvailable, reason))
+ if (conf.quotaEnabled) {
+ val (isAvailable, reason) =
+ quotaManager.checkQuotaSpaceAvailable(userIdentifier, userResourceConsumption)
+ context.reply(CheckQuotaResponse(isAvailable, reason))
+ } else {
+ context.reply(CheckQuotaResponse(true, ""))
+ }
}
private def handleCheckWorkersAvailable(context: RpcCallContext): Unit = {