You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@celeborn.apache.org by "waitinfuture (via GitHub)" <gi...@apache.org> on 2023/01/31 12:16:29 UTC
[GitHub] [incubator-celeborn] waitinfuture commented on a diff in pull request #1182: [CELEBORN-247] Add metrics for each user's quota usage
waitinfuture commented on code in PR #1182:
URL: https://github.com/apache/incubator-celeborn/pull/1182#discussion_r1091842862
##########
master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala:
##########
@@ -624,12 +625,34 @@ private[celeborn] class Master(
context.reply(OneWayMessageResponse)
}
+ private def computeUserResourceConsumption(userIdentifier: UserIdentifier)
+ : ResourceConsumption = {
+ statusSystem.workers.asScala.flatMap { workerInfo =>
+ workerInfo.userResourceConsumption.asScala.get(userIdentifier)
+ }.foldRight(ResourceConsumption(0, 0, 0, 0))(_ add _)
+ }
+
private def handleCheckQuota(
userIdentifier: UserIdentifier,
context: RpcCallContext): Unit = {
- val userResourceConsumption = statusSystem.workers.asScala.flatMap { workerInfo =>
- workerInfo.userResourceConsumption.asScala.get(userIdentifier)
- }.foldRight(ResourceConsumption(0, 0, 0, 0))(_ add _)
+ resourceConsumptionSource.addGauge(
+ "diskFIleCount",
+ _ => computeUserResourceConsumption(userIdentifier).diskFileCount,
+ userIdentifier.toMap)
+ resourceConsumptionSource.addGauge(
+ "diskBytesWritten",
+ _ => computeUserResourceConsumption(userIdentifier).diskBytesWritten,
+ userIdentifier.toMap)
+ resourceConsumptionSource.addGauge(
+ "hdfsFileCount",
+ _ => computeUserResourceConsumption(userIdentifier).hdfsFileCount,
+ userIdentifier.toMap)
+ resourceConsumptionSource.addGauge(
+ "hdfsBytesWritten",
+ _ => computeUserResourceConsumption(userIdentifier).hdfsBytesWritten,
+ userIdentifier.toMap)
Review Comment:
In fact, I think we can cache computeUserResourceConsumption for say 30s period in master to further release the burden. Master periodically compute the consumption, handleCheckQuota merely reads it.
##########
master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala:
##########
@@ -624,12 +625,34 @@ private[celeborn] class Master(
context.reply(OneWayMessageResponse)
}
+ private def computeUserResourceConsumption(userIdentifier: UserIdentifier)
+ : ResourceConsumption = {
+ statusSystem.workers.asScala.flatMap { workerInfo =>
+ workerInfo.userResourceConsumption.asScala.get(userIdentifier)
+ }.foldRight(ResourceConsumption(0, 0, 0, 0))(_ add _)
+ }
+
private def handleCheckQuota(
userIdentifier: UserIdentifier,
context: RpcCallContext): Unit = {
- val userResourceConsumption = statusSystem.workers.asScala.flatMap { workerInfo =>
- workerInfo.userResourceConsumption.asScala.get(userIdentifier)
- }.foldRight(ResourceConsumption(0, 0, 0, 0))(_ add _)
+ resourceConsumptionSource.addGauge(
+ "diskFIleCount",
+ _ => computeUserResourceConsumption(userIdentifier).diskFileCount,
+ userIdentifier.toMap)
+ resourceConsumptionSource.addGauge(
+ "diskBytesWritten",
+ _ => computeUserResourceConsumption(userIdentifier).diskBytesWritten,
+ userIdentifier.toMap)
+ resourceConsumptionSource.addGauge(
+ "hdfsFileCount",
+ _ => computeUserResourceConsumption(userIdentifier).hdfsFileCount,
+ userIdentifier.toMap)
+ resourceConsumptionSource.addGauge(
+ "hdfsBytesWritten",
+ _ => computeUserResourceConsumption(userIdentifier).hdfsBytesWritten,
+ userIdentifier.toMap)
Review Comment:
>
Agree. We can write as follows
```
val userResourceConsumption = computeUserResourceConsumption(userIdentifier)
resourceConsumptionSource.addGauge(
"diskFIleCount",
_ => userResourceConsumption.diskFileCount,
userIdentifier.toMap)
resourceConsumptionSource.addGauge(
"diskBytesWritten",
_ => userResourceConsumption.diskBytesWritten,
userIdentifier.toMap)
resourceConsumptionSource.addGauge(
"hdfsFileCount",
_ => userResourceConsumption.hdfsFileCount,
userIdentifier.toMap)
resourceConsumptionSource.addGauge(
"hdfsBytesWritten",
_ => userResourceConsumption.hdfsBytesWritten,
userIdentifier.toMap)
val quota = quotaManager.getQuota(userIdentifier)
val (isAvailable, reason) =
quota.checkQuotaSpaceAvailable(userIdentifier, userResourceConsumption)
context.reply(CheckQuotaResponse(isAvailable, reason))
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@celeborn.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org