You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2023/03/16 21:27:46 UTC
[spark] branch master updated: [SPARK-42817][CORE] Logging the shuffle service name once in ApplicationMaster
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new f025d5eb1c2 [SPARK-42817][CORE] Logging the shuffle service name once in ApplicationMaster
f025d5eb1c2 is described below
commit f025d5eb1c2c9a6f7933679aa80752e806df9d2a
Author: Chandni Singh <si...@gmail.com>
AuthorDate: Thu Mar 16 14:27:31 2023 -0700
[SPARK-42817][CORE] Logging the shuffle service name once in ApplicationMaster
### What changes were proposed in this pull request?
Removed the logging of shuffle service name multiple times in the driver log. It gets logged everytime a new executor is allocated.
### Why are the changes needed?
This is needed because currently the driver logs gets polluted by these logs:
```
22/08/03 20:42:07 INFO ExecutorRunnable: Initializing service data for shuffle service using name 'spark_shuffle_311'
22/08/03 20:42:07 INFO ExecutorRunnable: Initializing service data for shuffle service using name 'spark_shuffle_311'
22/08/03 20:42:07 INFO ExecutorRunnable: Initializing service data for shuffle service using name 'spark_shuffle_311'
22/08/03 20:42:07 INFO ExecutorRunnable: Initializing service data for shuffle service using name 'spark_shuffle_311'
22/08/03 20:42:07 INFO ExecutorRunnable: Initializing service data for shuffle service using name 'spark_shuffle_311'
22/08/03 20:42:07 INFO ExecutorRunnable: Initializing service data for shuffle service using name 'spark_shuffle_311'
22/08/03 20:42:07 INFO ExecutorRunnable: Initializing service data for shuffle service using name 'spark_shuffle_311'
22/08/03 20:42:07 INFO ExecutorRunnable: Initializing service data for shuffle service using name 'spark_shuffle_311'
22/08/03 20:42:07 INFO ExecutorRunnable: Initializing service data for shuffle service using name 'spark_shuffle_311'
22/08/03 20:42:07 INFO ExecutorRunnable: Initializing service data for shuffle service using name 'spark_shuffle_311'
22/08/03 20:42:07 INFO ExecutorRunnable: Initializing service data for shuffle service using name 'spark_shuffle_311'
22/08/03 20:42:07 INFO ExecutorRunnable: Initializing service data for shuffle service using name 'spark_shuffle_311'
22/08/03 20:42:07 INFO ExecutorRunnable: Initializing service data for shuffle service using name 'spark_shuffle_311'
```
### Does this PR introduce _any_ user-facing change?
Yes, the shuffle service name will be just logged once in the driver.
### How was this patch tested?
Tested manually since it just changes the logging.
With this see this logged in the driver logs:
`23/03/15 16:50:54 INFO ApplicationMaster: Initializing service data for shuffle service using name 'spark_shuffle_311'`
Closes #40448 from otterc/SPARK-42817.
Authored-by: Chandni Singh <si...@gmail.com>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala | 5 ++++-
.../main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala | 1 -
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index 252c84a1cd4..8bf31a9286e 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -498,7 +498,10 @@ private[spark] class ApplicationMaster(
// that when the driver sends an initial executor request (e.g. after an AM restart),
// the allocator is ready to service requests.
rpcEnv.setupEndpoint("YarnAM", new AMEndpoint(rpcEnv, driverRef))
-
+ if (_sparkConf.get(SHUFFLE_SERVICE_ENABLED)) {
+ logInfo("Initializing service data for shuffle service using name '" +
+ s"${_sparkConf.get(SHUFFLE_SERVICE_NAME)}'")
+ }
allocator.allocateResources()
val ms = MetricsSystem.createMetricsSystem(MetricsSystemInstances.APPLICATION_MASTER, sparkConf)
val prefix = _sparkConf.get(YARN_METRICS_NAMESPACE).getOrElse(appId)
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
index 0148b6f3c95..1f3121ed224 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
@@ -115,7 +115,6 @@ private[yarn] class ExecutorRunnable(
ByteBuffer.allocate(0)
}
val serviceName = sparkConf.get(SHUFFLE_SERVICE_NAME)
- logInfo(s"Initializing service data for shuffle service using name '$serviceName'")
ctx.setServiceData(Collections.singletonMap(serviceName, secretBytes))
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org