You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uniffle.apache.org by zu...@apache.org on 2024/06/14 05:46:16 UTC
(incubator-uniffle) branch master updated: [MINOR] refactor(spark): Remoev unnecessary logs (#1789)
This is an automated email from the ASF dual-hosted git repository.
zuston pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
The following commit(s) were added to refs/heads/master by this push:
new be46a03b7 [MINOR] refactor(spark): Remoev unnecessary logs (#1789)
be46a03b7 is described below
commit be46a03b7c59e84a5ef86ccedde9712f237e3e12
Author: Zhen Wang <64...@qq.com>
AuthorDate: Fri Jun 14 13:46:11 2024 +0800
[MINOR] refactor(spark): Remoev unnecessary logs (#1789)
### What changes were proposed in this pull request?
reduce minor logs
### Why are the changes needed?
a lot of minor rss logs are printed in spark executor.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
minor fix
---
.../java/org/apache/spark/shuffle/writer/BufferManagerOptions.java | 6 +++---
.../src/main/java/org/apache/spark/shuffle/RssShuffleManager.java | 1 -
.../main/java/org/apache/spark/shuffle/writer/RssShuffleWriter.java | 6 +++++-
3 files changed, 8 insertions(+), 5 deletions(-)
diff --git a/client-spark/common/src/main/java/org/apache/spark/shuffle/writer/BufferManagerOptions.java b/client-spark/common/src/main/java/org/apache/spark/shuffle/writer/BufferManagerOptions.java
index 7a2bcd74e..18938c610 100644
--- a/client-spark/common/src/main/java/org/apache/spark/shuffle/writer/BufferManagerOptions.java
+++ b/client-spark/common/src/main/java/org/apache/spark/shuffle/writer/BufferManagerOptions.java
@@ -64,9 +64,9 @@ public class BufferManagerOptions {
RssSparkConfig.RSS_WRITER_PRE_ALLOCATED_BUFFER_SIZE.defaultValue().get());
requireMemoryInterval = sparkConf.get(RssSparkConfig.RSS_WRITER_REQUIRE_MEMORY_INTERVAL);
requireMemoryRetryMax = sparkConf.get(RssSparkConfig.RSS_WRITER_REQUIRE_MEMORY_RETRY_MAX);
- LOG.info(RssSparkConfig.RSS_WRITER_BUFFER_SIZE.key() + "=" + bufferSize);
- LOG.info(RssSparkConfig.RSS_WRITER_BUFFER_SPILL_SIZE.key() + "=" + bufferSpillThreshold);
- LOG.info(
+ LOG.debug(RssSparkConfig.RSS_WRITER_BUFFER_SIZE.key() + "=" + bufferSize);
+ LOG.debug(RssSparkConfig.RSS_WRITER_BUFFER_SPILL_SIZE.key() + "=" + bufferSpillThreshold);
+ LOG.debug(
RssSparkConfig.RSS_WRITER_PRE_ALLOCATED_BUFFER_SIZE.key() + "=" + preAllocatedBufferSize);
checkBufferSize();
}
diff --git a/client-spark/spark3/src/main/java/org/apache/spark/shuffle/RssShuffleManager.java b/client-spark/spark3/src/main/java/org/apache/spark/shuffle/RssShuffleManager.java
index b5e89d1da..6354e17a2 100644
--- a/client-spark/spark3/src/main/java/org/apache/spark/shuffle/RssShuffleManager.java
+++ b/client-spark/spark3/src/main/java/org/apache/spark/shuffle/RssShuffleManager.java
@@ -505,7 +505,6 @@ public class RssShuffleManager extends RssShuffleManagerBase {
shuffleId, rssHandle.getPartitionToServers(), rssHandle.getRemoteStorage());
}
String taskId = "" + context.taskAttemptId() + "_" + context.attemptNumber();
- LOG.info("RssHandle appId {} shuffleId {} ", rssHandle.getAppId(), rssHandle.getShuffleId());
return new RssShuffleWriter<>(
rssHandle.getAppId(),
shuffleId,
diff --git a/client-spark/spark3/src/main/java/org/apache/spark/shuffle/writer/RssShuffleWriter.java b/client-spark/spark3/src/main/java/org/apache/spark/shuffle/writer/RssShuffleWriter.java
index 98c842b38..1f6f0b81b 100644
--- a/client-spark/spark3/src/main/java/org/apache/spark/shuffle/writer/RssShuffleWriter.java
+++ b/client-spark/spark3/src/main/java/org/apache/spark/shuffle/writer/RssShuffleWriter.java
@@ -182,7 +182,11 @@ public class RssShuffleWriter<K, V, C> extends ShuffleWriter<K, V> {
Function<String, Boolean> taskFailureCallback,
ShuffleHandleInfo shuffleHandleInfo,
TaskContext context) {
- LOG.info("RssShuffle start write taskAttemptId data" + taskAttemptId);
+ LOG.info(
+ "RssShuffle start write taskAttemptId[{}] data with RssHandle[appId {}, shuffleId {}].",
+ taskAttemptId,
+ rssHandle.getAppId(),
+ rssHandle.getShuffleId());
this.shuffleManager = shuffleManager;
this.appId = appId;
this.shuffleId = shuffleId;