You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by an...@apache.org on 2016/05/04 21:28:36 UTC
spark git commit: [SPARK-15121] Improve logging of external shuffle
handler
Repository: spark
Updated Branches:
refs/heads/master 6ae9fc00e -> 0c00391f7
[SPARK-15121] Improve logging of external shuffle handler
## What changes were proposed in this pull request?
Add more informative logging in the external shuffle service to aid in debugging who is connecting to the YARN Nodemanager when the external shuffle service runs under it.
## How was this patch tested?
Ran and saw logs coming out in log file.
Author: Thomas Graves <tg...@staydecay.corp.gq1.yahoo.com>
Closes #12900 from tgravescs/SPARK-15121.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0c00391f
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0c00391f
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0c00391f
Branch: refs/heads/master
Commit: 0c00391f77359efdbf9dbd26d4c8186be8839922
Parents: 6ae9fc0
Author: Thomas Graves <tg...@staydecay.corp.gq1.yahoo.com>
Authored: Wed May 4 14:28:26 2016 -0700
Committer: Andrew Or <an...@databricks.com>
Committed: Wed May 4 14:28:26 2016 -0700
----------------------------------------------------------------------
.../spark/network/shuffle/ExternalShuffleBlockHandler.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/0c00391f/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockHandler.java
----------------------------------------------------------------------
diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockHandler.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockHandler.java
index f8d03b3..fb1226c 100644
--- a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockHandler.java
+++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockHandler.java
@@ -35,6 +35,7 @@ import org.apache.spark.network.server.RpcHandler;
import org.apache.spark.network.server.StreamManager;
import org.apache.spark.network.shuffle.ExternalShuffleBlockResolver.AppExecId;
import org.apache.spark.network.shuffle.protocol.*;
+import org.apache.spark.network.util.NettyUtils;
import org.apache.spark.network.util.TransportConf;
@@ -86,7 +87,8 @@ public class ExternalShuffleBlockHandler extends RpcHandler {
blocks.add(blockManager.getBlockData(msg.appId, msg.execId, blockId));
}
long streamId = streamManager.registerStream(client.getClientId(), blocks.iterator());
- logger.trace("Registered streamId {} with {} buffers", streamId, msg.blockIds.length);
+ logger.trace("Registered streamId {} with {} buffers for client {} from host {}", streamId,
+ msg.blockIds.length, client.getClientId(), NettyUtils.getRemoteAddress(client.getChannel()));
callback.onSuccess(new StreamHandle(streamId, msg.blockIds.length).toByteBuffer());
} else if (msgObj instanceof RegisterExecutor) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org