You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by bi...@apache.org on 2014/09/29 02:35:53 UTC
[47/50] [abbrv] git commit: TEZ-1609. Add hostname to logIdentifiers
of fetchers for easy debugging (Rajesh Balamohan)
TEZ-1609. Add hostname to logIdentifiers of fetchers for easy debugging (Rajesh Balamohan)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/fa2928f6
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/fa2928f6
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/fa2928f6
Branch: refs/heads/branch-0.5
Commit: fa2928f6a9fae32fb7edc5dad6374ca625c3b417
Parents: 160a5d8
Author: Rajesh Balamohan <rb...@apache.org>
Authored: Sat Sep 27 04:27:31 2014 +0530
Committer: Rajesh Balamohan <rb...@apache.org>
Committed: Sat Sep 27 04:27:31 2014 +0530
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../tez/runtime/library/common/shuffle/impl/Fetcher.java | 5 ++++-
.../tez/runtime/library/shuffle/common/HttpConnection.java | 8 +++++---
.../runtime/library/shuffle/common/impl/ShuffleManager.java | 4 +++-
4 files changed, 13 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/fa2928f6/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index e96d7c6..741c99a 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -70,6 +70,7 @@ ALL CHANGES
TEZ-1433. Invalid credentials can be used when a DAG is submitted to a
session which has timed out
TEZ-1624. Flaky tests in TestContainerReuse due to race condition in DelayedContainerManager thread
+ TEZ-1609. Add hostname to logIdentifiers of fetchers for easy debugging
Release 0.5.0: 2014-09-03
http://git-wip-us.apache.org/repos/asf/tez/blob/fa2928f6/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/Fetcher.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/Fetcher.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/Fetcher.java
index 9b009ea..c29401b 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/Fetcher.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/Fetcher.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.net.NetUtils;
import org.apache.tez.common.TezUtilsInternal;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.tez.common.TezRuntimeFrameworkConfigs;
@@ -92,6 +93,8 @@ class Fetcher extends Thread {
HttpConnection httpConnection;
HttpConnectionParams httpConnectionParams;
+
+ final static String localhostName = NetUtils.getHostname();
public Fetcher(HttpConnectionParams httpConnectionParams,
ShuffleScheduler scheduler, MergeManager merger,
@@ -132,7 +135,7 @@ class Fetcher extends Thread {
this.localDiskFetchEnabled = localDiskFetchEnabled;
this.logIdentifier = "fetcher [" + TezUtilsInternal
- .cleanVertexName(inputContext.getSourceVertexName()) + "] #" + id;
+ .cleanVertexName(inputContext.getSourceVertexName()) + "] #" + id + " " + localhostName;
setName(logIdentifier);
setDaemon(true);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/fa2928f6/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/HttpConnection.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/HttpConnection.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/HttpConnection.java
index d664f88..1311a4f 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/HttpConnection.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/HttpConnection.java
@@ -210,11 +210,13 @@ public class HttpConnection {
if (replyHash == null) {
throw new IOException("security validation of TT Map output failed");
}
- LOG.debug("url=" + msgToEncode + ";encHash=" + encHash + ";replyHash="
- + replyHash);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("url=" + msgToEncode + ";encHash=" + encHash + ";replyHash="
+ + replyHash);
+ }
// verify that replyHash is HMac of encHash
SecureShuffleUtils.verifyReply(replyHash, encHash, jobTokenSecret);
- LOG.info("for url=" + msgToEncode +
+ LOG.info("for url=" + url +
" sent hash and receievd reply " + stopWatch.elapsedTime(TimeUnit.MILLISECONDS) + " ms");
}
http://git-wip-us.apache.org/repos/asf/tez/blob/fa2928f6/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/impl/ShuffleManager.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/impl/ShuffleManager.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/impl/ShuffleManager.java
index 8aa6582..ddf98b6 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/impl/ShuffleManager.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/impl/ShuffleManager.java
@@ -49,6 +49,7 @@ import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.net.NetUtils;
import org.apache.tez.common.TezRuntimeFrameworkConfigs;
import org.apache.tez.common.TezUtilsInternal;
import org.apache.tez.common.counters.TaskCounter;
@@ -153,6 +154,7 @@ public class ShuffleManager implements FetcherCallback {
private final LocalDirAllocator localDirAllocator;
private final RawLocalFileSystem localFs;
private final Path[] localDisks;
+ private final static String localhostName = NetUtils.getHostname();
// TODO More counters - FetchErrors, speed?
@@ -199,7 +201,7 @@ public class ShuffleManager implements FetcherCallback {
ExecutorService fetcherRawExecutor = Executors.newFixedThreadPool(
numFetchers,
new ThreadFactoryBuilder().setDaemon(true)
- .setNameFormat("Fetcher [" + srcNameTrimmed + "] #%d").build());
+ .setNameFormat("Fetcher [" + srcNameTrimmed + "] #%d " + localhostName).build());
this.fetcherExecutor = MoreExecutors.listeningDecorator(fetcherRawExecutor);
ExecutorService schedulerRawExecutor = Executors.newFixedThreadPool(1, new ThreadFactoryBuilder()