You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ju...@apache.org on 2017/04/12 21:07:11 UTC
[1/2] hadoop git commit: HDFS-11648. Lazy construct the IIP pathname.
Contributed by Daryn Sharp.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.8.1 a35382711 -> 56814bbd5
HDFS-11648. Lazy construct the IIP pathname. Contributed by Daryn Sharp.
(cherry picked from commit 8ed230c805625549b1cecc830e909a7027bb4961)
(cherry picked from commit 9dfe0b35158cc0046a82cbfea2218a4ad61f329e)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d0a40f26
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d0a40f26
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d0a40f26
Branch: refs/heads/branch-2.8.1
Commit: d0a40f26dc4435757da0ac67f6f207ff49efc364
Parents: a353827
Author: Kihwal Lee <ki...@apache.org>
Authored: Wed Apr 12 13:30:41 2017 -0500
Committer: Junping Du <ju...@apache.org>
Committed: Wed Apr 12 14:04:39 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/server/namenode/INodesInPath.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0a40f26/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
index b37321d..1d5dbf6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
@@ -238,7 +238,7 @@ public class INodesInPath {
}
private final byte[][] path;
- private final String pathname;
+ private volatile String pathname;
/**
* Array with the specified number of INodes resolved for a given path.
@@ -268,7 +268,6 @@ public class INodesInPath {
Preconditions.checkArgument(inodes != null && path != null);
this.inodes = inodes;
this.path = path;
- this.pathname = DFSUtil.byteArray2PathString(path);
this.isRaw = isRaw;
this.isSnapshot = isSnapshot;
this.snapshotId = snapshotId;
@@ -329,6 +328,9 @@ public class INodesInPath {
/** @return the full path in string form */
public String getPath() {
+ if (pathname == null) {
+ pathname = DFSUtil.byteArray2PathString(path);
+ }
return pathname;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/2] hadoop git commit: HDFS-11558. BPServiceActor thread name is
too long. Contributed by Xiaobing Zhou
Posted by ju...@apache.org.
HDFS-11558. BPServiceActor thread name is too long. Contributed by Xiaobing Zhou
(cherry picked from commit 9eebbcf459ae6b769f548ebbed1a18142e009478)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/56814bbd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/56814bbd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/56814bbd
Branch: refs/heads/branch-2.8.1
Commit: 56814bbd5a1983faa62e754588ba998010742eb0
Parents: d0a40f2
Author: Mingliang Liu <li...@apache.org>
Authored: Wed Apr 12 11:47:34 2017 -0700
Committer: Junping Du <ju...@apache.org>
Committed: Wed Apr 12 14:04:54 2017 -0700
----------------------------------------------------------------------
.../hdfs/server/datanode/BPOfferService.java | 19 +++++++++++++---
.../hdfs/server/datanode/BPServiceActor.java | 24 ++++++++++++--------
.../hdfs/server/datanode/BlockPoolManager.java | 8 ++++---
.../server/datanode/TestBPOfferService.java | 2 +-
.../server/datanode/TestBlockPoolManager.java | 4 +++-
5 files changed, 39 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56814bbd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
index 1b0806f..2f64b15 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
@@ -68,7 +68,8 @@ class BPOfferService {
* handshake.
*/
volatile DatanodeRegistration bpRegistration;
-
+
+ private final String nameserviceId;
private final DataNode dn;
/**
@@ -118,12 +119,16 @@ class BPOfferService {
mWriteLock.unlock();
}
- BPOfferService(List<InetSocketAddress> nnAddrs,
- List<InetSocketAddress> lifelineNnAddrs, DataNode dn) {
+ BPOfferService(
+ final String nameserviceId,
+ List<InetSocketAddress> nnAddrs,
+ List<InetSocketAddress> lifelineNnAddrs,
+ DataNode dn) {
Preconditions.checkArgument(!nnAddrs.isEmpty(),
"Must pass at least one NN.");
Preconditions.checkArgument(nnAddrs.size() == lifelineNnAddrs.size(),
"Must pass same number of NN addresses and lifeline addresses.");
+ this.nameserviceId = nameserviceId;
this.dn = dn;
for (int i = 0; i < nnAddrs.size(); ++i) {
@@ -168,6 +173,14 @@ class BPOfferService {
return false;
}
+ /**
+ * Gets nameservice id to which this {@link BPOfferService} maps to.
+ * @return nameservice id, which can be null.
+ */
+ String getNameserviceId() {
+ return nameserviceId;
+ }
+
String getBlockPoolId() {
readLock();
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56814bbd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
index 06c25d8..da37fd7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
@@ -264,7 +264,10 @@ class BPServiceActor implements Runnable {
// This also initializes our block pool in the DN if we are
// the first NN connection for this BP.
bpos.verifyAndSetNamespaceInfo(this, nsInfo);
-
+
+ /* set thread name again to include NamespaceInfo when it's available. */
+ this.bpThread.setName(formatThreadName("heartbeating", nnAddr));
+
// Second phase of the handshake with the NN.
register(nsInfo);
}
@@ -483,14 +486,15 @@ class BPServiceActor implements Runnable {
lifelineSender.start();
}
}
-
- private String formatThreadName(String action, InetSocketAddress addr) {
- Collection<StorageLocation> dataDirs =
- DataNode.getStorageLocations(dn.getConf());
- return "DataNode: [" + dataDirs.toString() + "] " +
- action + " to " + addr;
+
+ private String formatThreadName(
+ final String action,
+ final InetSocketAddress addr) {
+ final String prefix = bpos.getBlockPoolId() != null ? bpos.getBlockPoolId()
+ : bpos.getNameserviceId();
+ return prefix + " " + action + " to " + addr;
}
-
+
//This must be called only by blockPoolManager.
void stop() {
shouldServiceRun = false;
@@ -944,8 +948,8 @@ class BPServiceActor implements Runnable {
}
public void start() {
- lifelineThread = new Thread(this, formatThreadName("lifeline",
- lifelineNnAddr));
+ lifelineThread = new Thread(this,
+ formatThreadName("lifeline", lifelineNnAddr));
lifelineThread.setDaemon(true);
lifelineThread.setUncaughtExceptionHandler(
new Thread.UncaughtExceptionHandler() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56814bbd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolManager.java
index e94bbb7..f6a11c2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolManager.java
@@ -211,7 +211,7 @@ class BlockPoolManager {
lifelineAddrs.add(nnIdToLifelineAddr != null ?
nnIdToLifelineAddr.get(nnId) : null);
}
- BPOfferService bpos = createBPOS(addrs, lifelineAddrs);
+ BPOfferService bpos = createBPOS(nsToAdd, addrs, lifelineAddrs);
bpByNameserviceId.put(nsToAdd, bpos);
offerServices.add(bpos);
}
@@ -261,8 +261,10 @@ class BlockPoolManager {
/**
* Extracted out for test purposes.
*/
- protected BPOfferService createBPOS(List<InetSocketAddress> nnAddrs,
+ protected BPOfferService createBPOS(
+ final String nameserviceId,
+ List<InetSocketAddress> nnAddrs,
List<InetSocketAddress> lifelineNnAddrs) {
- return new BPOfferService(nnAddrs, lifelineNnAddrs, dn);
+ return new BPOfferService(nameserviceId, nnAddrs, lifelineNnAddrs, dn);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56814bbd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
index 957987b..fc95899 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
@@ -408,7 +408,7 @@ public class TestBPOfferService {
Mockito.eq(new InetSocketAddress(port)));
}
- return new BPOfferService(Lists.newArrayList(nnMap.keySet()),
+ return new BPOfferService("test_ns", Lists.newArrayList(nnMap.keySet()),
Collections.<InetSocketAddress>nCopies(nnMap.size(), null), mockDn);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56814bbd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolManager.java
index 48006dc..951adbd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolManager.java
@@ -51,7 +51,9 @@ public class TestBlockPoolManager {
bpm = new BlockPoolManager(mockDN){
@Override
- protected BPOfferService createBPOS(List<InetSocketAddress> nnAddrs,
+ protected BPOfferService createBPOS(
+ final String nameserviceId,
+ List<InetSocketAddress> nnAddrs,
List<InetSocketAddress> lifelineNnAddrs) {
final int idx = mockIdx++;
doLog("create #" + idx);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org