You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2017/12/15 18:39:28 UTC
[42/50] [abbrv] hadoop git commit: HDFS-12894. [READ] Skip setting
block count of ProvidedDatanodeStorageInfo on DN registration update
HDFS-12894. [READ] Skip setting block count of ProvidedDatanodeStorageInfo on DN registration update
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/27ce8136
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/27ce8136
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/27ce8136
Branch: refs/heads/HDFS-9806
Commit: 27ce8136032c1ece21f2f36cc21cd110ae703bdf
Parents: 31b56cd
Author: Virajith Jalaparti <vi...@apache.org>
Authored: Tue Dec 5 17:55:32 2017 -0800
Committer: Virajith Jalaparti <vi...@apache.org>
Committed: Fri Dec 15 10:25:34 2017 -0800
----------------------------------------------------------------------
.../server/blockmanagement/BlockManager.java | 5 +++++
.../blockmanagement/DatanodeDescriptor.java | 4 +++-
.../TestNameNodeProvidedImplementation.java | 20 +++++++++++++++++++-
3 files changed, 27 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27ce8136/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index f92c4e8..916cbaa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -4943,4 +4943,9 @@ public class BlockManager implements BlockStatsMXBean {
public void setBlockRecoveryTimeout(long blockRecoveryTimeout) {
pendingRecoveryBlocks.setRecoveryTimeoutInterval(blockRecoveryTimeout);
}
+
+ @VisibleForTesting
+ public ProvidedStorageMap getProvidedStorageMap() {
+ return providedStorageMap;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27ce8136/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
index 83c608f..fc58708 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
@@ -919,7 +919,9 @@ public class DatanodeDescriptor extends DatanodeInfo {
// must re-process IBR after re-registration
for(DatanodeStorageInfo storage : getStorageInfos()) {
- storage.setBlockReportCount(0);
+ if (storage.getStorageType() != StorageType.PROVIDED) {
+ storage.setBlockReportCount(0);
+ }
}
heartbeatedSinceRegistration = false;
forceRegistration = false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27ce8136/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeProvidedImplementation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeProvidedImplementation.java b/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeProvidedImplementation.java
index deaf9d5..d057247 100644
--- a/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeProvidedImplementation.java
+++ b/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeProvidedImplementation.java
@@ -559,7 +559,9 @@ public class TestNameNodeProvidedImplementation {
DataNode providedDatanode2 = cluster.getDataNodes().get(1);
DFSClient client = new DFSClient(new InetSocketAddress("localhost",
- cluster.getNameNodePort()), cluster.getConfiguration(0));
+ cluster.getNameNodePort()), cluster.getConfiguration(0));
+
+ DatanodeStorageInfo providedDNInfo = getProvidedDatanodeStorageInfo();
if (numFiles >= 1) {
String filename = "/" + filePrefix + (numFiles - 1) + fileSuffix;
@@ -596,10 +598,15 @@ public class TestNameNodeProvidedImplementation {
providedDatanode2.getDatanodeId().getXferAddr());
getAndCheckBlockLocations(client, filename, baseFileLen, 1, 0);
+ // BR count for the provided ProvidedDatanodeStorageInfo should reset to
+ // 0, when all DNs with PROVIDED storage fail.
+ assertEquals(0, providedDNInfo.getBlockReportCount());
//restart the provided datanode
cluster.restartDataNode(providedDNProperties1, true);
cluster.waitActive();
+ assertEquals(1, providedDNInfo.getBlockReportCount());
+
//should find the block on the 1st provided datanode now
dnInfos = getAndCheckBlockLocations(client, filename, baseFileLen, 1, 1);
//not comparing UUIDs as the datanode can now have a different one.
@@ -621,6 +628,8 @@ public class TestNameNodeProvidedImplementation {
false);
DataNode providedDatanode = cluster.getDataNodes().get(0);
+ DatanodeStorageInfo providedDNInfo = getProvidedDatanodeStorageInfo();
+ int initialBRCount = providedDNInfo.getBlockReportCount();
for (int i= 0; i < numFiles; i++) {
// expect to have 2 locations as we have 2 provided Datanodes.
verifyFileLocation(i, 2);
@@ -631,10 +640,19 @@ public class TestNameNodeProvidedImplementation {
cluster.waitActive();
cluster.triggerHeartbeats();
Thread.sleep(1000);
+ // the report count should just continue to increase.
+ assertEquals(initialBRCount + i + 1,
+ providedDNInfo.getBlockReportCount());
verifyFileLocation(i, 2);
}
}
+ private DatanodeStorageInfo getProvidedDatanodeStorageInfo() {
+ ProvidedStorageMap providedStorageMap =
+ cluster.getNamesystem().getBlockManager().getProvidedStorageMap();
+ return providedStorageMap.getProvidedStorageInfo();
+ }
+
@Test(timeout=30000)
public void testNamenodeRestart() throws Exception {
createImage(new FSTreeWalk(NAMEPATH, conf), NNDIRPATH,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org