You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2017/07/27 19:41:49 UTC
[4/4] hadoop git commit: HDFS-11896. Non-dfsUsed will be doubled on
dead node re-registration. Contributed by Brahma Reddy Battula.
HDFS-11896. Non-dfsUsed will be doubled on dead node re-registration. Contributed by Brahma Reddy Battula.
(cherry picked from commit c4a85c694fae3f814ab4e7f3c172da1df0e0e353)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f90b9d2b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f90b9d2b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f90b9d2b
Branch: refs/heads/branch-2.7
Commit: f90b9d2b25822868797d136de8bc268c5ca7ad2c
Parents: b516235
Author: Brahma Reddy Battula <br...@apache.org>
Authored: Thu Jul 27 12:16:26 2017 -0700
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Thu Jul 27 12:40:09 2017 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../blockmanagement/DatanodeDescriptor.java | 24 ++++++---
.../blockmanagement/HeartbeatManager.java | 3 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 4 +-
.../hdfs/server/namenode/TestDeadDatanode.java | 51 ++++++++++++++++++++
5 files changed, 72 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f90b9d2b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 5713513..51a11c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -396,6 +396,9 @@ Release 2.7.4 - UNRELEASED
HDFS-11742. Improve balancer usability after HDFS-8818. (kihwal)
+ HDFS-11896. Non-dfsUsed will be doubled on dead node re-registration.
+ (Brahma Reddy Battula via shv)
+
Release 2.7.3 - 2016-08-25
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f90b9d2b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
index 02b8eb9..66a0990 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
@@ -311,11 +311,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
}
public void resetBlocks() {
- setCapacity(0);
- setRemaining(0);
- setBlockPoolUsed(0);
- setDfsUsed(0);
- setXceiverCount(0);
+ updateStorageStats(this.getStorageReports(), 0L, 0L, 0, 0, null);
this.invalidateBlocks.clear();
this.volumeFailures = 0;
// pendingCached, cached, and pendingUncached are protected by the
@@ -346,6 +342,11 @@ public class DatanodeDescriptor extends DatanodeInfo {
return blocks;
}
+ @VisibleForTesting
+ public boolean isHeartbeatedSinceRegistration() {
+ return heartbeatedSinceRegistration;
+ }
+
/**
* Updates stats from datanode heartbeat.
*/
@@ -363,6 +364,16 @@ public class DatanodeDescriptor extends DatanodeInfo {
public void updateHeartbeatState(StorageReport[] reports, long cacheCapacity,
long cacheUsed, int xceiverCount, int volFailures,
VolumeFailureSummary volumeFailureSummary) {
+ updateStorageStats(reports, cacheCapacity, cacheUsed, xceiverCount,
+ volFailures, volumeFailureSummary);
+ setLastUpdate(Time.now());
+ setLastUpdateMonotonic(Time.monotonicNow());
+ rollBlocksScheduled(getLastUpdateMonotonic());
+ }
+
+ private void updateStorageStats(StorageReport[] reports, long cacheCapacity,
+ long cacheUsed, int xceiverCount, int volFailures,
+ VolumeFailureSummary volumeFailureSummary) {
long totalCapacity = 0;
long totalRemaining = 0;
long totalBlockPoolUsed = 0;
@@ -409,8 +420,6 @@ public class DatanodeDescriptor extends DatanodeInfo {
setCacheCapacity(cacheCapacity);
setCacheUsed(cacheUsed);
setXceiverCount(xceiverCount);
- setLastUpdate(Time.now());
- setLastUpdateMonotonic(Time.monotonicNow());
this.volumeFailures = volFailures;
this.volumeFailureSummary = volumeFailureSummary;
for (StorageReport report : reports) {
@@ -426,7 +435,6 @@ public class DatanodeDescriptor extends DatanodeInfo {
totalDfsUsed += report.getDfsUsed();
totalNonDfsUsed += report.getNonDfsUsed();
}
- rollBlocksScheduled(getLastUpdateMonotonic());
// Update total metrics for the node.
setCapacity(totalCapacity);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f90b9d2b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java
index 8952fdb..dfc3c60 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java
@@ -188,10 +188,9 @@ class HeartbeatManager implements DatanodeStatistics {
synchronized void register(final DatanodeDescriptor d) {
if (!d.isAlive) {
- addDatanode(d);
-
//update its timestamp
d.updateHeartbeatState(StorageReport.EMPTY_ARRAY, 0L, 0L, 0, 0, null);
+ addDatanode(d);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f90b9d2b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 19c488d..f23580c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -1066,10 +1066,8 @@ public class DataNode extends ReconfigurableBase
return bpos;
}
-
-
// used only for testing
- void setHeartbeatsDisabledForTests(
+ public void setHeartbeatsDisabledForTests(
boolean heartbeatsDisabledForTests) {
this.heartbeatsDisabledForTests = heartbeatsDisabledForTests;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f90b9d2b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
index f26f424..00ea58f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
@@ -19,10 +19,12 @@ package org.apache.hadoop.hdfs.server.namenode;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
+import com.google.common.base.Supplier;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -33,6 +35,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.protocol.BlockReportContext;
@@ -45,6 +48,7 @@ import org.apache.hadoop.hdfs.server.protocol.RegisterCommand;
import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport;
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.Test;
@@ -128,4 +132,51 @@ public class TestDeadDatanode {
assertEquals(cmd[0].getAction(), RegisterCommand.REGISTER
.getAction());
}
+
+ @Test
+ public void testNonDFSUsedONDeadNodeReReg() throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_KEY,
+ 6 * 1000);
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
+ long initialCapacity = cluster.getNamesystem(0).getCapacityTotal();
+ assertTrue(initialCapacity > 0);
+ DataNode dn1 = cluster.getDataNodes().get(0);
+ DataNode dn2 = cluster.getDataNodes().get(1);
+ final DatanodeDescriptor dn2Desc = cluster.getNamesystem(0)
+ .getBlockManager().getDatanodeManager()
+ .getDatanode(dn2.getDatanodeId());
+ dn1.setHeartbeatsDisabledForTests(true);
+ cluster.setDataNodeDead(dn1.getDatanodeId());
+ assertEquals("Capacity shouldn't include DeadNode", dn2Desc.getCapacity(),
+ cluster.getNamesystem(0).getCapacityTotal());
+ assertEquals("NonDFS-used shouldn't include DeadNode",
+ dn2Desc.getNonDfsUsed(),
+ cluster.getNamesystem(0).getNonDfsUsedSpace());
+ // Wait for re-registration and heartbeat
+ dn1.setHeartbeatsDisabledForTests(false);
+ final DatanodeDescriptor dn1Desc = cluster.getNamesystem(0)
+ .getBlockManager().getDatanodeManager()
+ .getDatanode(dn1.getDatanodeId());
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+
+ @Override
+ public Boolean get() {
+ return dn1Desc.isAlive && dn1Desc.isHeartbeatedSinceRegistration();
+ }
+ }, 100, 5000);
+ assertEquals("Capacity should be 0 after all DNs dead", initialCapacity,
+ cluster.getNamesystem(0).getCapacityTotal());
+ long nonDFSAfterReg = cluster.getNamesystem(0).getNonDfsUsedSpace();
+ assertEquals("NonDFS should include actual DN NonDFSUsed",
+ dn1Desc.getNonDfsUsed() + dn2Desc.getNonDfsUsed(), nonDFSAfterReg);
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org