You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2016/10/27 22:59:29 UTC
hadoop git commit: HDFS-9500. Fix software version counts for
DataNodes during rolling upgrade. Contributed by Erik Krogen.
Repository: hadoop
Updated Branches:
refs/heads/trunk 022bf783a -> f3ac1f41b
HDFS-9500. Fix software version counts for DataNodes during rolling upgrade. Contributed by Erik Krogen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f3ac1f41
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f3ac1f41
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f3ac1f41
Branch: refs/heads/trunk
Commit: f3ac1f41b8fa82a0ac87a207d7afa2061d90a9bd
Parents: 022bf78
Author: Erik Krogen <ek...@linkedin.com>
Authored: Thu Oct 27 15:14:21 2016 -0700
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Thu Oct 27 15:58:25 2016 -0700
----------------------------------------------------------------------
.../server/blockmanagement/DatanodeManager.java | 16 ++++++---
.../blockmanagement/TestDatanodeManager.java | 37 ++++++++++++++++++++
2 files changed, 49 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3ac1f41/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index 1a47835..47f15c4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
@@ -759,17 +759,25 @@ public class DatanodeManager {
}
}
+ /**
+ * Will return true for all Datanodes which have a non-null software
+ * version and are considered alive (by {@link DatanodeDescriptor#isAlive()}),
+ * indicating the node has not yet been removed. Use {@code isAlive}
+ * rather than {@link DatanodeManager#isDatanodeDead(DatanodeDescriptor)}
+ * to ensure that the version is decremented even if the datanode
+ * hasn't issued a heartbeat recently.
+ *
+ * @param node The datanode in question
+ * @return True iff its version count should be decremented
+ */
private boolean shouldCountVersion(DatanodeDescriptor node) {
- return node.getSoftwareVersion() != null && node.isAlive() &&
- !isDatanodeDead(node);
+ return node.getSoftwareVersion() != null && node.isAlive();
}
private void countSoftwareVersions() {
synchronized(this) {
datanodesSoftwareVersions.clear();
for(DatanodeDescriptor dn: datanodeMap.values()) {
- // Check isAlive too because right after removeDatanode(),
- // isDatanodeDead() is still true
if (shouldCountVersion(dn)) {
Integer num = datanodesSoftwareVersions.get(dn.getSoftwareVersion());
num = num == null ? 1 : num+1;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3ac1f41/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
index be8a0f0..30e2aaf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
@@ -86,6 +86,43 @@ public class TestDatanodeManager {
}
/**
+ * This test checks that if a node is re-registered with a new software
+ * version after the heartbeat expiry interval but before the HeartbeatManager
+ * has a chance to detect this and remove it, the node's version will still
+ * be correctly decremented.
+ */
+ @Test
+ public void testNumVersionsCorrectAfterReregister()
+ throws IOException, InterruptedException {
+ //Create the DatanodeManager which will be tested
+ FSNamesystem fsn = Mockito.mock(FSNamesystem.class);
+ Mockito.when(fsn.hasWriteLock()).thenReturn(true);
+ Configuration conf = new Configuration();
+ conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 0);
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 10);
+ DatanodeManager dm = mockDatanodeManager(fsn, conf);
+
+ String storageID = "someStorageID1";
+ String ip = "someIP" + storageID;
+
+ // Register then reregister the same node but with a different version
+ for (int i = 0; i <= 1; i++) {
+ dm.registerDatanode(new DatanodeRegistration(
+ new DatanodeID(ip, "", storageID, 9000, 0, 0, 0),
+ null, null, "version" + i));
+ if (i == 0) {
+ Thread.sleep(25);
+ }
+ }
+
+ //Verify DatanodeManager has the correct count
+ Map<String, Integer> mapToCheck = dm.getDatanodesSoftwareVersions();
+ assertNull("should be no more version0 nodes", mapToCheck.get("version0"));
+ assertEquals("should be one version1 node",
+ mapToCheck.get("version1").intValue(), 1);
+ }
+
+ /**
* This test sends a random sequence of node registrations and node removals
* to the DatanodeManager (of nodes with different IDs and versions), and
* checks that the DatanodeManager keeps a correct count of different software
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org