You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by da...@apache.org on 2012/11/06 20:29:06 UTC
svn commit: r1406279 - in
/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/
Author: daryn
Date: Tue Nov 6 19:29:06 2012
New Revision: 1406279
URL: http://svn.apache.org/viewvc?rev=1406279&view=rev
Log:
svn merge -c 1406278 FIXES: HDFS-4075. Reduce recommissioning overhead (Kihwal Lee via daryn)
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1406279&r1=1406278&r2=1406279&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Tue Nov 6 19:29:06 2012
@@ -1662,6 +1662,8 @@ Release 0.23.5 - UNRELEASED
OPTIMIZATIONS
+ HDFS-4075. Reduce recommissioning overhead (Kihwal Lee via daryn)
+
BUG FIXES
HDFS-3829. TestHftpURLTimeouts fails intermittently with JDK7 (Trevor
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java?rev=1406279&r1=1406278&r2=1406279&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java Tue Nov 6 19:29:06 2012
@@ -2725,6 +2725,7 @@ assert storedBlock.findDatanode(dn) < 0
void processOverReplicatedBlocksOnReCommission(
final DatanodeDescriptor srcNode) {
final Iterator<? extends Block> it = srcNode.getBlockIterator();
+ int numOverReplicated = 0;
while(it.hasNext()) {
final Block block = it.next();
BlockCollection bc = blocksMap.getBlockCollection(block);
@@ -2734,8 +2735,11 @@ assert storedBlock.findDatanode(dn) < 0
if (numCurrentReplica > expectedReplication) {
// over-replicated block
processOverReplicatedBlock(block, expectedReplication, null, null);
+ numOverReplicated++;
}
}
+ LOG.info("Invalidated " + numOverReplicated + " over-replicated blocks on " +
+ srcNode + " during recommissioning");
}
/**
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java?rev=1406279&r1=1406278&r2=1406279&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java Tue Nov 6 19:29:06 2012
@@ -591,7 +591,11 @@ public class DatanodeManager {
if (node.isDecommissionInProgress() || node.isDecommissioned()) {
LOG.info("Stop Decommissioning " + node);
heartbeatManager.stopDecommission(node);
- blockManager.processOverReplicatedBlocksOnReCommission(node);
+ // Over-replicated blocks will be detected and processed when
+ // the dead node comes back and send in its full block report.
+ if (node.isAlive) {
+ blockManager.processOverReplicatedBlocksOnReCommission(node);
+ }
}
}