You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sz...@apache.org on 2013/01/07 04:12:15 UTC
svn commit: r1429654 - in
/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/ src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/
src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/
Author: szetszwo
Date: Mon Jan 7 03:12:15 2013
New Revision: 1429654
URL: http://svn.apache.org/viewvc?rev=1429654&view=rev
Log:
svn merge -c 1429653 from trunk for HDFS-4351. In BlockPlacementPolicyDefault.chooseTarget(..), numOfReplicas needs to be updated when avoiding stale nodes.
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1429653
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1429654&r1=1429653&r2=1429654&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Mon Jan 7 03:12:15 2013
@@ -459,6 +459,9 @@ Release 2.0.3-alpha - Unreleased
HDFS-4017. Unclosed FileInputStream in GetJournalEditServlet
(Chao Shi via todd)
+ HDFS-4351. In BlockPlacementPolicyDefault.chooseTarget(..), numOfReplicas
+ needs to be updated when avoiding stale nodes. (Andrew Wang via szetszwo)
+
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1429653
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java?rev=1429654&r1=1429653&r2=1429654&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java Mon Jan 7 03:12:15 2013
@@ -234,13 +234,18 @@ public class BlockPlacementPolicyDefault
+ totalReplicasExpected + "\n"
+ e.getMessage());
if (avoidStaleNodes) {
- // ecxludedNodes now has - initial excludedNodes, any nodes that were
- // chosen and nodes that were tried but were not chosen because they
- // were stale, decommissioned or for any other reason a node is not
- // chosen for write. Retry again now not avoiding stale node
+ // Retry chooseTarget again, this time not avoiding stale nodes.
+
+ // excludedNodes contains the initial excludedNodes and nodes that were
+ // not chosen because they were stale, decommissioned, etc.
+ // We need to additionally exclude the nodes that were added to the
+ // result list in the successful calls to choose*() above.
for (Node node : results) {
oldExcludedNodes.put(node, node);
}
+ // Set numOfReplicas, since it can get out of sync with the result list
+ // if the NotEnoughReplicasException was thrown in chooseRandom().
+ numOfReplicas = totalReplicasExpected - results.size();
return chooseTarget(numOfReplicas, writer, oldExcludedNodes, blocksize,
maxNodesPerRack, results, false);
}
@@ -505,7 +510,7 @@ public class BlockPlacementPolicyDefault
if (LOG.isDebugEnabled()) {
threadLocalBuilder.get().append(node.toString()).append(": ")
.append("Node ").append(NodeBase.getPath(node))
- .append(" is not chosen because the node is staled ");
+ .append(" is not chosen because the node is stale ");
}
return false;
}
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java?rev=1429654&r1=1429653&r2=1429654&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java Mon Jan 7 03:12:15 2013
@@ -381,6 +381,24 @@ public class TestReplicationPolicy {
assertTrue(cluster.isOnSameRack(targets[1], targets[2]));
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
}
+
+ /**
+ * In this testcase, it tries to choose more targets than available nodes and
+ * check the result, with stale node avoidance on the write path enabled.
+ * @throws Exception
+ */
+ @Test
+ public void testChooseTargetWithMoreThanAvailableNodesWithStaleness()
+ throws Exception {
+ try {
+ namenode.getNamesystem().getBlockManager().getDatanodeManager()
+ .setAvoidStaleDataNodesForWrite(true);
+ testChooseTargetWithMoreThanAvailableNodes();
+ } finally {
+ namenode.getNamesystem().getBlockManager().getDatanodeManager()
+ .setAvoidStaleDataNodesForWrite(false);
+ }
+ }
/**
* In this testcase, it tries to choose more targets than available nodes and
@@ -388,7 +406,7 @@ public class TestReplicationPolicy {
* @throws Exception
*/
@Test
- public void testChooseTargetWithMoreThanAvaiableNodes() throws Exception {
+ public void testChooseTargetWithMoreThanAvailableNodes() throws Exception {
// make data node 0 & 1 to be not qualified to choose: not enough disk space
for(int i=0; i<2; i++) {
dataNodes[i].updateHeartbeat(