You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cd...@apache.org on 2008/03/18 00:55:15 UTC
svn commit: r638144 - in /hadoop/core/trunk: CHANGES.txt
src/java/org/apache/hadoop/dfs/ReplicationTargetChooser.java
src/test/org/apache/hadoop/dfs/TestReplicationPolicy.java
Author: cdouglas
Date: Mon Mar 17 16:55:09 2008
New Revision: 638144
URL: http://svn.apache.org/viewvc?rev=638144&view=rev
Log:
HADOOP-2559. Change DFS block placement to allocate the first replica
locally, the second off-rack, and the third intra-rack from the
second. Contributed by Lohit Vijayarenu.
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/java/org/apache/hadoop/dfs/ReplicationTargetChooser.java
hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestReplicationPolicy.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=638144&r1=638143&r2=638144&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Mon Mar 17 16:55:09 2008
@@ -101,6 +101,10 @@
HADOOP-2804. Add support to publish CHANGES.txt as HTML when running
the Ant 'docs' target. (nigel)
+ HADOOP-2559. Change DFS block placement to allocate the first replica
+ locally, the second off-rack, and the third intra-rack from the
+ second. (lohit vijayarenu via cdouglas)
+
OPTIMIZATIONS
HADOOP-2790. Fixed inefficient method hasSpeculativeTask by removing
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/ReplicationTargetChooser.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/ReplicationTargetChooser.java?rev=638144&r1=638143&r2=638144&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/ReplicationTargetChooser.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/ReplicationTargetChooser.java Mon Mar 17 16:55:09 2008
@@ -163,8 +163,8 @@
chooseRemoteRack(1, writer, excludedNodes,
blocksize, maxNodesPerRack, results);
} else {
- chooseLocalRack(writer, excludedNodes,
- blocksize, maxNodesPerRack, results);
+ chooseLocalRack(results.get(1), excludedNodes, blocksize,
+ maxNodesPerRack, results);
}
if (--numOfReplicas == 0) {
break;
@@ -290,7 +290,7 @@
maxReplicasPerRack, results);
}
}
-
+
/* Randomly choose one target from <i>nodes</i>.
* @return the choosen node
*/
Modified: hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestReplicationPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestReplicationPolicy.java?rev=638144&r1=638143&r2=638144&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestReplicationPolicy.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestReplicationPolicy.java Mon Mar 17 16:55:09 2008
@@ -73,10 +73,11 @@
/**
* In this testcase, client is dataNodes[0]. So the 1st replica should be
- * placed on dataNodes[0], the 2nd replica should be placed on dataNodes[1],
- * and the rest should be placed on different racks.
- * The only excpetion is when the <i>numOfReplicas</i> is 2, the 1st is on
- * dataNodes[0] and the 2nd is on a different rack.
+ * placed on dataNodes[0], the 2nd replica should be placed on
+ * different rack and third should be placed on different node
+ * of rack chosen for 2nd node.
+ * The only excpetion is when the <i>numOfReplicas</i> is 2,
+ * the 1st is on dataNodes[0] and the 2nd is on a different rack.
* @throws Exception
*/
public void testChooseTarget1() throws Exception {
@@ -104,17 +105,17 @@
3, dataNodes[0], null, BLOCK_SIZE);
assertEquals(targets.length, 3);
assertEquals(targets[0], dataNodes[0]);
- assertTrue(cluster.isOnSameRack(targets[0], targets[1]));
- assertFalse(cluster.isOnSameRack(targets[0], targets[2]));
-
+ assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
+ assertTrue(cluster.isOnSameRack(targets[1], targets[2]));
+
targets = replicator.chooseTarget(
- 4, dataNodes[0], null, BLOCK_SIZE);
+ 4, dataNodes[0], null, BLOCK_SIZE);
assertEquals(targets.length, 4);
assertEquals(targets[0], dataNodes[0]);
- assertTrue(cluster.isOnSameRack(targets[0], targets[1]));
+ assertTrue(cluster.isOnSameRack(targets[1], targets[2]) ||
+ cluster.isOnSameRack(targets[2], targets[3]));
assertFalse(cluster.isOnSameRack(targets[0], targets[2]));
- assertFalse(cluster.isOnSameRack(targets[0], targets[3]));
-
+
dataNodes[0].updateHeartbeat(
2*FSConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L,
FSConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0);
@@ -124,7 +125,7 @@
* In this testcase, client is dataNodes[0], but the dataNodes[1] is
* not allowed to be choosen. So the 1st replica should be
* placed on dataNodes[0], the 2nd replica should be placed on a different
- * rack, the 3rd should the same rack as the 3nd replic, and the rest
+ * rack, the 3rd should be on same rack as the 2nd replica, and the rest
* should be placed on a third rack.
* @throws Exception
*/
@@ -281,7 +282,7 @@
* In this testcase, client is is a node outside of file system.
* So the 1st replica can be placed on any node.
* the 2nd replica should be placed on a different rack,
- * the 3rd replica should be placed on the same rack as the 1st replica,
+ * the 3rd replica should be placed on the same rack as the 2nd replica,
* @throws Exception
*/
public void testChooseTarget5() throws Exception {
@@ -302,15 +303,15 @@
targets = replicator.chooseTarget(
3, NODE, null, BLOCK_SIZE);
assertEquals(targets.length, 3);
- assertTrue(cluster.isOnSameRack(targets[0], targets[1]));
- assertFalse(cluster.isOnSameRack(targets[0], targets[2]));
+ assertTrue(cluster.isOnSameRack(targets[1], targets[2]));
+ assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
}
/**
* This testcase tests re-replication, when dataNodes[0] is already choosen.
- * So the 1st replica can be placed on rack 1.
- * the 2nd replica should be placed on a different rack,
- * the 3rd replica can be placed randomly,
+ * So the 1st replica can be placed on random rack.
+ * the 2nd replica should be placed on different node by same rack as
+ * the 1st replica. The 3rd replica can be placed randomly.
* @throws Exception
*/
public void testRereplicate1() throws Exception {
@@ -330,15 +331,16 @@
targets = replicator.chooseTarget(
2, dataNodes[0], choosenNodes, null, BLOCK_SIZE);
assertEquals(targets.length, 2);
- assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0]));
- assertFalse(cluster.isOnSameRack(dataNodes[0], targets[1]));
+ assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0]));
+ assertTrue(cluster.isOnSameRack(targets[0], targets[1]));
targets = replicator.chooseTarget(
3, dataNodes[0], choosenNodes, null, BLOCK_SIZE);
assertEquals(targets.length, 3);
- assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0]));
- assertFalse(cluster.isOnSameRack(dataNodes[0], targets[1]));
- assertFalse(cluster.isOnSameRack(dataNodes[0], targets[2]));
+ assertTrue(cluster.isOnSameRack(targets[0], targets[1]) ||
+ cluster.isOnSameRack(targets[1], targets[2]));
+ assertFalse(cluster.isOnSameRack(targets[0], targets[2]));
+ assertFalse(cluster.isOnSameRack(dataNodes[0], targets[1]));
}
/**
@@ -373,7 +375,7 @@
/**
* This testcase tests re-replication,
* when dataNodes[0] and dataNodes[2] are already choosen.
- * So the 1st replica should be placed on rack 1.
+ * So the 1st replica should be placed on rack 2.
* the rest replicas can be placed randomly,
* @throws Exception
*/
@@ -390,12 +392,13 @@
targets = replicator.chooseTarget(
1, dataNodes[0], choosenNodes, null, BLOCK_SIZE);
assertEquals(targets.length, 1);
- assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0]));
+ assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0]));
+ assertTrue(cluster.isOnSameRack(dataNodes[2], targets[0]));
targets = replicator.chooseTarget(
2, dataNodes[0], choosenNodes, null, BLOCK_SIZE);
assertEquals(targets.length, 2);
- assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0]));
- assertFalse(cluster.isOnSameRack(dataNodes[0], targets[1]));
+ assertTrue(cluster.isOnSameRack(dataNodes[2], targets[0]) ||
+ cluster.isOnSameRack(dataNodes[2], targets[1]));
}
}