You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2014/07/22 19:47:50 UTC
svn commit: r1612625 - in
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src:
main/java/org/apache/hadoop/net/ test/java/org/apache/hadoop/net/
Author: wang
Date: Tue Jul 22 17:47:49 2014
New Revision: 1612625
URL: http://svn.apache.org/r1612625
Log:
HDFS-6701. Make seed optional in NetworkTopology#sortByDistance. Contributed by Ashwin Shankar.
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java?rev=1612625&r1=1612624&r2=1612625&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java Tue Jul 22 17:47:49 2014
@@ -883,8 +883,8 @@ public class NetworkTopology {
* @param seed Used to seed the pseudo-random generator that randomizes the
* set of nodes at each network distance.
*/
- public void sortByDistance(Node reader, Node[] nodes,
- int activeLen, long seed) {
+ public void sortByDistance(Node reader, Node[] nodes, int activeLen,
+ long seed, boolean randomizeBlockLocationsPerBlock) {
/** Sort weights for the nodes array */
int[] weights = new int[activeLen];
for (int i=0; i<activeLen; i++) {
@@ -906,8 +906,11 @@ public class NetworkTopology {
// Seed is normally the block id
// This means we use the same pseudo-random order for each block, for
// potentially better page cache usage.
+ // Seed is not used if we want to randomize block location for every block
Random rand = getRandom();
- rand.setSeed(seed);
+ if (!randomizeBlockLocationsPerBlock) {
+ rand.setSeed(seed);
+ }
int idx = 0;
for (List<Node> list: tree.values()) {
if (list != null) {
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java?rev=1612625&r1=1612624&r2=1612625&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java Tue Jul 22 17:47:49 2014
@@ -279,8 +279,8 @@ public class NetworkTopologyWithNodeGrou
* set of nodes at each network distance.
*/
@Override
- public void sortByDistance( Node reader, Node[] nodes,
- int activeLen, long seed) {
+ public void sortByDistance(Node reader, Node[] nodes, int activeLen,
+ long seed, boolean randomizeBlockLocationsPerBlock) {
// If reader is not a datanode (not in NetworkTopology tree), we need to
// replace this reader with a sibling leaf node in tree.
if (reader != null && !this.contains(reader)) {
@@ -293,7 +293,8 @@ public class NetworkTopologyWithNodeGrou
return;
}
}
- super.sortByDistance(reader, nodes, nodes.length, seed);
+ super.sortByDistance(reader, nodes, nodes.length, seed,
+ randomizeBlockLocationsPerBlock);
}
/** InnerNodeWithNodeGroup represents a switch/router of a data center, rack
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java?rev=1612625&r1=1612624&r2=1612625&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java Tue Jul 22 17:47:49 2014
@@ -105,7 +105,7 @@ public class TestNetworkTopologyWithNode
testNodes[2] = dataNodes[3];
testNodes[3] = dataNodes[0];
cluster.sortByDistance(dataNodes[0], testNodes,
- testNodes.length, 0xDEADBEEF);
+ testNodes.length, 0xDEADBEEF, false);
assertTrue(testNodes[0] == dataNodes[0]);
assertTrue(testNodes[1] == dataNodes[1]);
assertTrue(testNodes[2] == dataNodes[2]);
@@ -117,7 +117,7 @@ public class TestNetworkTopologyWithNode
testNodes[2] = dataNodes[1];
testNodes[3] = dataNodes[0];
cluster.sortByDistance(dataNodes[0], testNodes,
- testNodes.length, 0xDEADBEEF);
+ testNodes.length, 0xDEADBEEF, false);
assertTrue(testNodes[0] == dataNodes[0]);
assertTrue(testNodes[1] == dataNodes[1]);
@@ -127,7 +127,7 @@ public class TestNetworkTopologyWithNode
testNodes[2] = dataNodes[2];
testNodes[3] = dataNodes[0];
cluster.sortByDistance(dataNodes[0], testNodes,
- testNodes.length, 0xDEADBEEF);
+ testNodes.length, 0xDEADBEEF, false);
assertTrue(testNodes[0] == dataNodes[0]);
assertTrue(testNodes[1] == dataNodes[2]);
@@ -137,7 +137,7 @@ public class TestNetworkTopologyWithNode
testNodes[2] = dataNodes[2];
testNodes[3] = dataNodes[0];
cluster.sortByDistance(computeNode, testNodes,
- testNodes.length, 0xDEADBEEF);
+ testNodes.length, 0xDEADBEEF, false);
assertTrue(testNodes[0] == dataNodes[0]);
assertTrue(testNodes[1] == dataNodes[2]);
}