You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ar...@apache.org on 2013/12/12 08:17:58 UTC
svn commit: r1550363 [7/8] - in
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/java/
src/main/java/org/apache/hadoop/hdfs/
src/main/java/org/apache/hadoop/hdfs/protocol/
src/main/java/org/apache/hadoop/hdfs/protocolPB/ src/main/java/...
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java Thu Dec 12 07:17:51 2013
@@ -36,6 +36,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.net.NetworkTopology;
@@ -57,41 +58,57 @@ public class TestReplicationPolicyWithNo
private BlockPlacementPolicy replicator;
private static final String filename = "/dummyfile.txt";
- private final static DatanodeDescriptor dataNodes[] = new DatanodeDescriptor[] {
- DFSTestUtil.getDatanodeDescriptor("1.1.1.1", "/d1/r1/n1"),
- DFSTestUtil.getDatanodeDescriptor("2.2.2.2", "/d1/r1/n1"),
- DFSTestUtil.getDatanodeDescriptor("3.3.3.3", "/d1/r1/n2"),
- DFSTestUtil.getDatanodeDescriptor("4.4.4.4", "/d1/r2/n3"),
- DFSTestUtil.getDatanodeDescriptor("5.5.5.5", "/d1/r2/n3"),
- DFSTestUtil.getDatanodeDescriptor("6.6.6.6", "/d1/r2/n4"),
- DFSTestUtil.getDatanodeDescriptor("7.7.7.7", "/d2/r3/n5"),
- DFSTestUtil.getDatanodeDescriptor("8.8.8.8", "/d2/r3/n6")
- };
-
- private final static DatanodeDescriptor dataNodesInBoundaryCase[] =
- new DatanodeDescriptor[] {
- DFSTestUtil.getDatanodeDescriptor("1.1.1.1", "/d1/r1/n1"),
- DFSTestUtil.getDatanodeDescriptor("2.2.2.2", "/d1/r1/n1"),
- DFSTestUtil.getDatanodeDescriptor("3.3.3.3", "/d1/r1/n1"),
- DFSTestUtil.getDatanodeDescriptor("4.4.4.4", "/d1/r1/n2"),
- DFSTestUtil.getDatanodeDescriptor("5.5.5.5", "/d1/r2/n3"),
- DFSTestUtil.getDatanodeDescriptor("6.6.6.6", "/d1/r2/n3")
- };
-
- private final static DatanodeDescriptor dataNodesInMoreTargetsCase[] =
- new DatanodeDescriptor[] {
- DFSTestUtil.getDatanodeDescriptor("1.1.1.1", "/r1/n1"),
- DFSTestUtil.getDatanodeDescriptor("2.2.2.2", "/r1/n1"),
- DFSTestUtil.getDatanodeDescriptor("3.3.3.3", "/r1/n2"),
- DFSTestUtil.getDatanodeDescriptor("4.4.4.4", "/r1/n2"),
- DFSTestUtil.getDatanodeDescriptor("5.5.5.5", "/r1/n3"),
- DFSTestUtil.getDatanodeDescriptor("6.6.6.6", "/r1/n3"),
- DFSTestUtil.getDatanodeDescriptor("7.7.7.7", "/r2/n4"),
- DFSTestUtil.getDatanodeDescriptor("8.8.8.8", "/r2/n4"),
- DFSTestUtil.getDatanodeDescriptor("9.9.9.9", "/r2/n5"),
- DFSTestUtil.getDatanodeDescriptor("10.10.10.10", "/r2/n5"),
- DFSTestUtil.getDatanodeDescriptor("11.11.11.11", "/r2/n6"),
- DFSTestUtil.getDatanodeDescriptor("12.12.12.12", "/r2/n6"),
+ private static final DatanodeStorageInfo[] storages;
+ private static final DatanodeDescriptor[] dataNodes;
+ static {
+ final String[] racks = {
+ "/d1/r1/n1",
+ "/d1/r1/n1",
+ "/d1/r1/n2",
+ "/d1/r2/n3",
+ "/d1/r2/n3",
+ "/d1/r2/n4",
+ "/d2/r3/n5",
+ "/d2/r3/n6"
+ };
+ storages = DFSTestUtil.createDatanodeStorageInfos(racks);
+ dataNodes = DFSTestUtil.toDatanodeDescriptor(storages);
+ }
+
+ private static final DatanodeStorageInfo[] storagesInBoundaryCase;
+ private static final DatanodeDescriptor[] dataNodesInBoundaryCase;
+ static {
+ final String[] racksInBoundaryCase = {
+ "/d1/r1/n1",
+ "/d1/r1/n1",
+ "/d1/r1/n1",
+ "/d1/r1/n2",
+ "/d1/r2/n3",
+ "/d1/r2/n3"
+ };
+ storagesInBoundaryCase = DFSTestUtil.createDatanodeStorageInfos(racksInBoundaryCase);
+ dataNodesInBoundaryCase = DFSTestUtil.toDatanodeDescriptor(storagesInBoundaryCase);
+ }
+
+ private static final DatanodeStorageInfo[] storagesInMoreTargetsCase;
+ private final static DatanodeDescriptor[] dataNodesInMoreTargetsCase;
+ static {
+ final String[] racksInMoreTargetsCase = {
+ "/r1/n1",
+ "/r1/n1",
+ "/r1/n2",
+ "/r1/n2",
+ "/r1/n3",
+ "/r1/n3",
+ "/r2/n4",
+ "/r2/n4",
+ "/r2/n5",
+ "/r2/n5",
+ "/r2/n6",
+ "/r2/n6"
+ };
+ storagesInMoreTargetsCase = DFSTestUtil.createDatanodeStorageInfos(racksInMoreTargetsCase);
+ dataNodesInMoreTargetsCase = DFSTestUtil.toDatanodeDescriptor(storagesInMoreTargetsCase);
};
private final static DatanodeDescriptor NODE =
@@ -129,9 +146,20 @@ public class TestReplicationPolicyWithNo
namenode.stop();
}
+ private static void updateHeartbeatWithUsage(DatanodeDescriptor dn,
+ long capacity, long dfsUsed, long remaining, long blockPoolUsed,
+ long dnCacheCapacity, long dnCacheUsed, int xceiverCount,
+ int volFailures) {
+ dn.getStorageInfos()[0].setUtilizationForTesting(
+ capacity, dfsUsed, remaining, blockPoolUsed);
+ dn.updateHeartbeat(
+ BlockManagerTestUtil.getStorageReportsForDatanode(dn),
+ dnCacheCapacity, dnCacheUsed, xceiverCount, volFailures);
+ }
+
private static void setupDataNodeCapacity() {
for(int i=0; i<NUM_OF_DATANODES; i++) {
- dataNodes[i].updateHeartbeat(
+ updateHeartbeatWithUsage(dataNodes[i],
2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L,
2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 0L, 0L, 0, 0);
}
@@ -142,11 +170,12 @@ public class TestReplicationPolicyWithNo
* Return false if two targets are found on the same NodeGroup.
*/
private static boolean checkTargetsOnDifferentNodeGroup(
- DatanodeDescriptor[] targets) {
+ DatanodeStorageInfo[] targets) {
if(targets.length == 0)
return true;
Set<String> targetSet = new HashSet<String>();
- for(DatanodeDescriptor node:targets) {
+ for(DatanodeStorageInfo storage:targets) {
+ final DatanodeDescriptor node = storage.getDatanodeDescriptor();
String nodeGroup = NetworkTopology.getLastHalf(node.getNetworkLocation());
if(targetSet.contains(nodeGroup)) {
return false;
@@ -156,34 +185,50 @@ public class TestReplicationPolicyWithNo
}
return true;
}
-
- private DatanodeDescriptor[] chooseTarget(int numOfReplicas) {
+
+ private boolean isOnSameRack(DatanodeStorageInfo left, DatanodeStorageInfo right) {
+ return isOnSameRack(left.getDatanodeDescriptor(), right);
+ }
+
+ private boolean isOnSameRack(DatanodeDescriptor left, DatanodeStorageInfo right) {
+ return cluster.isOnSameRack(left, right.getDatanodeDescriptor());
+ }
+
+ private boolean isOnSameNodeGroup(DatanodeStorageInfo left, DatanodeStorageInfo right) {
+ return isOnSameNodeGroup(left.getDatanodeDescriptor(), right);
+ }
+
+ private boolean isOnSameNodeGroup(DatanodeDescriptor left, DatanodeStorageInfo right) {
+ return cluster.isOnSameNodeGroup(left, right.getDatanodeDescriptor());
+ }
+
+ private DatanodeStorageInfo[] chooseTarget(int numOfReplicas) {
return chooseTarget(numOfReplicas, dataNodes[0]);
}
- private DatanodeDescriptor[] chooseTarget(int numOfReplicas,
+ private DatanodeStorageInfo[] chooseTarget(int numOfReplicas,
DatanodeDescriptor writer) {
return chooseTarget(numOfReplicas, writer,
- new ArrayList<DatanodeDescriptor>());
+ new ArrayList<DatanodeStorageInfo>());
}
- private DatanodeDescriptor[] chooseTarget(int numOfReplicas,
- List<DatanodeDescriptor> chosenNodes) {
+ private DatanodeStorageInfo[] chooseTarget(int numOfReplicas,
+ List<DatanodeStorageInfo> chosenNodes) {
return chooseTarget(numOfReplicas, dataNodes[0], chosenNodes);
}
- private DatanodeDescriptor[] chooseTarget(int numOfReplicas,
- DatanodeDescriptor writer, List<DatanodeDescriptor> chosenNodes) {
+ private DatanodeStorageInfo[] chooseTarget(int numOfReplicas,
+ DatanodeDescriptor writer, List<DatanodeStorageInfo> chosenNodes) {
return chooseTarget(numOfReplicas, writer, chosenNodes, null);
}
- private DatanodeDescriptor[] chooseTarget(
+ private DatanodeStorageInfo[] chooseTarget(
int numOfReplicas,
DatanodeDescriptor writer,
- List<DatanodeDescriptor> chosenNodes,
+ List<DatanodeStorageInfo> chosenNodes,
Set<Node> excludedNodes) {
return replicator.chooseTarget(filename, numOfReplicas, writer, chosenNodes,
- false, excludedNodes, BLOCK_SIZE);
+ false, excludedNodes, BLOCK_SIZE, StorageType.DEFAULT);
}
/**
@@ -197,49 +242,53 @@ public class TestReplicationPolicyWithNo
*/
@Test
public void testChooseTarget1() throws Exception {
- dataNodes[0].updateHeartbeat(
+ updateHeartbeatWithUsage(dataNodes[0],
2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L,
HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L,
0L, 0L, 4, 0); // overloaded
- DatanodeDescriptor[] targets;
+ DatanodeStorageInfo[] targets;
targets = chooseTarget(0);
assertEquals(targets.length, 0);
targets = chooseTarget(1);
assertEquals(targets.length, 1);
- assertEquals(targets[0], dataNodes[0]);
+ assertEquals(storages[0], targets[0]);
+
targets = chooseTarget(2);
assertEquals(targets.length, 2);
- assertEquals(targets[0], dataNodes[0]);
- assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
+ assertEquals(storages[0], targets[0]);
+
+ assertFalse(isOnSameRack(targets[0], targets[1]));
targets = chooseTarget(3);
assertEquals(targets.length, 3);
- assertEquals(targets[0], dataNodes[0]);
- assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
- assertTrue(cluster.isOnSameRack(targets[1], targets[2]));
- assertFalse(cluster.isOnSameNodeGroup(targets[1], targets[2]));
+ assertEquals(storages[0], targets[0]);
+
+ assertFalse(isOnSameRack(targets[0], targets[1]));
+ assertTrue(isOnSameRack(targets[1], targets[2]));
+ assertFalse(isOnSameNodeGroup(targets[1], targets[2]));
targets = chooseTarget(4);
assertEquals(targets.length, 4);
- assertEquals(targets[0], dataNodes[0]);
- assertTrue(cluster.isOnSameRack(targets[1], targets[2]) ||
- cluster.isOnSameRack(targets[2], targets[3]));
- assertFalse(cluster.isOnSameRack(targets[0], targets[2]));
+ assertEquals(storages[0], targets[0]);
+
+ assertTrue(isOnSameRack(targets[1], targets[2]) ||
+ isOnSameRack(targets[2], targets[3]));
+ assertFalse(isOnSameRack(targets[0], targets[2]));
// Make sure no more than one replicas are on the same nodegroup
verifyNoTwoTargetsOnSameNodeGroup(targets);
- dataNodes[0].updateHeartbeat(
+ updateHeartbeatWithUsage(dataNodes[0],
2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L,
HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 0L, 0L, 0, 0);
}
- private void verifyNoTwoTargetsOnSameNodeGroup(DatanodeDescriptor[] targets) {
+ private void verifyNoTwoTargetsOnSameNodeGroup(DatanodeStorageInfo[] targets) {
Set<String> nodeGroupSet = new HashSet<String>();
- for (DatanodeDescriptor target: targets) {
- nodeGroupSet.add(target.getNetworkLocation());
+ for (DatanodeStorageInfo target: targets) {
+ nodeGroupSet.add(target.getDatanodeDescriptor().getNetworkLocation());
}
assertEquals(nodeGroupSet.size(), targets.length);
}
@@ -254,36 +303,37 @@ public class TestReplicationPolicyWithNo
*/
@Test
public void testChooseTarget2() throws Exception {
- DatanodeDescriptor[] targets;
+ DatanodeStorageInfo[] targets;
BlockPlacementPolicyDefault repl = (BlockPlacementPolicyDefault)replicator;
- List<DatanodeDescriptor> chosenNodes = new ArrayList<DatanodeDescriptor>();
+ List<DatanodeStorageInfo> chosenNodes = new ArrayList<DatanodeStorageInfo>();
Set<Node> excludedNodes = new HashSet<Node>();
excludedNodes.add(dataNodes[1]);
targets = repl.chooseTarget(filename, 4, dataNodes[0], chosenNodes, false,
- excludedNodes, BLOCK_SIZE);
+ excludedNodes, BLOCK_SIZE, StorageType.DEFAULT);
assertEquals(targets.length, 4);
- assertEquals(targets[0], dataNodes[0]);
+ assertEquals(storages[0], targets[0]);
+
assertTrue(cluster.isNodeGroupAware());
// Make sure no replicas are on the same nodegroup
for (int i=1;i<4;i++) {
- assertFalse(cluster.isOnSameNodeGroup(targets[0], targets[i]));
+ assertFalse(isOnSameNodeGroup(targets[0], targets[i]));
}
- assertTrue(cluster.isOnSameRack(targets[1], targets[2]) ||
- cluster.isOnSameRack(targets[2], targets[3]));
- assertFalse(cluster.isOnSameRack(targets[1], targets[3]));
+ assertTrue(isOnSameRack(targets[1], targets[2]) ||
+ isOnSameRack(targets[2], targets[3]));
+ assertFalse(isOnSameRack(targets[1], targets[3]));
excludedNodes.clear();
chosenNodes.clear();
excludedNodes.add(dataNodes[1]);
- chosenNodes.add(dataNodes[2]);
+ chosenNodes.add(storages[2]);
targets = repl.chooseTarget(filename, 1, dataNodes[0], chosenNodes, true,
- excludedNodes, BLOCK_SIZE);
+ excludedNodes, BLOCK_SIZE, StorageType.DEFAULT);
System.out.println("targets=" + Arrays.asList(targets));
assertEquals(2, targets.length);
//make sure that the chosen node is in the target.
int i = 0;
- for(; i < targets.length && !dataNodes[2].equals(targets[i]); i++);
+ for(; i < targets.length && !storages[2].equals(targets[i]); i++);
assertTrue(i < targets.length);
}
@@ -298,39 +348,39 @@ public class TestReplicationPolicyWithNo
@Test
public void testChooseTarget3() throws Exception {
// make data node 0 to be not qualified to choose
- dataNodes[0].updateHeartbeat(
+ updateHeartbeatWithUsage(dataNodes[0],
2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L,
(HdfsConstants.MIN_BLOCKS_FOR_WRITE-1)*BLOCK_SIZE, 0L,
0L, 0L, 0, 0); // no space
- DatanodeDescriptor[] targets;
+ DatanodeStorageInfo[] targets;
targets = chooseTarget(0);
assertEquals(targets.length, 0);
targets = chooseTarget(1);
assertEquals(targets.length, 1);
- assertEquals(targets[0], dataNodes[1]);
+ assertEquals(storages[1], targets[0]);
targets = chooseTarget(2);
assertEquals(targets.length, 2);
- assertEquals(targets[0], dataNodes[1]);
- assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
+ assertEquals(storages[1], targets[0]);
+ assertFalse(isOnSameRack(targets[0], targets[1]));
targets = chooseTarget(3);
assertEquals(targets.length, 3);
- assertEquals(targets[0], dataNodes[1]);
- assertTrue(cluster.isOnSameRack(targets[1], targets[2]));
- assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
+ assertEquals(storages[1], targets[0]);
+ assertTrue(isOnSameRack(targets[1], targets[2]));
+ assertFalse(isOnSameRack(targets[0], targets[1]));
targets = chooseTarget(4);
assertEquals(targets.length, 4);
- assertEquals(targets[0], dataNodes[1]);
+ assertEquals(storages[1], targets[0]);
assertTrue(cluster.isNodeGroupAware());
verifyNoTwoTargetsOnSameNodeGroup(targets);
- assertTrue(cluster.isOnSameRack(targets[1], targets[2]) ||
- cluster.isOnSameRack(targets[2], targets[3]));
+ assertTrue(isOnSameRack(targets[1], targets[2]) ||
+ isOnSameRack(targets[2], targets[3]));
- dataNodes[0].updateHeartbeat(
+ updateHeartbeatWithUsage(dataNodes[0],
2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L,
HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 0L, 0L, 0, 0);
}
@@ -348,33 +398,33 @@ public class TestReplicationPolicyWithNo
public void testChooseTarget4() throws Exception {
// make data node 0-2 to be not qualified to choose: not enough disk space
for(int i=0; i<3; i++) {
- dataNodes[i].updateHeartbeat(
+ updateHeartbeatWithUsage(dataNodes[i],
2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L,
(HdfsConstants.MIN_BLOCKS_FOR_WRITE-1)*BLOCK_SIZE, 0L, 0L, 0L, 0, 0);
}
- DatanodeDescriptor[] targets;
+ DatanodeStorageInfo[] targets;
targets = chooseTarget(0);
assertEquals(targets.length, 0);
targets = chooseTarget(1);
assertEquals(targets.length, 1);
- assertFalse(cluster.isOnSameRack(targets[0], dataNodes[0]));
+ assertFalse(isOnSameRack(dataNodes[0], targets[0]));
targets = chooseTarget(2);
assertEquals(targets.length, 2);
- assertFalse(cluster.isOnSameRack(targets[0], dataNodes[0]));
- assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
+ assertFalse(isOnSameRack(dataNodes[0], targets[0]));
+ assertFalse(isOnSameRack(targets[0], targets[1]));
targets = chooseTarget(3);
assertEquals(targets.length, 3);
for(int i=0; i<3; i++) {
- assertFalse(cluster.isOnSameRack(targets[i], dataNodes[0]));
+ assertFalse(isOnSameRack(dataNodes[0], targets[i]));
}
verifyNoTwoTargetsOnSameNodeGroup(targets);
- assertTrue(cluster.isOnSameRack(targets[0], targets[1]) ||
- cluster.isOnSameRack(targets[1], targets[2]));
- assertFalse(cluster.isOnSameRack(targets[0], targets[2]));
+ assertTrue(isOnSameRack(targets[0], targets[1]) ||
+ isOnSameRack(targets[1], targets[2]));
+ assertFalse(isOnSameRack(targets[0], targets[2]));
}
/**
@@ -387,7 +437,7 @@ public class TestReplicationPolicyWithNo
@Test
public void testChooseTarget5() throws Exception {
setupDataNodeCapacity();
- DatanodeDescriptor[] targets;
+ DatanodeStorageInfo[] targets;
targets = chooseTarget(0, NODE);
assertEquals(targets.length, 0);
@@ -396,12 +446,12 @@ public class TestReplicationPolicyWithNo
targets = chooseTarget(2, NODE);
assertEquals(targets.length, 2);
- assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
+ assertFalse(isOnSameRack(targets[0], targets[1]));
targets = chooseTarget(3, NODE);
assertEquals(targets.length, 3);
- assertTrue(cluster.isOnSameRack(targets[1], targets[2]));
- assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
+ assertTrue(isOnSameRack(targets[1], targets[2]));
+ assertFalse(isOnSameRack(targets[0], targets[1]));
verifyNoTwoTargetsOnSameNodeGroup(targets);
}
@@ -415,27 +465,27 @@ public class TestReplicationPolicyWithNo
@Test
public void testRereplicate1() throws Exception {
setupDataNodeCapacity();
- List<DatanodeDescriptor> chosenNodes = new ArrayList<DatanodeDescriptor>();
- chosenNodes.add(dataNodes[0]);
- DatanodeDescriptor[] targets;
+ List<DatanodeStorageInfo> chosenNodes = new ArrayList<DatanodeStorageInfo>();
+ chosenNodes.add(storages[0]);
+ DatanodeStorageInfo[] targets;
targets = chooseTarget(0, chosenNodes);
assertEquals(targets.length, 0);
targets = chooseTarget(1, chosenNodes);
assertEquals(targets.length, 1);
- assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0]));
+ assertFalse(isOnSameRack(dataNodes[0], targets[0]));
targets = chooseTarget(2, chosenNodes);
assertEquals(targets.length, 2);
- assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0]));
- assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
+ assertTrue(isOnSameRack(dataNodes[0], targets[0]));
+ assertFalse(isOnSameRack(targets[0], targets[1]));
targets = chooseTarget(3, chosenNodes);
assertEquals(targets.length, 3);
- assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0]));
- assertFalse(cluster.isOnSameNodeGroup(dataNodes[0], targets[0]));
- assertFalse(cluster.isOnSameRack(targets[0], targets[2]));
+ assertTrue(isOnSameRack(dataNodes[0], targets[0]));
+ assertFalse(isOnSameNodeGroup(dataNodes[0], targets[0]));
+ assertFalse(isOnSameRack(targets[0], targets[2]));
}
/**
@@ -448,22 +498,22 @@ public class TestReplicationPolicyWithNo
@Test
public void testRereplicate2() throws Exception {
setupDataNodeCapacity();
- List<DatanodeDescriptor> chosenNodes = new ArrayList<DatanodeDescriptor>();
- chosenNodes.add(dataNodes[0]);
- chosenNodes.add(dataNodes[1]);
+ List<DatanodeStorageInfo> chosenNodes = new ArrayList<DatanodeStorageInfo>();
+ chosenNodes.add(storages[0]);
+ chosenNodes.add(storages[1]);
- DatanodeDescriptor[] targets;
+ DatanodeStorageInfo[] targets;
targets = chooseTarget(0, chosenNodes);
assertEquals(targets.length, 0);
targets = chooseTarget(1, chosenNodes);
assertEquals(targets.length, 1);
- assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0]));
+ assertFalse(isOnSameRack(dataNodes[0], targets[0]));
targets = chooseTarget(2, chosenNodes);
assertEquals(targets.length, 2);
- assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0]) &&
- cluster.isOnSameRack(dataNodes[0], targets[1]));
+ assertFalse(isOnSameRack(dataNodes[0], targets[0]) &&
+ isOnSameRack(dataNodes[0], targets[1]));
}
/**
@@ -476,33 +526,33 @@ public class TestReplicationPolicyWithNo
@Test
public void testRereplicate3() throws Exception {
setupDataNodeCapacity();
- List<DatanodeDescriptor> chosenNodes = new ArrayList<DatanodeDescriptor>();
- chosenNodes.add(dataNodes[0]);
- chosenNodes.add(dataNodes[3]);
+ List<DatanodeStorageInfo> chosenNodes = new ArrayList<DatanodeStorageInfo>();
+ chosenNodes.add(storages[0]);
+ chosenNodes.add(storages[3]);
- DatanodeDescriptor[] targets;
+ DatanodeStorageInfo[] targets;
targets = chooseTarget(0, chosenNodes);
assertEquals(targets.length, 0);
targets = chooseTarget(1, chosenNodes);
assertEquals(targets.length, 1);
- assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0]));
- assertFalse(cluster.isOnSameRack(dataNodes[3], targets[0]));
+ assertTrue(isOnSameRack(dataNodes[0], targets[0]));
+ assertFalse(isOnSameRack(dataNodes[3], targets[0]));
targets = chooseTarget(1, dataNodes[3], chosenNodes);
assertEquals(targets.length, 1);
- assertTrue(cluster.isOnSameRack(dataNodes[3], targets[0]));
- assertFalse(cluster.isOnSameNodeGroup(dataNodes[3], targets[0]));
- assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0]));
+ assertTrue(isOnSameRack(dataNodes[3], targets[0]));
+ assertFalse(isOnSameNodeGroup(dataNodes[3], targets[0]));
+ assertFalse(isOnSameRack(dataNodes[0], targets[0]));
targets = chooseTarget(2, chosenNodes);
assertEquals(targets.length, 2);
- assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0]));
- assertFalse(cluster.isOnSameNodeGroup(dataNodes[0], targets[0]));
+ assertTrue(isOnSameRack(dataNodes[0], targets[0]));
+ assertFalse(isOnSameNodeGroup(dataNodes[0], targets[0]));
targets = chooseTarget(2, dataNodes[3], chosenNodes);
assertEquals(targets.length, 2);
- assertTrue(cluster.isOnSameRack(dataNodes[3], targets[0]));
+ assertTrue(isOnSameRack(dataNodes[3], targets[0]));
}
/**
@@ -576,16 +626,17 @@ public class TestReplicationPolicyWithNo
cluster.add(dataNodesInBoundaryCase[i]);
}
for(int i=0; i<NUM_OF_DATANODES_BOUNDARY; i++) {
- dataNodes[0].updateHeartbeat(
+ updateHeartbeatWithUsage(dataNodes[0],
2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L,
- (HdfsConstants.MIN_BLOCKS_FOR_WRITE-1)*BLOCK_SIZE, 0L, 0L, 0L, 0, 0);
-
- dataNodesInBoundaryCase[i].updateHeartbeat(
+ (HdfsConstants.MIN_BLOCKS_FOR_WRITE-1)*BLOCK_SIZE,
+ 0L, 0L, 0L, 0, 0);
+
+ updateHeartbeatWithUsage(dataNodesInBoundaryCase[i],
2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L,
2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 0L, 0L, 0, 0);
}
- DatanodeDescriptor[] targets;
+ DatanodeStorageInfo[] targets;
targets = chooseTarget(0, dataNodesInBoundaryCase[0]);
assertEquals(targets.length, 0);
@@ -594,7 +645,7 @@ public class TestReplicationPolicyWithNo
targets = chooseTarget(2, dataNodesInBoundaryCase[0]);
assertEquals(targets.length, 2);
- assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
+ assertFalse(isOnSameRack(targets[0], targets[1]));
targets = chooseTarget(3, dataNodesInBoundaryCase[0]);
assertEquals(targets.length, 3);
@@ -611,19 +662,17 @@ public class TestReplicationPolicyWithNo
@Test
public void testRereplicateOnBoundaryTopology() throws Exception {
for(int i=0; i<NUM_OF_DATANODES_BOUNDARY; i++) {
- dataNodesInBoundaryCase[i].updateHeartbeat(
+ updateHeartbeatWithUsage(dataNodesInBoundaryCase[i],
2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L,
2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 0L, 0L, 0, 0);
}
- List<DatanodeDescriptor> chosenNodes = new ArrayList<DatanodeDescriptor>();
- chosenNodes.add(dataNodesInBoundaryCase[0]);
- chosenNodes.add(dataNodesInBoundaryCase[5]);
- DatanodeDescriptor[] targets;
+ List<DatanodeStorageInfo> chosenNodes = new ArrayList<DatanodeStorageInfo>();
+ chosenNodes.add(storagesInBoundaryCase[0]);
+ chosenNodes.add(storagesInBoundaryCase[5]);
+ DatanodeStorageInfo[] targets;
targets = chooseTarget(1, dataNodesInBoundaryCase[0], chosenNodes);
- assertFalse(cluster.isOnSameNodeGroup(targets[0],
- dataNodesInBoundaryCase[0]));
- assertFalse(cluster.isOnSameNodeGroup(targets[0],
- dataNodesInBoundaryCase[5]));
+ assertFalse(isOnSameNodeGroup(dataNodesInBoundaryCase[0], targets[0]));
+ assertFalse(isOnSameNodeGroup(dataNodesInBoundaryCase[5], targets[0]));
assertTrue(checkTargetsOnDifferentNodeGroup(targets));
}
@@ -651,12 +700,12 @@ public class TestReplicationPolicyWithNo
}
for(int i=0; i<NUM_OF_DATANODES_MORE_TARGETS; i++) {
- dataNodesInMoreTargetsCase[i].updateHeartbeat(
+ updateHeartbeatWithUsage(dataNodesInMoreTargetsCase[i],
2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L,
2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 0L, 0L, 0, 0);
}
- DatanodeDescriptor[] targets;
+ DatanodeStorageInfo[] targets;
// Test normal case -- 3 replicas
targets = chooseTarget(3, dataNodesInMoreTargetsCase[0]);
assertEquals(targets.length, 3);
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlocks.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlocks.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlocks.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlocks.java Thu Dec 12 07:17:51 2013
@@ -48,7 +48,8 @@ public class TestUnderReplicatedBlocks {
// but the block does not get put into the under-replicated blocks queue
final BlockManager bm = cluster.getNamesystem().getBlockManager();
ExtendedBlock b = DFSTestUtil.getFirstBlock(fs, FILE_PATH);
- DatanodeDescriptor dn = bm.blocksMap.nodeIterator(b.getLocalBlock()).next();
+ DatanodeDescriptor dn = bm.blocksMap.getStorages(b.getLocalBlock())
+ .iterator().next().getDatanodeDescriptor();
bm.addToInvalidates(b.getLocalBlock(), dn);
// Compute the invalidate work in NN, and trigger the heartbeat from DN
BlockManagerTestUtil.computeAllPendingWork(bm);
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java Thu Dec 12 07:17:51 2013
@@ -17,14 +17,37 @@
*/
package org.apache.hadoop.hdfs.server.common;
-import com.google.common.base.Strings;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.net.InetSocketAddress;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+
+import javax.servlet.ServletContext;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.jsp.JspWriter;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.namenode.NameNodeHttpServer;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
+import org.apache.hadoop.hdfs.server.protocol.StorageReport;
import org.apache.hadoop.hdfs.web.resources.DoAsParam;
import org.apache.hadoop.hdfs.web.resources.UserParam;
import org.apache.hadoop.io.DataInputBuffer;
@@ -46,20 +69,7 @@ import org.mockito.stubbing.Answer;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
-import javax.servlet.ServletContext;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.jsp.JspWriter;
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-import java.io.IOException;
-import java.io.StringReader;
-import java.net.InetSocketAddress;
-import java.text.MessageFormat;
-import java.util.ArrayList;
-
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.*;
+import com.google.common.base.Strings;
public class TestJspHelper {
@@ -447,14 +457,28 @@ public class TestJspHelper {
@Test
public void testSortNodeByFields() throws Exception {
- DatanodeID dnId1 = new DatanodeID("127.0.0.1", "localhost1", "storage1",
+ DatanodeID dnId1 = new DatanodeID("127.0.0.1", "localhost1", "datanode1",
1234, 2345, 3456, 4567);
- DatanodeID dnId2 = new DatanodeID("127.0.0.2", "localhost2", "storage2",
+ DatanodeID dnId2 = new DatanodeID("127.0.0.2", "localhost2", "datanode2",
1235, 2346, 3457, 4568);
- DatanodeDescriptor dnDesc1 = new DatanodeDescriptor(dnId1, "rack1", 1024,
- 100, 924, 100, 5l, 3l, 10, 2);
- DatanodeDescriptor dnDesc2 = new DatanodeDescriptor(dnId2, "rack2", 2500,
- 200, 1848, 200, 10l, 2l, 20, 1);
+
+ // Setup DatanodeDescriptors with one storage each.
+ DatanodeDescriptor dnDesc1 = new DatanodeDescriptor(dnId1, "rack1");
+ DatanodeDescriptor dnDesc2 = new DatanodeDescriptor(dnId2, "rack2");
+
+ // Update the DatanodeDescriptors with their attached storages.
+ BlockManagerTestUtil.updateStorage(dnDesc1, new DatanodeStorage("dnStorage1"));
+ BlockManagerTestUtil.updateStorage(dnDesc2, new DatanodeStorage("dnStorage2"));
+
+ StorageReport[] report1 = new StorageReport[] {
+ new StorageReport("dnStorage1", false, 1024, 100, 924, 100)
+ };
+ StorageReport[] report2 = new StorageReport[] {
+ new StorageReport("dnStorage2", false, 2500, 200, 1848, 200)
+ };
+ dnDesc1.updateHeartbeat(report1, 5l, 3l, 10, 2);
+ dnDesc2.updateHeartbeat(report2, 10l, 2l, 20, 1);
+
ArrayList<DatanodeDescriptor> live = new ArrayList<DatanodeDescriptor>();
live.add(dnDesc1);
live.add(dnDesc2);
@@ -615,3 +639,4 @@ public class TestJspHelper {
MessageFormat.format(EXPECTED__NOTF_PATTERN, version)));
}
}
+
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java Thu Dec 12 07:17:51 2013
@@ -22,11 +22,11 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import java.util.Random;
import javax.management.NotCompliantMBeanException;
import javax.management.ObjectName;
@@ -48,7 +48,9 @@ import org.apache.hadoop.hdfs.server.dat
import org.apache.hadoop.hdfs.server.datanode.fsdataset.RollingLogs;
import org.apache.hadoop.hdfs.server.datanode.metrics.FSDatasetMBean;
import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo;
+import org.apache.hadoop.hdfs.server.protocol.StorageReport;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.util.DataChecksum;
@@ -74,7 +76,7 @@ public class SimulatedFSDataset implemen
@Override
public SimulatedFSDataset newInstance(DataNode datanode,
DataStorage storage, Configuration conf) throws IOException {
- return new SimulatedFSDataset(datanode, storage, conf);
+ return new SimulatedFSDataset(storage, conf);
}
@Override
@@ -136,6 +138,11 @@ public class SimulatedFSDataset implemen
}
@Override
+ public String getStorageUuid() {
+ return storage.getStorageUuid();
+ }
+
+ @Override
synchronized public long getGenerationStamp() {
return theBlock.getGenerationStamp();
}
@@ -318,13 +325,15 @@ public class SimulatedFSDataset implemen
private static class SimulatedStorage {
private Map<String, SimulatedBPStorage> map =
new HashMap<String, SimulatedBPStorage>();
- private long capacity; // in bytes
+ private final String storageUuid = "SimulatedStroage-" + DatanodeStorage.generateUuid();
+
+ private final long capacity; // in bytes
synchronized long getFree() {
return capacity - getUsed();
}
- synchronized long getCapacity() {
+ long getCapacity() {
return capacity;
}
@@ -379,22 +388,33 @@ public class SimulatedFSDataset implemen
}
return bpStorage;
}
+
+ String getStorageUuid() {
+ return storageUuid;
+ }
+
+ synchronized StorageReport getStorageReport(String bpid) {
+ return new StorageReport(getStorageUuid(), false, getCapacity(),
+ getUsed(), getFree(), map.get(bpid).getUsed());
+ }
}
private final Map<String, Map<Block, BInfo>> blockMap
= new HashMap<String, Map<Block,BInfo>>();
private final SimulatedStorage storage;
- private final String storageId;
+ private final String datanodeUuid;
- public SimulatedFSDataset(DataNode datanode, DataStorage storage,
- Configuration conf) {
+ public SimulatedFSDataset(DataStorage storage, Configuration conf) {
if (storage != null) {
- storage.createStorageID(datanode.getXferPort());
- this.storageId = storage.getStorageID();
+ for (int i = 0; i < storage.getNumStorageDirs(); ++i) {
+ storage.createStorageID(storage.getStorageDir(i));
+ }
+ this.datanodeUuid = storage.getDatanodeUuid();
} else {
- this.storageId = "unknownStorageId" + new Random().nextInt();
+ this.datanodeUuid = "SimulatedDatanode-" + DataNode.generateUuid();
}
- registerMBean(storageId);
+
+ registerMBean(datanodeUuid);
this.storage = new SimulatedStorage(
conf.getLong(CONFIG_PROPERTY_CAPACITY, DEFAULT_CAPACITY));
}
@@ -451,8 +471,7 @@ public class SimulatedFSDataset implemen
}
}
- @Override
- public synchronized BlockListAsLongs getBlockReport(String bpid) {
+ synchronized BlockListAsLongs getBlockReport(String bpid) {
final List<Block> blocks = new ArrayList<Block>();
final Map<Block, BInfo> map = blockMap.get(bpid);
if (map != null) {
@@ -465,6 +484,12 @@ public class SimulatedFSDataset implemen
return new BlockListAsLongs(blocks, null);
}
+ @Override
+ public synchronized Map<DatanodeStorage, BlockListAsLongs> getBlockReports(
+ String bpid) {
+ return Collections.singletonMap(new DatanodeStorage(storage.storageUuid), getBlockReport(bpid));
+ }
+
@Override // FsDatasetSpi
public List<Long> getCacheReport(String bpid) {
return new LinkedList<Long>();
@@ -661,7 +686,7 @@ public class SimulatedFSDataset implemen
}
@Override // FsDatasetSpi
- public void recoverClose(ExtendedBlock b, long newGS, long expectedBlockLen)
+ public String recoverClose(ExtendedBlock b, long newGS, long expectedBlockLen)
throws IOException {
final Map<Block, BInfo> map = getMap(b.getBlockPoolId());
BInfo binfo = map.get(b.getLocalBlock());
@@ -675,6 +700,7 @@ public class SimulatedFSDataset implemen
map.remove(b.getLocalBlock());
binfo.theBlock.setGenerationStamp(newGS);
map.put(binfo.theBlock, binfo);
+ return binfo.getStorageUuid();
}
@Override // FsDatasetSpi
@@ -931,7 +957,7 @@ public class SimulatedFSDataset implemen
@Override
public String getStorageInfo() {
- return "Simulated FSDataset-" + storageId;
+ return "Simulated FSDataset-" + datanodeUuid;
}
@Override
@@ -958,7 +984,8 @@ public class SimulatedFSDataset implemen
public String updateReplicaUnderRecovery(ExtendedBlock oldBlock,
long recoveryId,
long newlength) {
- return storageId;
+ // Caller does not care about the exact Storage UUID returned.
+ return datanodeUuid;
}
@Override // FsDatasetSpi
@@ -1013,11 +1040,6 @@ public class SimulatedFSDataset implemen
}
@Override
- public String[] getBlockPoolList() {
- throw new UnsupportedOperationException();
- }
-
- @Override
public void checkAndUpdate(String bpid, long blockId, File diskFile,
File diskMetaFile, FsVolumeSpi vol) {
throw new UnsupportedOperationException();
@@ -1029,7 +1051,12 @@ public class SimulatedFSDataset implemen
}
@Override
- public List<Block> getFinalizedBlocks(String bpid) {
+ public StorageReport[] getStorageReports(String bpid) {
+ return new StorageReport[] {storage.getStorageReport(bpid)};
+ }
+
+ @Override
+ public List<FinalizedReplica> getFinalizedBlocks(String bpid) {
throw new UnsupportedOperationException();
}
@@ -1048,3 +1075,4 @@ public class SimulatedFSDataset implemen
throw new UnsupportedOperationException();
}
}
+
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java Thu Dec 12 07:17:51 2013
@@ -104,7 +104,7 @@ public class TestBPOfferService {
.when(mockDn).getMetrics();
// Set up a simulated dataset with our fake BP
- mockFSDataset = Mockito.spy(new SimulatedFSDataset(null, null, conf));
+ mockFSDataset = Mockito.spy(new SimulatedFSDataset(null, conf));
mockFSDataset.addBlockPool(FAKE_BPID, conf);
// Wire the dataset to the DN.
@@ -180,7 +180,7 @@ public class TestBPOfferService {
waitForBlockReport(mockNN2);
// When we receive a block, it should report it to both NNs
- bpos.notifyNamenodeReceivedBlock(FAKE_BLOCK, "");
+ bpos.notifyNamenodeReceivedBlock(FAKE_BLOCK, "", "");
ReceivedDeletedBlockInfo[] ret = waitForBlockReceived(FAKE_BLOCK, mockNN1);
assertEquals(1, ret.length);
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java Thu Dec 12 07:17:51 2013
@@ -35,6 +35,7 @@ import static org.mockito.Mockito.when;
import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
+import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -121,7 +122,7 @@ public class TestBlockRecovery {
* @throws IOException
*/
@Before
- public void startUp() throws IOException {
+ public void startUp() throws IOException, URISyntaxException {
tearDownDone = false;
conf = new HdfsConfiguration();
conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, DATA_DIR);
@@ -131,11 +132,12 @@ public class TestBlockRecovery {
conf.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
FileSystem.setDefaultUri(conf,
"hdfs://" + NN_ADDR.getHostName() + ":" + NN_ADDR.getPort());
- ArrayList<File> dirs = new ArrayList<File>();
+ ArrayList<StorageLocation> locations = new ArrayList<StorageLocation>();
File dataDir = new File(DATA_DIR);
FileUtil.fullyDelete(dataDir);
dataDir.mkdirs();
- dirs.add(dataDir);
+ StorageLocation location = StorageLocation.parse(dataDir.getPath());
+ locations.add(location);
final DatanodeProtocolClientSideTranslatorPB namenode =
mock(DatanodeProtocolClientSideTranslatorPB.class);
@@ -163,7 +165,7 @@ public class TestBlockRecovery {
new DatanodeCommand[0],
new NNHAStatusHeartbeat(HAServiceState.ACTIVE, 1)));
- dn = new DataNode(conf, dirs, null) {
+ dn = new DataNode(conf, locations, null) {
@Override
DatanodeProtocolClientSideTranslatorPB connectToNN(
InetSocketAddress nnAddr) throws IOException {
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java Thu Dec 12 07:17:51 2013
@@ -265,7 +265,7 @@ public class TestBlockReplacement {
// sendRequest
DataOutputStream out = new DataOutputStream(sock.getOutputStream());
new Sender(out).replaceBlock(block, BlockTokenSecretManager.DUMMY_TOKEN,
- source.getStorageID(), sourceProxy);
+ source.getDatanodeUuid(), sourceProxy);
out.flush();
// receiveResponse
DataInputStream reply = new DataInputStream(sock.getInputStream());
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java Thu Dec 12 07:17:51 2013
@@ -17,7 +17,9 @@
*/
package org.apache.hadoop.hdfs.server.datanode;
+import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import java.io.File;
@@ -25,6 +27,7 @@ import java.io.FilenameFilter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.Random;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeoutException;
@@ -88,7 +91,7 @@ public class TestBlockReport {
private MiniDFSCluster cluster;
private DistributedFileSystem fs;
- Random rand = new Random(RAND_LIMIT);
+ private static Random rand = new Random(RAND_LIMIT);
private static Configuration conf;
@@ -112,6 +115,48 @@ public class TestBlockReport {
cluster.shutdown();
}
+ // Generate a block report, optionally corrupting the generation
+ // stamp and/or length of one block.
+ private static StorageBlockReport[] getBlockReports(
+ DataNode dn, String bpid, boolean corruptOneBlockGs,
+ boolean corruptOneBlockLen) {
+ Map<DatanodeStorage, BlockListAsLongs> perVolumeBlockLists =
+ dn.getFSDataset().getBlockReports(bpid);
+
+ // Send block report
+ StorageBlockReport[] reports =
+ new StorageBlockReport[perVolumeBlockLists.size()];
+ boolean corruptedGs = false;
+ boolean corruptedLen = false;
+
+ int reportIndex = 0;
+ for(Map.Entry<DatanodeStorage, BlockListAsLongs> kvPair : perVolumeBlockLists.entrySet()) {
+ DatanodeStorage dnStorage = kvPair.getKey();
+ BlockListAsLongs blockList = kvPair.getValue();
+
+ // Walk the list of blocks until we find one each to corrupt the
+ // generation stamp and length, if so requested.
+ for (int i = 0; i < blockList.getNumberOfBlocks(); ++i) {
+ if (corruptOneBlockGs && !corruptedGs) {
+ blockList.corruptBlockGSForTesting(i, rand);
+ LOG.info("Corrupted the GS for block ID " + i);
+ corruptedGs = true;
+ } else if (corruptOneBlockLen && !corruptedLen) {
+ blockList.corruptBlockLengthForTesting(i, rand);
+ LOG.info("Corrupted the length for block ID " + i);
+ corruptedLen = true;
+ } else {
+ break;
+ }
+ }
+
+ reports[reportIndex++] =
+ new StorageBlockReport(dnStorage, blockList.getBlockListAsLongs());
+ }
+
+ return reports;
+ }
+
/**
* Test write a file, verifies and closes it. Then the length of the blocks
* are messed up and BlockReport is forced.
@@ -152,10 +197,8 @@ public class TestBlockReport {
DataNode dn = cluster.getDataNodes().get(DN_N0);
String poolId = cluster.getNamesystem().getBlockPoolId();
DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId);
- StorageBlockReport[] report = { new StorageBlockReport(
- new DatanodeStorage(dnR.getStorageID()),
- new BlockListAsLongs(blocks, null).getBlockListAsLongs()) };
- cluster.getNameNodeRpc().blockReport(dnR, poolId, report);
+ StorageBlockReport[] reports = getBlockReports(dn, poolId, false, false);
+ cluster.getNameNodeRpc().blockReport(dnR, poolId, reports);
List<LocatedBlock> blocksAfterReport =
DFSTestUtil.getAllBlocks(fs.open(filePath));
@@ -210,7 +253,6 @@ public class TestBlockReport {
for (Integer aRemovedIndex : removedIndex) {
blocks2Remove.add(lBlocks.get(aRemovedIndex).getBlock());
}
- ArrayList<Block> blocks = locatedToBlocks(lBlocks, removedIndex);
if(LOG.isDebugEnabled()) {
LOG.debug("Number of blocks allocated " + lBlocks.size());
@@ -224,8 +266,11 @@ public class TestBlockReport {
for (File f : findAllFiles(dataDir,
new MyFileFilter(b.getBlockName(), true))) {
DataNodeTestUtils.getFSDataset(dn0).unfinalizeBlock(b);
- if (!f.delete())
+ if (!f.delete()) {
LOG.warn("Couldn't delete " + b.getBlockName());
+ } else {
+ LOG.debug("Deleted file " + f.toString());
+ }
}
}
@@ -234,10 +279,8 @@ public class TestBlockReport {
// all blocks belong to the same file, hence same BP
String poolId = cluster.getNamesystem().getBlockPoolId();
DatanodeRegistration dnR = dn0.getDNRegistrationForBP(poolId);
- StorageBlockReport[] report = { new StorageBlockReport(
- new DatanodeStorage(dnR.getStorageID()),
- new BlockListAsLongs(blocks, null).getBlockListAsLongs()) };
- cluster.getNameNodeRpc().blockReport(dnR, poolId, report);
+ StorageBlockReport[] reports = getBlockReports(dn0, poolId, false, false);
+ cluster.getNameNodeRpc().blockReport(dnR, poolId, reports);
BlockManagerTestUtil.getComputedDatanodeWork(cluster.getNamesystem()
.getBlockManager());
@@ -252,9 +295,8 @@ public class TestBlockReport {
/**
- * Test writes a file and closes it. Then test finds a block
- * and changes its GS to be < of original one.
- * New empty block is added to the list of blocks.
+ * Test writes a file and closes it.
+ * Block reported is generated with a bad GS for a single block.
* Block report is forced and the check for # of corrupted blocks is performed.
*
* @throws IOException in case of an error
@@ -263,50 +305,64 @@ public class TestBlockReport {
public void blockReport_03() throws IOException {
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path filePath = new Path("/" + METHOD_NAME + ".dat");
-
- ArrayList<Block> blocks =
- prepareForRide(filePath, METHOD_NAME, FILE_SIZE);
-
- // The block with modified GS won't be found. Has to be deleted
- blocks.get(0).setGenerationStamp(rand.nextLong());
- // This new block is unknown to NN and will be mark for deletion.
- blocks.add(new Block());
+ ArrayList<Block> blocks = writeFile(METHOD_NAME, FILE_SIZE, filePath);
// all blocks belong to the same file, hence same BP
DataNode dn = cluster.getDataNodes().get(DN_N0);
String poolId = cluster.getNamesystem().getBlockPoolId();
DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId);
- StorageBlockReport[] report = { new StorageBlockReport(
- new DatanodeStorage(dnR.getStorageID()),
- new BlockListAsLongs(blocks, null).getBlockListAsLongs()) };
+ StorageBlockReport[] reports = getBlockReports(dn, poolId, true, false);
DatanodeCommand dnCmd =
- cluster.getNameNodeRpc().blockReport(dnR, poolId, report);
+ cluster.getNameNodeRpc().blockReport(dnR, poolId, reports);
if(LOG.isDebugEnabled()) {
LOG.debug("Got the command: " + dnCmd);
}
printStats();
- assertEquals("Wrong number of CorruptedReplica+PendingDeletion " +
- "blocks is found", 2,
- cluster.getNamesystem().getCorruptReplicaBlocks() +
- cluster.getNamesystem().getPendingDeletionBlocks());
+ assertThat("Wrong number of corrupt blocks",
+ cluster.getNamesystem().getCorruptReplicaBlocks(), is(1L));
+ assertThat("Wrong number of PendingDeletion blocks",
+ cluster.getNamesystem().getPendingDeletionBlocks(), is(0L));
}
/**
- * This test isn't a representative case for BlockReport
- * The empty method is going to be left here to keep the naming
- * of the test plan in synch with the actual implementation
+ * Test writes a file and closes it.
+ * Block reported is generated with an extra block.
+ * Block report is forced and the check for # of pendingdeletion
+ * blocks is performed.
+ *
+ * @throws IOException in case of an error
*/
- public void blockReport_04() {
- }
+ @Test
+ public void blockReport_04() throws IOException {
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ Path filePath = new Path("/" + METHOD_NAME + ".dat");
+ DFSTestUtil.createFile(fs, filePath,
+ FILE_SIZE, REPL_FACTOR, rand.nextLong());
+
- // Client requests new block from NN. The test corrupts this very block
- // and forces new block report.
- // The test case isn't specific for BlockReport because it relies on
- // BlockScanner which is out of scope of this test
- // Keeping the name to be in synch with the test plan
- //
- public void blockReport_05() {
+ DataNode dn = cluster.getDataNodes().get(DN_N0);
+ // all blocks belong to the same file, hence same BP
+ String poolId = cluster.getNamesystem().getBlockPoolId();
+
+ // Create a bogus new block which will not be present on the namenode.
+ ExtendedBlock b = new ExtendedBlock(
+ poolId, rand.nextLong(), 1024L, rand.nextLong());
+ dn.getFSDataset().createRbw(b);
+
+ DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId);
+ StorageBlockReport[] reports = getBlockReports(dn, poolId, false, false);
+ DatanodeCommand dnCmd =
+ cluster.getNameNodeRpc().blockReport(dnR, poolId, reports);
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Got the command: " + dnCmd);
+ }
+ printStats();
+
+ assertThat("Wrong number of corrupt blocks",
+ cluster.getNamesystem().getCorruptReplicaBlocks(), is(0L));
+ assertThat("Wrong number of PendingDeletion blocks",
+ cluster.getNamesystem().getPendingDeletionBlocks(), is(1L));
}
/**
@@ -323,17 +379,15 @@ public class TestBlockReport {
Path filePath = new Path("/" + METHOD_NAME + ".dat");
final int DN_N1 = DN_N0 + 1;
- ArrayList<Block> blocks = writeFile(METHOD_NAME, FILE_SIZE, filePath);
+ writeFile(METHOD_NAME, FILE_SIZE, filePath);
startDNandWait(filePath, true);
- // all blocks belong to the same file, hence same BP
+ // all blocks belong to the same file, hence same BP
DataNode dn = cluster.getDataNodes().get(DN_N1);
String poolId = cluster.getNamesystem().getBlockPoolId();
DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId);
- StorageBlockReport[] report = { new StorageBlockReport(
- new DatanodeStorage(dnR.getStorageID()),
- new BlockListAsLongs(blocks, null).getBlockListAsLongs()) };
- cluster.getNameNodeRpc().blockReport(dnR, poolId, report);
+ StorageBlockReport[] reports = getBlockReports(dn, poolId, false, false);
+ cluster.getNameNodeRpc().blockReport(dnR, poolId, reports);
printStats();
assertEquals("Wrong number of PendingReplication Blocks",
0, cluster.getNamesystem().getUnderReplicatedBlocks());
@@ -353,68 +407,40 @@ public class TestBlockReport {
* @throws IOException in case of an error
*/
@Test
- // Currently this test is failing as expected 'cause the correct behavior is
- // not yet implemented (9/15/09)
public void blockReport_07() throws Exception {
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path filePath = new Path("/" + METHOD_NAME + ".dat");
final int DN_N1 = DN_N0 + 1;
// write file and start second node to be "older" than the original
- ArrayList<Block> blocks = writeFile(METHOD_NAME, FILE_SIZE, filePath);
+ writeFile(METHOD_NAME, FILE_SIZE, filePath);
startDNandWait(filePath, true);
- int randIndex = rand.nextInt(blocks.size());
- // Get a block and screw its GS
- Block corruptedBlock = blocks.get(randIndex);
- String secondNode = cluster.getDataNodes().get(DN_N1).getStorageId();
- if(LOG.isDebugEnabled()) {
- LOG.debug("Working with " + secondNode);
- LOG.debug("BlockGS before " + blocks.get(randIndex).getGenerationStamp());
- }
- corruptBlockGS(corruptedBlock);
- if(LOG.isDebugEnabled()) {
- LOG.debug("BlockGS after " + blocks.get(randIndex).getGenerationStamp());
- LOG.debug("Done corrupting GS of " + corruptedBlock.getBlockName());
- }
// all blocks belong to the same file, hence same BP
DataNode dn = cluster.getDataNodes().get(DN_N1);
String poolId = cluster.getNamesystem().getBlockPoolId();
DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId);
- StorageBlockReport[] report = { new StorageBlockReport(
- new DatanodeStorage(dnR.getStorageID()),
- new BlockListAsLongs(blocks, null).getBlockListAsLongs()) };
- cluster.getNameNodeRpc().blockReport(dnR, poolId, report);
+ StorageBlockReport[] reports = getBlockReports(dn, poolId, true, false);
+ cluster.getNameNodeRpc().blockReport(dnR, poolId, reports);
printStats();
- assertEquals("Wrong number of Corrupted blocks",
- 1, cluster.getNamesystem().getCorruptReplicaBlocks() +
-// the following might have to be added into the equation if
-// the same block could be in two different states at the same time
-// and then the expected number of has to be changed to '2'
-// cluster.getNamesystem().getPendingReplicationBlocks() +
- cluster.getNamesystem().getPendingDeletionBlocks());
-
- // Get another block and screw its length to be less than original
- if (randIndex == 0)
- randIndex++;
- else
- randIndex--;
- corruptedBlock = blocks.get(randIndex);
- corruptBlockLen(corruptedBlock);
- if(LOG.isDebugEnabled()) {
- LOG.debug("Done corrupting length of " + corruptedBlock.getBlockName());
- }
-
- report[0] = new StorageBlockReport(
- new DatanodeStorage(dnR.getStorageID()),
- new BlockListAsLongs(blocks, null).getBlockListAsLongs());
- cluster.getNameNodeRpc().blockReport(dnR, poolId, report);
+
+ assertThat("Wrong number of corrupt blocks",
+ cluster.getNamesystem().getCorruptReplicaBlocks(), is(1L));
+ assertThat("Wrong number of PendingDeletion blocks",
+ cluster.getNamesystem().getPendingDeletionBlocks(), is(0L));
+ assertThat("Wrong number of PendingReplication blocks",
+ cluster.getNamesystem().getPendingReplicationBlocks(), is(0L));
+
+ reports = getBlockReports(dn, poolId, true, true);
+ cluster.getNameNodeRpc().blockReport(dnR, poolId, reports);
printStats();
- assertEquals("Wrong number of Corrupted blocks",
- 2, cluster.getNamesystem().getCorruptReplicaBlocks() +
- cluster.getNamesystem().getPendingReplicationBlocks() +
- cluster.getNamesystem().getPendingDeletionBlocks());
+ assertThat("Wrong number of corrupt blocks",
+ cluster.getNamesystem().getCorruptReplicaBlocks(), is(2L));
+ assertThat("Wrong number of PendingDeletion blocks",
+ cluster.getNamesystem().getPendingDeletionBlocks(), is(0L));
+ assertThat("Wrong number of PendingReplication blocks",
+ cluster.getNamesystem().getPendingReplicationBlocks(), is(0L));
printStats();
@@ -457,9 +483,7 @@ public class TestBlockReport {
DataNode dn = cluster.getDataNodes().get(DN_N1);
String poolId = cluster.getNamesystem().getBlockPoolId();
DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId);
- StorageBlockReport[] report = { new StorageBlockReport(
- new DatanodeStorage(dnR.getStorageID()),
- new BlockListAsLongs(blocks, null).getBlockListAsLongs()) };
+ StorageBlockReport[] report = getBlockReports(dn, poolId, false, false);
cluster.getNameNodeRpc().blockReport(dnR, poolId, report);
printStats();
assertEquals("Wrong number of PendingReplication blocks",
@@ -490,14 +514,11 @@ public class TestBlockReport {
// write file and start second node to be "older" than the original
try {
- ArrayList<Block> blocks =
- writeFile(METHOD_NAME, 12 * bytesChkSum, filePath);
+ writeFile(METHOD_NAME, 12 * bytesChkSum, filePath);
Block bl = findBlock(filePath, 12 * bytesChkSum);
BlockChecker bc = new BlockChecker(filePath);
bc.start();
- corruptBlockGS(bl);
- corruptBlockLen(bl);
waitForTempReplica(bl, DN_N1);
@@ -505,9 +526,7 @@ public class TestBlockReport {
DataNode dn = cluster.getDataNodes().get(DN_N1);
String poolId = cluster.getNamesystem().getBlockPoolId();
DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId);
- StorageBlockReport[] report = { new StorageBlockReport(
- new DatanodeStorage(dnR.getStorageID()),
- new BlockListAsLongs(blocks, null).getBlockListAsLongs()) };
+ StorageBlockReport[] report = getBlockReports(dn, poolId, true, true);
cluster.getNameNodeRpc().blockReport(dnR, poolId, report);
printStats();
assertEquals("Wrong number of PendingReplication blocks",
@@ -783,38 +802,6 @@ public class TestBlockReport {
((Log4JLogger) TestBlockReport.LOG).getLogger().setLevel(Level.ALL);
}
- private void corruptBlockLen(final Block block)
- throws IOException {
- if (block == null) {
- throw new IOException("Block isn't suppose to be null");
- }
- long oldLen = block.getNumBytes();
- long newLen = oldLen - rand.nextLong();
- assertTrue("Old and new length shouldn't be the same",
- block.getNumBytes() != newLen);
- block.setNumBytes(newLen);
- if(LOG.isDebugEnabled()) {
- LOG.debug("Length of " + block.getBlockName() +
- " is changed to " + newLen + " from " + oldLen);
- }
- }
-
- private void corruptBlockGS(final Block block)
- throws IOException {
- if (block == null) {
- throw new IOException("Block isn't suppose to be null");
- }
- long oldGS = block.getGenerationStamp();
- long newGS = oldGS - rand.nextLong();
- assertTrue("Old and new GS shouldn't be the same",
- block.getGenerationStamp() != newGS);
- block.setGenerationStamp(newGS);
- if(LOG.isDebugEnabled()) {
- LOG.debug("Generation stamp of " + block.getBlockName() +
- " is changed to " + block.getGenerationStamp() + " from " + oldGS);
- }
- }
-
private Block findBlock(Path path, long size) throws IOException {
Block ret;
List<LocatedBlock> lbs =
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java Thu Dec 12 07:17:51 2013
@@ -19,12 +19,14 @@
package org.apache.hadoop.hdfs.server.datanode;
import java.io.*;
-import java.net.URI;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
+import java.util.*;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.StorageType;
import org.junit.Test;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY;
+import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.*;
import static org.mockito.Mockito.*;
@@ -34,19 +36,69 @@ import org.apache.hadoop.hdfs.server.dat
public class TestDataDirs {
- @Test (timeout = 10000)
- public void testGetDataDirsFromURIs() throws Throwable {
+ @Test (timeout = 30000)
+ public void testDataDirParsing() throws Throwable {
+ Configuration conf = new Configuration();
+ List<StorageLocation> locations;
+ File dir0 = new File("/dir0");
+ File dir1 = new File("/dir1");
+ File dir2 = new File("/dir2");
+ File dir3 = new File("/dir3");
+
+ // Verify that a valid string is correctly parsed, and that storage
+ // type is not case-sensitive
+ String locations1 = "[disk]/dir0,[DISK]/dir1,[sSd]/dir2,[disK]/dir3";
+ conf.set(DFS_DATANODE_DATA_DIR_KEY, locations1);
+ locations = DataNode.getStorageLocations(conf);
+ assertThat(locations.size(), is(4));
+ assertThat(locations.get(0).getStorageType(), is(StorageType.DISK));
+ assertThat(locations.get(0).getUri(), is(dir0.toURI()));
+ assertThat(locations.get(1).getStorageType(), is(StorageType.DISK));
+ assertThat(locations.get(1).getUri(), is(dir1.toURI()));
+ assertThat(locations.get(2).getStorageType(), is(StorageType.SSD));
+ assertThat(locations.get(2).getUri(), is(dir2.toURI()));
+ assertThat(locations.get(3).getStorageType(), is(StorageType.DISK));
+ assertThat(locations.get(3).getUri(), is(dir3.toURI()));
+
+ // Verify that an unrecognized storage type result in an exception.
+ String locations2 = "[BadMediaType]/dir0,[ssd]/dir1,[disk]/dir2";
+ conf.set(DFS_DATANODE_DATA_DIR_KEY, locations2);
+ try {
+ locations = DataNode.getStorageLocations(conf);
+ fail();
+ } catch(IllegalArgumentException iae) {
+ DataNode.LOG.info("The exception is expected.", iae);
+ }
+
+ // Assert that a string with no storage type specified is
+ // correctly parsed and the default storage type is picked up.
+ String locations3 = "/dir0,/dir1";
+ conf.set(DFS_DATANODE_DATA_DIR_KEY, locations3);
+ locations = DataNode.getStorageLocations(conf);
+ assertThat(locations.size(), is(2));
+ assertThat(locations.get(0).getStorageType(), is(StorageType.DISK));
+ assertThat(locations.get(0).getUri(), is(dir0.toURI()));
+ assertThat(locations.get(1).getStorageType(), is(StorageType.DISK));
+ assertThat(locations.get(1).getUri(), is(dir1.toURI()));
+ }
+
+ @Test (timeout = 30000)
+ public void testDataDirValidation() throws Throwable {
DataNodeDiskChecker diskChecker = mock(DataNodeDiskChecker.class);
doThrow(new IOException()).doThrow(new IOException()).doNothing()
.when(diskChecker).checkDir(any(LocalFileSystem.class), any(Path.class));
LocalFileSystem fs = mock(LocalFileSystem.class);
- Collection<URI> uris = Arrays.asList(new URI("file:/p1/"),
- new URI("file:/p2/"), new URI("file:/p3/"));
+ AbstractList<StorageLocation> locations = new ArrayList<StorageLocation>();
- List<File> dirs = DataNode.getDataDirsFromURIs(uris, fs, diskChecker);
- assertEquals("number of valid data dirs", 1, dirs.size());
- String validDir = dirs.iterator().next().getPath();
- assertEquals("p3 should be valid", new File("/p3").getPath(), validDir);
+ locations.add(StorageLocation.parse("file:/p1/"));
+ locations.add(StorageLocation.parse("file:/p2/"));
+ locations.add(StorageLocation.parse("file:/p3/"));
+
+ List<StorageLocation> checkedLocations =
+ DataNode.checkStorageLocations(locations, fs, diskChecker);
+ assertEquals("number of valid data dirs", 1, checkedLocations.size());
+ String validDir = checkedLocations.iterator().next().getFile().getPath();
+ assertThat("p3 should be valid", new File("/p3/").getPath(), is(validDir));
}
}
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java Thu Dec 12 07:17:51 2013
@@ -163,7 +163,7 @@ public class TestDataNodeMultipleRegistr
for (BPOfferService bpos : dn.getAllBpOs()) {
LOG.info("reg: bpid=" + "; name=" + bpos.bpRegistration + "; sid="
- + bpos.bpRegistration.getStorageID() + "; nna=" +
+ + bpos.bpRegistration.getDatanodeUuid() + "; nna=" +
getNNSocketAddress(bpos));
}
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java Thu Dec 12 07:17:51 2013
@@ -42,11 +42,13 @@ import org.apache.hadoop.hdfs.HdfsConfig
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.net.TcpPeerServer;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
@@ -151,13 +153,23 @@ public class TestDataNodeVolumeFailure {
DataNode dn = cluster.getDataNodes().get(1); //corresponds to dir data3
String bpid = cluster.getNamesystem().getBlockPoolId();
DatanodeRegistration dnR = dn.getDNRegistrationForBP(bpid);
- final StorageBlockReport[] report = {
- new StorageBlockReport(
- new DatanodeStorage(dnR.getStorageID()),
- DataNodeTestUtils.getFSDataset(dn).getBlockReport(bpid
- ).getBlockListAsLongs())
- };
- cluster.getNameNodeRpc().blockReport(dnR, bpid, report);
+
+ Map<DatanodeStorage, BlockListAsLongs> perVolumeBlockLists =
+ dn.getFSDataset().getBlockReports(bpid);
+
+ // Send block report
+ StorageBlockReport[] reports =
+ new StorageBlockReport[perVolumeBlockLists.size()];
+
+ int reportIndex = 0;
+ for(Map.Entry<DatanodeStorage, BlockListAsLongs> kvPair : perVolumeBlockLists.entrySet()) {
+ DatanodeStorage dnStorage = kvPair.getKey();
+ BlockListAsLongs blockList = kvPair.getValue();
+ reports[reportIndex++] =
+ new StorageBlockReport(dnStorage, blockList.getBlockListAsLongs());
+ }
+
+ cluster.getNameNodeRpc().blockReport(dnR, bpid, reports);
// verify number of blocks and files...
verify(filename, filesize);
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java Thu Dec 12 07:17:51 2013
@@ -40,6 +40,7 @@ import org.apache.hadoop.hdfs.DFSConfigK
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.common.GenerationStamp;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
@@ -406,6 +407,16 @@ public class TestDirectoryScanner {
public File getFinalizedDir(String bpid) throws IOException {
return new File("/base/current/" + bpid + "/finalized");
}
+
+ @Override
+ public StorageType getStorageType() {
+ return StorageType.DEFAULT;
+ }
+
+ @Override
+ public String getStorageID() {
+ return "";
+ }
}
private final static TestFsVolumeSpi TEST_VOLUME = new TestFsVolumeSpi();
@@ -436,7 +447,7 @@ public class TestDirectoryScanner {
void testScanInfoObject(long blockId) throws Exception {
DirectoryScanner.ScanInfo scanInfo =
- new DirectoryScanner.ScanInfo(blockId);
+ new DirectoryScanner.ScanInfo(blockId, null, null, null);
assertEquals(blockId, scanInfo.getBlockId());
assertNull(scanInfo.getBlockFile());
assertNull(scanInfo.getMetaFile());
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java Thu Dec 12 07:17:51 2013
@@ -311,7 +311,7 @@ public class TestSimulatedFSDataset {
}
private SimulatedFSDataset getSimulatedFSDataset() {
- SimulatedFSDataset fsdataset = new SimulatedFSDataset(null, null, conf);
+ SimulatedFSDataset fsdataset = new SimulatedFSDataset(null, conf);
fsdataset.addBlockPool(bpid, conf);
return fsdataset;
}
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java Thu Dec 12 07:17:51 2013
@@ -17,6 +17,14 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.EnumSet;
+import java.util.List;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
@@ -25,28 +33,40 @@ import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.hdfs.protocol.*;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataStorage;
-import org.apache.hadoop.hdfs.server.protocol.*;
+import org.apache.hadoop.hdfs.server.protocol.BlockCommand;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
+import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
+import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
+import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
+import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport;
+import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
+import org.apache.hadoop.hdfs.server.protocol.StorageReport;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.net.DNS;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.security.Groups;
-import org.apache.hadoop.util.*;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Time;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.util.VersionInfo;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.EnumSet;
-import java.util.List;
-
/**
* Main class for a series of name-node benchmarks.
*
@@ -817,17 +837,16 @@ public class NNThroughputBenchmark imple
dnRegistration = new DatanodeRegistration(
new DatanodeID(DNS.getDefaultIP("default"),
DNS.getDefaultHost("default", "default"),
- "", getNodePort(dnIdx),
+ DataNode.generateUuid(), getNodePort(dnIdx),
DFSConfigKeys.DFS_DATANODE_HTTP_DEFAULT_PORT,
DFSConfigKeys.DFS_DATANODE_HTTPS_DEFAULT_PORT,
DFSConfigKeys.DFS_DATANODE_IPC_DEFAULT_PORT),
- new DataStorage(nsInfo, ""),
+ new DataStorage(nsInfo),
new ExportedBlockKeys(), VersionInfo.getVersion());
- DataNode.setNewStorageID(dnRegistration);
// register datanode
dnRegistration = nameNodeProto.registerDatanode(dnRegistration);
//first block reports
- storage = new DatanodeStorage(dnRegistration.getStorageID());
+ storage = new DatanodeStorage(dnRegistration.getDatanodeUuid());
final StorageBlockReport[] reports = {
new StorageBlockReport(storage,
new BlockListAsLongs(null, null).getBlockListAsLongs())
@@ -843,7 +862,7 @@ public class NNThroughputBenchmark imple
void sendHeartbeat() throws IOException {
// register datanode
// TODO:FEDERATION currently a single block pool is supported
- StorageReport[] rep = { new StorageReport(dnRegistration.getStorageID(),
+ StorageReport[] rep = { new StorageReport(dnRegistration.getDatanodeUuid(),
false, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED) };
DatanodeCommand[] cmds = nameNodeProto.sendHeartbeat(dnRegistration, rep,
0L, 0L, 0, 0, 0).getCommands();
@@ -890,7 +909,7 @@ public class NNThroughputBenchmark imple
@SuppressWarnings("unused") // keep it for future blockReceived benchmark
int replicateBlocks() throws IOException {
// register datanode
- StorageReport[] rep = { new StorageReport(dnRegistration.getStorageID(),
+ StorageReport[] rep = { new StorageReport(dnRegistration.getDatanodeUuid(),
false, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED) };
DatanodeCommand[] cmds = nameNodeProto.sendHeartbeat(dnRegistration,
rep, 0L, 0L, 0, 0, 0).getCommands();
@@ -920,14 +939,14 @@ public class NNThroughputBenchmark imple
DatanodeInfo dnInfo = blockTargets[t];
DatanodeRegistration receivedDNReg;
receivedDNReg = new DatanodeRegistration(dnInfo,
- new DataStorage(nsInfo, dnInfo.getStorageID()),
+ new DataStorage(nsInfo),
new ExportedBlockKeys(), VersionInfo.getVersion());
ReceivedDeletedBlockInfo[] rdBlocks = {
new ReceivedDeletedBlockInfo(
blocks[i], ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK,
null) };
StorageReceivedDeletedBlocks[] report = { new StorageReceivedDeletedBlocks(
- receivedDNReg.getStorageID(), rdBlocks) };
+ receivedDNReg.getDatanodeUuid(), rdBlocks) };
nameNodeProto.blockReceivedAndDeleted(receivedDNReg, nameNode
.getNamesystem().getBlockPoolId(), report);
}
@@ -1050,7 +1069,7 @@ public class NNThroughputBenchmark imple
loc.getBlock().getLocalBlock(),
ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, null) };
StorageReceivedDeletedBlocks[] report = { new StorageReceivedDeletedBlocks(
- datanodes[dnIdx].dnRegistration.getStorageID(), rdBlocks) };
+ datanodes[dnIdx].dnRegistration.getDatanodeUuid(), rdBlocks) };
nameNodeProto.blockReceivedAndDeleted(datanodes[dnIdx].dnRegistration, loc
.getBlock().getBlockPoolId(), report);
}
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java Thu Dec 12 07:17:51 2013
@@ -32,6 +32,7 @@ import org.apache.hadoop.hdfs.protocol.D
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.MkdirOp;
@@ -111,8 +112,8 @@ public class NameNodeAdapter {
public static HeartbeatResponse sendHeartBeat(DatanodeRegistration nodeReg,
DatanodeDescriptor dd, FSNamesystem namesystem) throws IOException {
- return namesystem.handleHeartbeat(nodeReg, dd.getCapacity(),
- dd.getDfsUsed(), dd.getRemaining(), dd.getBlockPoolUsed(),
+ return namesystem.handleHeartbeat(nodeReg,
+ BlockManagerTestUtil.getStorageReportsForDatanode(dd),
dd.getCacheCapacity(), dd.getCacheRemaining(), 0, 0, 0);
}
@@ -243,3 +244,4 @@ public class NameNodeAdapter {
return NNStorage.getInProgressEditsFile(sd, startTxId);
}
}
+
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java Thu Dec 12 07:17:51 2013
@@ -39,6 +39,7 @@ import org.apache.hadoop.hdfs.protocol.L
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.net.Node;
@@ -99,13 +100,13 @@ public class TestAddBlockRetry {
bmField.setAccessible(true);
bmField.set(ns, spyBM);
- doAnswer(new Answer<DatanodeDescriptor[]>() {
+ doAnswer(new Answer<DatanodeStorageInfo[]>() {
@Override
- public DatanodeDescriptor[] answer(InvocationOnMock invocation)
+ public DatanodeStorageInfo[] answer(InvocationOnMock invocation)
throws Throwable {
LOG.info("chooseTarget for " + src);
- DatanodeDescriptor[] ret =
- (DatanodeDescriptor[]) invocation.callRealMethod();
+ DatanodeStorageInfo[] ret =
+ (DatanodeStorageInfo[]) invocation.callRealMethod();
count++;
if(count == 1) { // run second addBlock()
LOG.info("Starting second addBlock for " + src);
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java?rev=1550363&r1=1550362&r2=1550363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java Thu Dec 12 07:17:51 2013
@@ -24,7 +24,7 @@ import org.apache.hadoop.hdfs.protocol.D
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
-import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.junit.Test;
@@ -47,7 +47,7 @@ public class TestCommitBlockSynchronizat
throws IOException {
Configuration conf = new Configuration();
FSImage image = new FSImage(conf);
- DatanodeDescriptor[] targets = new DatanodeDescriptor[0];
+ final DatanodeStorageInfo[] targets = {};
FSNamesystem namesystem = new FSNamesystem(conf, image);
FSNamesystem namesystemSpy = spy(namesystem);