You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sz...@apache.org on 2012/10/19 04:28:07 UTC
svn commit: r1399950 [22/27] - in
/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project: ./
hadoop-hdfs-httpfs/ hadoop-hdfs-httpfs/dev-support/
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/
hadoop-hdfs-httpfs/src/main/java/org/apac...
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java?rev=1399950&r1=1399949&r2=1399950&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java Fri Oct 19 02:25:55 2012
@@ -17,21 +17,25 @@
*/
package org.apache.hadoop.hdfs.server.blockmanagement;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
import java.util.ArrayList;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.common.GenerationStamp;
-
-import junit.framework.TestCase;
+import org.junit.Test;
/**
* This class tests that methods in DatanodeDescriptor
*/
-public class TestDatanodeDescriptor extends TestCase {
+public class TestDatanodeDescriptor {
/**
* Test that getInvalidateBlocks observes the maxlimit.
*/
+ @Test
public void testGetInvalidateBlocks() throws Exception {
final int MAX_BLOCKS = 10;
final int REMAINING_BLOCKS = 2;
@@ -49,6 +53,7 @@ public class TestDatanodeDescriptor exte
assertEquals(bc.length, REMAINING_BLOCKS);
}
+ @Test
public void testBlocksCounter() throws Exception {
DatanodeDescriptor dd = DFSTestUtil.getLocalDatanodeDescriptor();
assertEquals(0, dd.numBlocks());
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java?rev=1399950&r1=1399949&r2=1399950&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java Fri Oct 19 02:25:55 2012
@@ -17,9 +17,9 @@
*/
package org.apache.hadoop.hdfs.server.blockmanagement;
-import java.util.ArrayList;
+import static org.junit.Assert.assertEquals;
-import junit.framework.TestCase;
+import java.util.ArrayList;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -34,17 +34,19 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
+import org.junit.Test;
/**
* Test if FSNamesystem handles heartbeat right
*/
-public class TestHeartbeatHandling extends TestCase {
+public class TestHeartbeatHandling {
/**
* Test if
* {@link FSNamesystem#handleHeartbeat}
* can pick up replication and/or invalidate requests and observes the max
* limit
*/
+ @Test
public void testHeartbeat() throws Exception {
final Configuration conf = new HdfsConfiguration();
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHost2NodesMap.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHost2NodesMap.java?rev=1399950&r1=1399949&r2=1399950&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHost2NodesMap.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHost2NodesMap.java Fri Oct 19 02:25:55 2012
@@ -18,13 +18,15 @@
package org.apache.hadoop.hdfs.server.blockmanagement;
-import org.apache.hadoop.hdfs.DFSTestUtil;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import org.apache.hadoop.hdfs.DFSTestUtil;
import org.junit.Before;
import org.junit.Test;
-import static org.junit.Assert.*;
-
public class TestHost2NodesMap {
private Host2NodesMap map = new Host2NodesMap();
private DatanodeDescriptor dataNodes[];
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java?rev=1399950&r1=1399949&r2=1399950&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java Fri Oct 19 02:25:55 2012
@@ -17,12 +17,12 @@
*/
package org.apache.hadoop.hdfs.server.blockmanagement;
+import static org.junit.Assert.assertTrue;
+
import java.util.Collection;
import java.util.Iterator;
import java.util.concurrent.TimeoutException;
-import junit.framework.TestCase;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -32,10 +32,9 @@ import org.apache.hadoop.hdfs.MiniDFSClu
import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
-import org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager;
-import org.apache.hadoop.hdfs.server.blockmanagement.NumberReplicas;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.util.Time;
+import org.junit.Test;
/**
* Test if live nodes count per node is correct
@@ -44,7 +43,7 @@ import org.apache.hadoop.hdfs.server.nam
* Two of the "while" loops below use "busy wait"
* because they are detecting transient states.
*/
-public class TestNodeCount extends TestCase {
+public class TestNodeCount {
final short REPLICATION_FACTOR = (short)2;
final long TIMEOUT = 20000L;
long timeout = 0;
@@ -52,6 +51,7 @@ public class TestNodeCount extends TestC
Block lastBlock = null;
NumberReplicas lastNum = null;
+ @Test
public void testNodeCount() throws Exception {
// start a mini dfs cluster of 2 nodes
final Configuration conf = new HdfsConfiguration();
@@ -140,7 +140,7 @@ public class TestNodeCount extends TestC
void initializeTimeout(long timeout) {
this.timeout = timeout;
- this.failtime = System.currentTimeMillis()
+ this.failtime = Time.now()
+ ((timeout <= 0) ? Long.MAX_VALUE : timeout);
}
@@ -151,7 +151,7 @@ public class TestNodeCount extends TestC
/* check for timeout, then wait for cycleTime msec */
void checkTimeout(String testLabel, long cycleTime) throws TimeoutException {
- if (System.currentTimeMillis() > failtime) {
+ if (Time.now() > failtime) {
throw new TimeoutException("Timeout: "
+ testLabel + " for block " + lastBlock + " after " + timeout
+ " msec. Last counts: live = " + lastNum.liveReplicas()
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java?rev=1399950&r1=1399949&r2=1399950&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java Fri Oct 19 02:25:55 2012
@@ -17,13 +17,14 @@
*/
package org.apache.hadoop.hdfs.server.blockmanagement;
-import static org.apache.hadoop.hdfs.server.common.Util.now;
-import static org.junit.Assert.*;
+import static org.apache.hadoop.util.Time.now;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
import java.io.File;
import java.io.IOException;
import java.util.Collection;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -52,7 +53,7 @@ public class TestOverReplicatedBlocks {
* corrupt ones.
*/
@Test
- public void testProcesOverReplicateBlock() throws IOException {
+ public void testProcesOverReplicateBlock() throws Exception {
Configuration conf = new HdfsConfiguration();
conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 1000L);
conf.set(
@@ -140,7 +141,7 @@ public class TestOverReplicatedBlocks {
* send heartbeats.
*/
@Test
- public void testChooseReplicaToDelete() throws IOException {
+ public void testChooseReplicaToDelete() throws Exception {
MiniDFSCluster cluster = null;
FileSystem fs = null;
try {
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingDataNodeMessages.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingDataNodeMessages.java?rev=1399950&r1=1399949&r2=1399950&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingDataNodeMessages.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingDataNodeMessages.java Fri Oct 19 02:25:55 2012
@@ -17,13 +17,13 @@
*/
package org.apache.hadoop.hdfs.server.blockmanagement;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
import java.util.Queue;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.server.blockmanagement.PendingDataNodeMessages.ReportedBlockInfo;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
import org.junit.Test;
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java?rev=1399950&r1=1399949&r2=1399950&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java Fri Oct 19 02:25:55 2012
@@ -17,17 +17,19 @@
*/
package org.apache.hadoop.hdfs.server.blockmanagement;
-import junit.framework.TestCase;
-import java.lang.System;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.junit.Test;
/**
* This class tests the internals of PendingReplicationBlocks.java
*/
-public class TestPendingReplication extends TestCase {
+public class TestPendingReplication {
final static int TIMEOUT = 3; // 3 seconds
+ @Test
public void testPendingReplication() {
PendingReplicationBlocks pendingReplications;
pendingReplications = new PendingReplicationBlocks(TIMEOUT * 1000);
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestRBWBlockInvalidation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestRBWBlockInvalidation.java?rev=1399950&r1=1399949&r2=1399950&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestRBWBlockInvalidation.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestRBWBlockInvalidation.java Fri Oct 19 02:25:55 2012
@@ -17,6 +17,9 @@
*/
package org.apache.hadoop.hdfs.server.blockmanagement;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
import java.io.File;
import java.io.IOException;
@@ -34,8 +37,6 @@ import org.apache.hadoop.hdfs.server.dat
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
/**
* Test when RBW block is removed. Invalidation of the corrupted block happens
@@ -52,7 +53,7 @@ public class TestRBWBlockInvalidation {
* datanode, namenode should ask to invalidate that corrupted block and
* schedule replication for one more replica for that under replicated block.
*/
- @Test
+ @Test(timeout=60000)
public void testBlockInvalidationWhenRBWReplicaMissedInDN()
throws IOException, InterruptedException {
Configuration conf = new HdfsConfiguration();
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java?rev=1399950&r1=1399949&r2=1399950&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java Fri Oct 19 02:25:55 2012
@@ -17,7 +17,9 @@
*/
package org.apache.hadoop.hdfs.server.blockmanagement;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import java.io.File;
import java.util.ArrayList;
@@ -36,11 +38,16 @@ import org.apache.hadoop.hdfs.HdfsConfig
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.net.Node;
+import org.apache.hadoop.util.Time;
import org.junit.BeforeClass;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
public class TestReplicationPolicy {
private Random random = DFSUtil.getRandom();
@@ -51,7 +58,13 @@ public class TestReplicationPolicy {
private static BlockPlacementPolicy replicator;
private static final String filename = "/dummyfile.txt";
private static DatanodeDescriptor dataNodes[];
+ // The interval for marking a datanode as stale,
+ private static long staleInterval =
+ DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_DEFAULT;
+ @Rule
+ public ExpectedException exception = ExpectedException.none();
+
@BeforeClass
public static void setupCluster() throws Exception {
Configuration conf = new HdfsConfiguration();
@@ -70,6 +83,8 @@ public class TestReplicationPolicy {
"test.build.data", "build/test/data"), "dfs/");
conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
new File(baseDir, "name").getPath());
+ // Enable the checking for stale datanodes in the beginning
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_CHECK_STALE_DATANODE_KEY, true);
DFSTestUtil.formatNameNode(conf);
namenode = new NameNode(conf);
@@ -104,30 +119,30 @@ public class TestReplicationPolicy {
HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 4, 0); // overloaded
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename,
- 0, dataNodes[0], BLOCK_SIZE);
+ targets = replicator.chooseTarget(filename, 0, dataNodes[0],
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename,
- 1, dataNodes[0], BLOCK_SIZE);
+ targets = replicator.chooseTarget(filename, 1, dataNodes[0],
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 1);
assertEquals(targets[0], dataNodes[0]);
targets = replicator.chooseTarget(filename,
- 2, dataNodes[0], BLOCK_SIZE);
+ 2, dataNodes[0], new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 2);
assertEquals(targets[0], dataNodes[0]);
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
- targets = replicator.chooseTarget(filename,
- 3, dataNodes[0], BLOCK_SIZE);
+ targets = replicator.chooseTarget(filename, 3, dataNodes[0],
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 3);
assertEquals(targets[0], dataNodes[0]);
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
assertTrue(cluster.isOnSameRack(targets[1], targets[2]));
- targets = replicator.chooseTarget(filename,
- 4, dataNodes[0], BLOCK_SIZE);
+ targets = replicator.chooseTarget(filename, 4, dataNodes[0],
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 4);
assertEquals(targets[0], dataNodes[0]);
assertTrue(cluster.isOnSameRack(targets[1], targets[2]) ||
@@ -222,7 +237,7 @@ public class TestReplicationPolicy {
assertEquals(2, targets.length);
//make sure that the chosen node is in the target.
int i = 0;
- for(; i < targets.length && !dataNodes[2].equals(targets[i]); i++);
+ for (; i < targets.length && !dataNodes[2].equals(targets[i]); i++);
assertTrue(i < targets.length);
}
@@ -242,30 +257,30 @@ public class TestReplicationPolicy {
(HdfsConstants.MIN_BLOCKS_FOR_WRITE-1)*BLOCK_SIZE, 0L, 0, 0); // no space
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename,
- 0, dataNodes[0], BLOCK_SIZE);
+ targets = replicator.chooseTarget(filename, 0, dataNodes[0],
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename,
- 1, dataNodes[0], BLOCK_SIZE);
+ targets = replicator.chooseTarget(filename, 1, dataNodes[0],
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 1);
assertEquals(targets[0], dataNodes[1]);
- targets = replicator.chooseTarget(filename,
- 2, dataNodes[0], BLOCK_SIZE);
+ targets = replicator.chooseTarget(filename, 2, dataNodes[0],
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 2);
assertEquals(targets[0], dataNodes[1]);
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
- targets = replicator.chooseTarget(filename,
- 3, dataNodes[0], BLOCK_SIZE);
+ targets = replicator.chooseTarget(filename, 3, dataNodes[0],
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 3);
assertEquals(targets[0], dataNodes[1]);
assertTrue(cluster.isOnSameRack(targets[1], targets[2]));
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
- targets = replicator.chooseTarget(filename,
- 4, dataNodes[0], BLOCK_SIZE);
+ targets = replicator.chooseTarget(filename, 4, dataNodes[0],
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 4);
assertEquals(targets[0], dataNodes[1]);
for(int i=1; i<4; i++) {
@@ -298,23 +313,23 @@ public class TestReplicationPolicy {
}
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename,
- 0, dataNodes[0], BLOCK_SIZE);
+ targets = replicator.chooseTarget(filename, 0, dataNodes[0],
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename,
- 1, dataNodes[0], BLOCK_SIZE);
+ targets = replicator.chooseTarget(filename, 1, dataNodes[0],
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 1);
assertFalse(cluster.isOnSameRack(targets[0], dataNodes[0]));
- targets = replicator.chooseTarget(filename,
- 2, dataNodes[0], BLOCK_SIZE);
+ targets = replicator.chooseTarget(filename, 2, dataNodes[0],
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 2);
assertFalse(cluster.isOnSameRack(targets[0], dataNodes[0]));
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
- targets = replicator.chooseTarget(filename,
- 3, dataNodes[0], BLOCK_SIZE);
+ targets = replicator.chooseTarget(filename, 3, dataNodes[0],
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 3);
for(int i=0; i<3; i++) {
assertFalse(cluster.isOnSameRack(targets[i], dataNodes[0]));
@@ -343,25 +358,221 @@ public class TestReplicationPolicy {
DFSTestUtil.getDatanodeDescriptor("7.7.7.7", "/d2/r4");
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename,
- 0, writerDesc, BLOCK_SIZE);
+ targets = replicator.chooseTarget(filename, 0, writerDesc,
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 0);
-
- targets = replicator.chooseTarget(filename,
- 1, writerDesc, BLOCK_SIZE);
+
+ targets = replicator.chooseTarget(filename, 1, writerDesc,
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 1);
-
- targets = replicator.chooseTarget(filename,
- 2, writerDesc, BLOCK_SIZE);
+
+ targets = replicator.chooseTarget(filename, 2, writerDesc,
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 2);
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
-
- targets = replicator.chooseTarget(filename,
- 3, writerDesc, BLOCK_SIZE);
+
+ targets = replicator.chooseTarget(filename, 3, writerDesc,
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 3);
assertTrue(cluster.isOnSameRack(targets[1], targets[2]));
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
}
+
+ private boolean containsWithinRange(DatanodeDescriptor target,
+ DatanodeDescriptor[] nodes, int startIndex, int endIndex) {
+ assert startIndex >= 0 && startIndex < nodes.length;
+ assert endIndex >= startIndex && endIndex < nodes.length;
+ for (int i = startIndex; i <= endIndex; i++) {
+ if (nodes[i].equals(target)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Test
+ public void testChooseTargetWithStaleNodes() throws Exception {
+ // Enable avoidng writing to stale datanodes
+ namenode.getNamesystem().getBlockManager().getDatanodeManager()
+ .setAvoidStaleDataNodesForWrite(true);
+ // Set dataNodes[0] as stale
+ dataNodes[0].setLastUpdate(Time.now() - staleInterval - 1);
+
+ DatanodeDescriptor[] targets;
+ // We set the datanode[0] as stale, thus should choose datanode[1] since
+ // datanode[1] is on the same rack with datanode[0] (writer)
+ targets = replicator.chooseTarget(filename, 1, dataNodes[0],
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ assertEquals(targets.length, 1);
+ assertEquals(targets[0], dataNodes[1]);
+
+ HashMap<Node, Node> excludedNodes = new HashMap<Node, Node>();
+ excludedNodes.put(dataNodes[1], dataNodes[1]);
+ List<DatanodeDescriptor> chosenNodes = new ArrayList<DatanodeDescriptor>();
+ BlockPlacementPolicyDefault repl = (BlockPlacementPolicyDefault)replicator;
+ targets = chooseTarget(repl, 1, dataNodes[0], chosenNodes, excludedNodes,
+ BLOCK_SIZE);
+ assertEquals(targets.length, 1);
+ assertFalse(cluster.isOnSameRack(targets[0], dataNodes[0]));
+
+ // reset
+ namenode.getNamesystem().getBlockManager().getDatanodeManager()
+ .setAvoidStaleDataNodesForWrite(false);
+ dataNodes[0].setLastUpdate(Time.now());
+ }
+
+ /**
+ * In this testcase, we set 3 nodes (dataNodes[0] ~ dataNodes[2]) as stale,
+ * and when the number of replicas is less or equal to 3, all the healthy
+ * datanodes should be returned by the chooseTarget method. When the number
+ * of replicas is 4, a stale node should be included.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testChooseTargetWithHalfStaleNodes() throws Exception {
+ // Enable stale datanodes checking
+ namenode.getNamesystem().getBlockManager().getDatanodeManager()
+ .setAvoidStaleDataNodesForWrite(true);
+ // Set dataNodes[0], dataNodes[1], and dataNodes[2] as stale
+ for (int i = 0; i < 3; i++) {
+ dataNodes[i].setLastUpdate(Time.now() - staleInterval - 1);
+ }
+
+ DatanodeDescriptor[] targets;
+ targets = replicator.chooseTarget(filename, 0, dataNodes[0],
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ assertEquals(targets.length, 0);
+
+ // We set the datanode[0] as stale, thus should choose datanode[1]
+ targets = replicator.chooseTarget(filename, 1, dataNodes[0],
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ assertEquals(targets.length, 1);
+ assertFalse(containsWithinRange(targets[0], dataNodes, 0, 2));
+
+ targets = replicator.chooseTarget(filename, 2, dataNodes[0],
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ assertEquals(targets.length, 2);
+ assertFalse(containsWithinRange(targets[0], dataNodes, 0, 2));
+ assertFalse(containsWithinRange(targets[1], dataNodes, 0, 2));
+
+ targets = replicator.chooseTarget(filename, 3, dataNodes[0],
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ assertEquals(targets.length, 3);
+ assertTrue(containsWithinRange(targets[0], dataNodes, 3, 5));
+ assertTrue(containsWithinRange(targets[1], dataNodes, 3, 5));
+ assertTrue(containsWithinRange(targets[2], dataNodes, 3, 5));
+
+ targets = replicator.chooseTarget(filename, 4, dataNodes[0],
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ assertEquals(targets.length, 4);
+ assertTrue(containsWithinRange(dataNodes[3], targets, 0, 3));
+ assertTrue(containsWithinRange(dataNodes[4], targets, 0, 3));
+ assertTrue(containsWithinRange(dataNodes[5], targets, 0, 3));
+
+ // reset
+ namenode.getNamesystem().getBlockManager().getDatanodeManager()
+ .setAvoidStaleDataNodesForWrite(false);
+ for (int i = 0; i < dataNodes.length; i++) {
+ dataNodes[i].setLastUpdate(Time.now());
+ }
+ }
+
+ @Test
+ public void testChooseTargetWithMoreThanHalfStaleNodes() throws Exception {
+ HdfsConfiguration conf = new HdfsConfiguration();
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_CHECK_STALE_DATANODE_KEY, true);
+ conf.setBoolean(
+ DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_WRITE_KEY, true);
+ String[] hosts = new String[]{"host1", "host2", "host3",
+ "host4", "host5", "host6"};
+ String[] racks = new String[]{"/d1/r1", "/d1/r1", "/d1/r2",
+ "/d1/r2", "/d2/r3", "/d2/r3"};
+ MiniDFSCluster miniCluster = new MiniDFSCluster.Builder(conf).racks(racks)
+ .hosts(hosts).numDataNodes(hosts.length).build();
+ miniCluster.waitActive();
+
+ try {
+ // Step 1. Make two datanodes as stale, check whether the
+ // avoidStaleDataNodesForWrite calculation is correct.
+ // First stop the heartbeat of host1 and host2
+ for (int i = 0; i < 2; i++) {
+ DataNode dn = miniCluster.getDataNodes().get(i);
+ DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);
+ miniCluster.getNameNode().getNamesystem().getBlockManager()
+ .getDatanodeManager().getDatanode(dn.getDatanodeId())
+ .setLastUpdate(Time.now() - staleInterval - 1);
+ }
+ // Instead of waiting, explicitly call heartbeatCheck to
+ // let heartbeat manager to detect stale nodes
+ miniCluster.getNameNode().getNamesystem().getBlockManager()
+ .getDatanodeManager().getHeartbeatManager().heartbeatCheck();
+ int numStaleNodes = miniCluster.getNameNode().getNamesystem()
+ .getBlockManager().getDatanodeManager().getNumStaleNodes();
+ assertEquals(numStaleNodes, 2);
+ assertTrue(miniCluster.getNameNode().getNamesystem().getBlockManager()
+ .getDatanodeManager().isAvoidingStaleDataNodesForWrite());
+ // Call chooseTarget
+ DatanodeDescriptor staleNodeInfo = miniCluster.getNameNode()
+ .getNamesystem().getBlockManager().getDatanodeManager()
+ .getDatanode(miniCluster.getDataNodes().get(0).getDatanodeId());
+ BlockPlacementPolicy replicator = miniCluster.getNameNode()
+ .getNamesystem().getBlockManager().getBlockPlacementPolicy();
+ DatanodeDescriptor[] targets = replicator.chooseTarget(filename, 3,
+ staleNodeInfo, new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ assertEquals(targets.length, 3);
+ assertFalse(cluster.isOnSameRack(targets[0], staleNodeInfo));
+
+ // Step 2. Set more than half of the datanodes as stale
+ for (int i = 0; i < 4; i++) {
+ DataNode dn = miniCluster.getDataNodes().get(i);
+ DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);
+ miniCluster.getNameNode().getNamesystem().getBlockManager()
+ .getDatanodeManager().getDatanode(dn.getDatanodeId())
+ .setLastUpdate(Time.now() - staleInterval - 1);
+ }
+ // Explicitly call heartbeatCheck
+ miniCluster.getNameNode().getNamesystem().getBlockManager()
+ .getDatanodeManager().getHeartbeatManager().heartbeatCheck();
+ numStaleNodes = miniCluster.getNameNode().getNamesystem()
+ .getBlockManager().getDatanodeManager().getNumStaleNodes();
+ assertEquals(numStaleNodes, 4);
+ // According to our strategy, stale datanodes will be included for writing
+ // to avoid hotspots
+ assertFalse(miniCluster.getNameNode().getNamesystem().getBlockManager()
+ .getDatanodeManager().isAvoidingStaleDataNodesForWrite());
+ // Call chooseTarget
+ targets = replicator.chooseTarget(filename, 3,
+ staleNodeInfo, new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ assertEquals(targets.length, 3);
+ assertTrue(cluster.isOnSameRack(targets[0], staleNodeInfo));
+
+ // Step 3. Set 2 stale datanodes back to healthy nodes,
+ // still have 2 stale nodes
+ for (int i = 2; i < 4; i++) {
+ DataNode dn = miniCluster.getDataNodes().get(i);
+ DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, false);
+ miniCluster.getNameNode().getNamesystem().getBlockManager()
+ .getDatanodeManager().getDatanode(dn.getDatanodeId())
+ .setLastUpdate(Time.now());
+ }
+ // Explicitly call heartbeatCheck
+ miniCluster.getNameNode().getNamesystem().getBlockManager()
+ .getDatanodeManager().getHeartbeatManager().heartbeatCheck();
+ numStaleNodes = miniCluster.getNameNode().getNamesystem()
+ .getBlockManager().getDatanodeManager().getNumStaleNodes();
+ assertEquals(numStaleNodes, 2);
+ assertTrue(miniCluster.getNameNode().getNamesystem().getBlockManager()
+ .getDatanodeManager().isAvoidingStaleDataNodesForWrite());
+ // Call chooseTarget
+ targets = replicator.chooseTarget(filename, 3,
+ staleNodeInfo, new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ assertEquals(targets.length, 3);
+ assertFalse(cluster.isOnSameRack(targets[0], staleNodeInfo));
+ } finally {
+ miniCluster.shutdown();
+ }
+ }
/**
* This testcase tests re-replication, when dataNodes[0] is already chosen.
@@ -483,8 +694,8 @@ public class TestReplicationPolicy {
.format(true).build();
try {
cluster.waitActive();
- final UnderReplicatedBlocks neededReplications = (UnderReplicatedBlocks) cluster
- .getNameNode().getNamesystem().getBlockManager().neededReplications;
+ final UnderReplicatedBlocks neededReplications = cluster.getNameNode()
+ .getNamesystem().getBlockManager().neededReplications;
for (int i = 0; i < 100; i++) {
// Adding the blocks directly to normal priority
neededReplications.add(new Block(random.nextLong()), 2, 0, 3);
@@ -522,10 +733,10 @@ public class TestReplicationPolicy {
// Adding QUEUE_VERY_UNDER_REPLICATED block
underReplicatedBlocks.add(new Block(random.nextLong()), 2, 0, 7);
- // Adding QUEUE_UNDER_REPLICATED block
+ // Adding QUEUE_REPLICAS_BADLY_DISTRIBUTED block
underReplicatedBlocks.add(new Block(random.nextLong()), 6, 0, 6);
- // Adding QUEUE_REPLICAS_BADLY_DISTRIBUTED block
+ // Adding QUEUE_UNDER_REPLICATED block
underReplicatedBlocks.add(new Block(random.nextLong()), 5, 0, 6);
// Adding QUEUE_WITH_CORRUPT_BLOCKS block
@@ -611,6 +822,11 @@ public class TestReplicationPolicy {
dataNodes[5].setRemaining(1*1024*1024);
replicaNodeList.add(dataNodes[5]);
+ // Refresh the last update time for all the datanodes
+ for (int i = 0; i < dataNodes.length; i++) {
+ dataNodes[i].setLastUpdate(Time.now());
+ }
+
List<DatanodeDescriptor> first = new ArrayList<DatanodeDescriptor>();
List<DatanodeDescriptor> second = new ArrayList<DatanodeDescriptor>();
replicator.splitNodesWithRack(
@@ -633,4 +849,92 @@ public class TestReplicationPolicy {
null, null, (short)2, first, second);
assertEquals(chosenNode, dataNodes[5]);
}
+
+ /**
+ * This testcase tests whether the default value returned by
+ * DFSUtil.getInvalidateWorkPctPerIteration() is positive,
+ * and whether an IllegalArgumentException will be thrown
+ * when 0.0f is retrieved
+ */
+ @Test
+ public void testGetInvalidateWorkPctPerIteration() {
+ Configuration conf = new Configuration();
+ float blocksInvalidateWorkPct = DFSUtil
+ .getInvalidateWorkPctPerIteration(conf);
+ assertTrue(blocksInvalidateWorkPct > 0);
+
+ conf.set(DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION,
+ "0.5f");
+ blocksInvalidateWorkPct = DFSUtil.getInvalidateWorkPctPerIteration(conf);
+ assertEquals(blocksInvalidateWorkPct, 0.5f, blocksInvalidateWorkPct * 1e-7);
+
+ conf.set(DFSConfigKeys.
+ DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION, "1.0f");
+ blocksInvalidateWorkPct = DFSUtil.getInvalidateWorkPctPerIteration(conf);
+ assertEquals(blocksInvalidateWorkPct, 1.0f, blocksInvalidateWorkPct * 1e-7);
+
+ conf.set(DFSConfigKeys.
+ DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION, "0.0f");
+ exception.expect(IllegalArgumentException.class);
+ blocksInvalidateWorkPct = DFSUtil.getInvalidateWorkPctPerIteration(conf);
+ }
+
+ /**
+ * This testcase tests whether an IllegalArgumentException
+ * will be thrown when a negative value is retrieved by
+ * DFSUtil#getInvalidateWorkPctPerIteration
+ */
+ @Test
+ public void testGetInvalidateWorkPctPerIteration_NegativeValue() {
+ Configuration conf = new Configuration();
+ float blocksInvalidateWorkPct = DFSUtil
+ .getInvalidateWorkPctPerIteration(conf);
+ assertTrue(blocksInvalidateWorkPct > 0);
+
+ conf.set(DFSConfigKeys.
+ DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION, "-0.5f");
+ exception.expect(IllegalArgumentException.class);
+ blocksInvalidateWorkPct = DFSUtil.getInvalidateWorkPctPerIteration(conf);
+ }
+
+ /**
+ * This testcase tests whether an IllegalArgumentException
+ * will be thrown when a value greater than 1 is retrieved by
+ * DFSUtil#getInvalidateWorkPctPerIteration
+ */
+ @Test
+ public void testGetInvalidateWorkPctPerIteration_GreaterThanOne() {
+ Configuration conf = new Configuration();
+ float blocksInvalidateWorkPct = DFSUtil
+ .getInvalidateWorkPctPerIteration(conf);
+ assertTrue(blocksInvalidateWorkPct > 0);
+
+ conf.set(DFSConfigKeys.
+ DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION, "1.5f");
+ exception.expect(IllegalArgumentException.class);
+ blocksInvalidateWorkPct = DFSUtil.getInvalidateWorkPctPerIteration(conf);
+ }
+
+ /**
+ * This testcase tests whether the value returned by
+ * DFSUtil.getReplWorkMultiplier() is positive,
+ * and whether an IllegalArgumentException will be thrown
+ * when a non-positive value is retrieved
+ */
+ @Test
+ public void testGetReplWorkMultiplier() {
+ Configuration conf = new Configuration();
+ int blocksReplWorkMultiplier = DFSUtil.getReplWorkMultiplier(conf);
+ assertTrue(blocksReplWorkMultiplier > 0);
+
+ conf.set(DFSConfigKeys.
+ DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION,"3");
+ blocksReplWorkMultiplier = DFSUtil.getReplWorkMultiplier(conf);
+ assertEquals(blocksReplWorkMultiplier, 3);
+
+ conf.set(DFSConfigKeys.
+ DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION,"-1");
+ exception.expect(IllegalArgumentException.class);
+ blocksReplWorkMultiplier = DFSUtil.getReplWorkMultiplier(conf);
+ }
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java?rev=1399950&r1=1399949&r2=1399950&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java Fri Oct 19 02:25:55 2012
@@ -17,6 +17,10 @@
*/
package org.apache.hadoop.hdfs.server.blockmanagement;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -26,7 +30,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import junit.framework.TestCase;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystem;
@@ -39,7 +42,7 @@ import org.apache.hadoop.net.NetworkTopo
import org.apache.hadoop.net.Node;
import org.junit.Test;
-public class TestReplicationPolicyWithNodeGroup extends TestCase {
+public class TestReplicationPolicyWithNodeGroup {
private static final int BLOCK_SIZE = 1024;
private static final int NUM_OF_DATANODES = 8;
private static final Configuration CONF = new HdfsConfiguration();
@@ -104,37 +107,38 @@ public class TestReplicationPolicyWithNo
* the 1st is on dataNodes[0] and the 2nd is on a different rack.
* @throws Exception
*/
+ @Test
public void testChooseTarget1() throws Exception {
dataNodes[0].updateHeartbeat(
2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L,
HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 4, 0); // overloaded
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename,
- 0, dataNodes[0], BLOCK_SIZE);
+ targets = replicator.chooseTarget(filename, 0, dataNodes[0],
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename,
- 1, dataNodes[0], BLOCK_SIZE);
+ targets = replicator.chooseTarget(filename, 1, dataNodes[0],
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 1);
assertEquals(targets[0], dataNodes[0]);
- targets = replicator.chooseTarget(filename,
- 2, dataNodes[0], BLOCK_SIZE);
+ targets = replicator.chooseTarget(filename, 2, dataNodes[0],
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 2);
assertEquals(targets[0], dataNodes[0]);
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
- targets = replicator.chooseTarget(filename,
- 3, dataNodes[0], BLOCK_SIZE);
+ targets = replicator.chooseTarget(filename, 3, dataNodes[0],
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 3);
assertEquals(targets[0], dataNodes[0]);
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
assertTrue(cluster.isOnSameRack(targets[1], targets[2]));
assertFalse(cluster.isOnSameNodeGroup(targets[1], targets[2]));
- targets = replicator.chooseTarget(filename,
- 4, dataNodes[0], BLOCK_SIZE);
+ targets = replicator.chooseTarget(filename, 4, dataNodes[0],
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 4);
assertEquals(targets[0], dataNodes[0]);
assertTrue(cluster.isOnSameRack(targets[1], targets[2]) ||
@@ -164,6 +168,7 @@ public class TestReplicationPolicyWithNo
* node group, and the rest should be placed on a third rack.
* @throws Exception
*/
+ @Test
public void testChooseTarget2() throws Exception {
HashMap<Node, Node> excludedNodes;
DatanodeDescriptor[] targets;
@@ -207,6 +212,7 @@ public class TestReplicationPolicyWithNo
* and the rest should be placed on the third rack.
* @throws Exception
*/
+ @Test
public void testChooseTarget3() throws Exception {
// make data node 0 to be not qualified to choose
dataNodes[0].updateHeartbeat(
@@ -214,30 +220,30 @@ public class TestReplicationPolicyWithNo
(HdfsConstants.MIN_BLOCKS_FOR_WRITE-1)*BLOCK_SIZE, 0L, 0, 0); // no space
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename,
- 0, dataNodes[0], BLOCK_SIZE);
+ targets = replicator.chooseTarget(filename, 0, dataNodes[0],
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename,
- 1, dataNodes[0], BLOCK_SIZE);
+ targets = replicator.chooseTarget(filename, 1, dataNodes[0],
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 1);
assertEquals(targets[0], dataNodes[1]);
- targets = replicator.chooseTarget(filename,
- 2, dataNodes[0], BLOCK_SIZE);
+ targets = replicator.chooseTarget(filename, 2, dataNodes[0],
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 2);
assertEquals(targets[0], dataNodes[1]);
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
- targets = replicator.chooseTarget(filename,
- 3, dataNodes[0], BLOCK_SIZE);
+ targets = replicator.chooseTarget(filename, 3, dataNodes[0],
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 3);
assertEquals(targets[0], dataNodes[1]);
assertTrue(cluster.isOnSameRack(targets[1], targets[2]));
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
- targets = replicator.chooseTarget(filename,
- 4, dataNodes[0], BLOCK_SIZE);
+ targets = replicator.chooseTarget(filename, 4, dataNodes[0],
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 4);
assertEquals(targets[0], dataNodes[1]);
assertTrue(cluster.isNodeGroupAware());
@@ -259,6 +265,7 @@ public class TestReplicationPolicyWithNo
* in different node group.
* @throws Exception
*/
+ @Test
public void testChooseTarget4() throws Exception {
// make data node 0-2 to be not qualified to choose: not enough disk space
for(int i=0; i<3; i++) {
@@ -268,23 +275,23 @@ public class TestReplicationPolicyWithNo
}
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename,
- 0, dataNodes[0], BLOCK_SIZE);
+ targets = replicator.chooseTarget(filename, 0, dataNodes[0],
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename,
- 1, dataNodes[0], BLOCK_SIZE);
+ targets = replicator.chooseTarget(filename, 1, dataNodes[0],
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 1);
assertFalse(cluster.isOnSameRack(targets[0], dataNodes[0]));
- targets = replicator.chooseTarget(filename,
- 2, dataNodes[0], BLOCK_SIZE);
+ targets = replicator.chooseTarget(filename, 2, dataNodes[0],
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 2);
assertFalse(cluster.isOnSameRack(targets[0], dataNodes[0]));
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
- targets = replicator.chooseTarget(filename,
- 3, dataNodes[0], BLOCK_SIZE);
+ targets = replicator.chooseTarget(filename, 3, dataNodes[0],
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 3);
for(int i=0; i<3; i++) {
assertFalse(cluster.isOnSameRack(targets[i], dataNodes[0]));
@@ -302,24 +309,25 @@ public class TestReplicationPolicyWithNo
* the 3rd replica should be placed on the same rack as the 2nd replica,
* @throws Exception
*/
+ @Test
public void testChooseTarget5() throws Exception {
setupDataNodeCapacity();
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename,
- 0, NODE, BLOCK_SIZE);
+ targets = replicator.chooseTarget(filename, 0, NODE,
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 0);
-
- targets = replicator.chooseTarget(filename,
- 1, NODE, BLOCK_SIZE);
+
+ targets = replicator.chooseTarget(filename, 1, NODE,
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 1);
-
- targets = replicator.chooseTarget(filename,
- 2, NODE, BLOCK_SIZE);
+
+ targets = replicator.chooseTarget(filename, 2, NODE,
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 2);
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
-
- targets = replicator.chooseTarget(filename,
- 3, NODE, BLOCK_SIZE);
+
+ targets = replicator.chooseTarget(filename, 3, NODE,
+ new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
assertEquals(targets.length, 3);
assertTrue(cluster.isOnSameRack(targets[1], targets[2]));
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
@@ -333,6 +341,7 @@ public class TestReplicationPolicyWithNo
* the 1st replica. The 3rd replica can be placed randomly.
* @throws Exception
*/
+ @Test
public void testRereplicate1() throws Exception {
setupDataNodeCapacity();
List<DatanodeDescriptor> chosenNodes = new ArrayList<DatanodeDescriptor>();
@@ -369,6 +378,7 @@ public class TestReplicationPolicyWithNo
* the rest replicas can be placed randomly,
* @throws Exception
*/
+ @Test
public void testRereplicate2() throws Exception {
setupDataNodeCapacity();
List<DatanodeDescriptor> chosenNodes = new ArrayList<DatanodeDescriptor>();
@@ -399,6 +409,7 @@ public class TestReplicationPolicyWithNo
* the rest replicas can be placed randomly,
* @throws Exception
*/
+ @Test
public void testRereplicate3() throws Exception {
setupDataNodeCapacity();
List<DatanodeDescriptor> chosenNodes = new ArrayList<DatanodeDescriptor>();
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlocks.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlocks.java?rev=1399950&r1=1399949&r2=1399950&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlocks.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlocks.java Fri Oct 19 02:25:55 2012
@@ -17,7 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.blockmanagement;
-import junit.framework.TestCase;
+import static org.junit.Assert.assertEquals;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -27,8 +27,10 @@ import org.apache.hadoop.hdfs.DFSTestUti
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.junit.Test;
-public class TestUnderReplicatedBlocks extends TestCase {
+public class TestUnderReplicatedBlocks {
+ @Test(timeout=300000) // 5 min timeout
public void testSetrepIncWithUnderReplicatedBlocks() throws Exception {
Configuration conf = new HdfsConfiguration();
final short REPLICATION_FACTOR = 2;
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestGetUriFromString.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestGetUriFromString.java?rev=1399950&r1=1399949&r2=1399950&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestGetUriFromString.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestGetUriFromString.java Fri Oct 19 02:25:55 2012
@@ -17,19 +17,21 @@
*/
package org.apache.hadoop.hdfs.server.common;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
import java.io.IOException;
import java.net.URI;
-import junit.framework.TestCase;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.junit.Test;
/**
* This is a unit test, which tests {@link Util#stringAsURI(String)}
* for Windows and Unix style file paths.
*/
-public class TestGetUriFromString extends TestCase {
+public class TestGetUriFromString {
private static final Log LOG = LogFactory.getLog(TestGetUriFromString.class);
private static final String RELATIVE_FILE_PATH = "relativeFilePath";
@@ -49,6 +51,7 @@ public class TestGetUriFromString extend
* Test for a relative path, os independent
* @throws IOException
*/
+ @Test
public void testRelativePathAsURI() throws IOException {
URI u = Util.stringAsURI(RELATIVE_FILE_PATH);
LOG.info("Uri: " + u);
@@ -59,6 +62,7 @@ public class TestGetUriFromString extend
* Test for an OS dependent absolute paths.
* @throws IOException
*/
+ @Test
public void testAbsolutePathAsURI() throws IOException {
URI u = null;
u = Util.stringAsURI(ABSOLUTE_PATH_WINDOWS);
@@ -74,6 +78,7 @@ public class TestGetUriFromString extend
* Test for a URI
* @throws IOException
*/
+ @Test
public void testURI() throws IOException {
LOG.info("Testing correct Unix URI: " + URI_UNIX);
URI u = Util.stringAsURI(URI_UNIX);
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java?rev=1399950&r1=1399949&r2=1399950&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java Fri Oct 19 02:25:55 2012
@@ -22,6 +22,7 @@ import static org.mockito.Mockito.when;
import java.io.IOException;
import java.net.InetSocketAddress;
+import java.util.ArrayList;
import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
@@ -29,10 +30,17 @@ import javax.servlet.http.HttpServletReq
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.namenode.NameNodeHttpServer;
+import org.apache.hadoop.hdfs.web.resources.DoAsParam;
+import org.apache.hadoop.hdfs.web.resources.UserParam;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
+import org.apache.hadoop.security.authorize.AuthorizationException;
+import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager;
@@ -136,4 +144,301 @@ public class TestJspHelper {
Assert.assertEquals("", delegationTokenParam);
}
+ @Test
+ public void testGetUgiFromToken() throws IOException {
+ conf.set(DFSConfigKeys.FS_DEFAULT_NAME_KEY, "hdfs://localhost:4321/");
+ ServletContext context = mock(ServletContext.class);
+ String realUser = "TheDoctor";
+ String user = "TheNurse";
+ conf.set(DFSConfigKeys.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
+ UserGroupInformation.setConfiguration(conf);
+ UserGroupInformation ugi;
+ HttpServletRequest request;
+
+ Text ownerText = new Text(user);
+ DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(
+ ownerText, ownerText, new Text(realUser));
+ Token<DelegationTokenIdentifier> token = new Token<DelegationTokenIdentifier>(
+ dtId, new DummySecretManager(0, 0, 0, 0));
+ String tokenString = token.encodeToUrlString();
+
+ // token with no auth-ed user
+ request = getMockRequest(null, null, null);
+ when(request.getParameter(JspHelper.DELEGATION_PARAMETER_NAME)).thenReturn(
+ tokenString);
+ ugi = JspHelper.getUGI(context, request, conf);
+ Assert.assertNotNull(ugi.getRealUser());
+ Assert.assertEquals(ugi.getRealUser().getShortUserName(), realUser);
+ Assert.assertEquals(ugi.getShortUserName(), user);
+ checkUgiFromToken(ugi);
+
+ // token with auth-ed user
+ request = getMockRequest(realUser, null, null);
+ when(request.getParameter(JspHelper.DELEGATION_PARAMETER_NAME)).thenReturn(
+ tokenString);
+ ugi = JspHelper.getUGI(context, request, conf);
+ Assert.assertNotNull(ugi.getRealUser());
+ Assert.assertEquals(ugi.getRealUser().getShortUserName(), realUser);
+ Assert.assertEquals(ugi.getShortUserName(), user);
+ checkUgiFromToken(ugi);
+
+ // completely different user, token trumps auth
+ request = getMockRequest("rogue", null, null);
+ when(request.getParameter(JspHelper.DELEGATION_PARAMETER_NAME)).thenReturn(
+ tokenString);
+ ugi = JspHelper.getUGI(context, request, conf);
+ Assert.assertNotNull(ugi.getRealUser());
+ Assert.assertEquals(ugi.getRealUser().getShortUserName(), realUser);
+ Assert.assertEquals(ugi.getShortUserName(), user);
+ checkUgiFromToken(ugi);
+
+ // expected case
+ request = getMockRequest(null, user, null);
+ when(request.getParameter(JspHelper.DELEGATION_PARAMETER_NAME)).thenReturn(
+ tokenString);
+ ugi = JspHelper.getUGI(context, request, conf);
+ Assert.assertNotNull(ugi.getRealUser());
+ Assert.assertEquals(ugi.getRealUser().getShortUserName(), realUser);
+ Assert.assertEquals(ugi.getShortUserName(), user);
+ checkUgiFromToken(ugi);
+
+ // can't proxy with a token!
+ request = getMockRequest(null, null, "rogue");
+ when(request.getParameter(JspHelper.DELEGATION_PARAMETER_NAME)).thenReturn(
+ tokenString);
+ try {
+ JspHelper.getUGI(context, request, conf);
+ Assert.fail("bad request allowed");
+ } catch (IOException ioe) {
+ Assert.assertEquals(
+ "Usernames not matched: name=rogue != expected="+user,
+ ioe.getMessage());
+ }
+
+ // can't proxy with a token!
+ request = getMockRequest(null, user, "rogue");
+ when(request.getParameter(JspHelper.DELEGATION_PARAMETER_NAME)).thenReturn(
+ tokenString);
+ try {
+ JspHelper.getUGI(context, request, conf);
+ Assert.fail("bad request allowed");
+ } catch (IOException ioe) {
+ Assert.assertEquals(
+ "Usernames not matched: name=rogue != expected="+user,
+ ioe.getMessage());
+ }
+ }
+
+ @Test
+ public void testGetNonProxyUgi() throws IOException {
+ conf.set(DFSConfigKeys.FS_DEFAULT_NAME_KEY, "hdfs://localhost:4321/");
+ ServletContext context = mock(ServletContext.class);
+ String realUser = "TheDoctor";
+ String user = "TheNurse";
+ conf.set(DFSConfigKeys.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
+ UserGroupInformation.setConfiguration(conf);
+ UserGroupInformation ugi;
+ HttpServletRequest request;
+
+ // have to be auth-ed with remote user
+ request = getMockRequest(null, null, null);
+ try {
+ JspHelper.getUGI(context, request, conf);
+ Assert.fail("bad request allowed");
+ } catch (IOException ioe) {
+ Assert.assertEquals(
+ "Security enabled but user not authenticated by filter",
+ ioe.getMessage());
+ }
+ request = getMockRequest(null, realUser, null);
+ try {
+ JspHelper.getUGI(context, request, conf);
+ Assert.fail("bad request allowed");
+ } catch (IOException ioe) {
+ Assert.assertEquals(
+ "Security enabled but user not authenticated by filter",
+ ioe.getMessage());
+ }
+
+ // ugi for remote user
+ request = getMockRequest(realUser, null, null);
+ ugi = JspHelper.getUGI(context, request, conf);
+ Assert.assertNull(ugi.getRealUser());
+ Assert.assertEquals(ugi.getShortUserName(), realUser);
+ checkUgiFromAuth(ugi);
+
+ // ugi for remote user = real user
+ request = getMockRequest(realUser, realUser, null);
+ ugi = JspHelper.getUGI(context, request, conf);
+ Assert.assertNull(ugi.getRealUser());
+ Assert.assertEquals(ugi.getShortUserName(), realUser);
+ checkUgiFromAuth(ugi);
+
+ // ugi for remote user != real user
+ request = getMockRequest(realUser, user, null);
+ try {
+ JspHelper.getUGI(context, request, conf);
+ Assert.fail("bad request allowed");
+ } catch (IOException ioe) {
+ Assert.assertEquals(
+ "Usernames not matched: name="+user+" != expected="+realUser,
+ ioe.getMessage());
+ }
+ }
+
+ @Test
+ public void testGetProxyUgi() throws IOException {
+ conf.set(DFSConfigKeys.FS_DEFAULT_NAME_KEY, "hdfs://localhost:4321/");
+ ServletContext context = mock(ServletContext.class);
+ String realUser = "TheDoctor";
+ String user = "TheNurse";
+ conf.set(DFSConfigKeys.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
+
+ conf.set(ProxyUsers.CONF_HADOOP_PROXYUSER+realUser+".groups", "*");
+ conf.set(ProxyUsers.CONF_HADOOP_PROXYUSER+realUser+".hosts", "*");
+ ProxyUsers.refreshSuperUserGroupsConfiguration(conf);
+ UserGroupInformation.setConfiguration(conf);
+ UserGroupInformation ugi;
+ HttpServletRequest request;
+
+ // have to be auth-ed with remote user
+ request = getMockRequest(null, null, user);
+ try {
+ JspHelper.getUGI(context, request, conf);
+ Assert.fail("bad request allowed");
+ } catch (IOException ioe) {
+ Assert.assertEquals(
+ "Security enabled but user not authenticated by filter",
+ ioe.getMessage());
+ }
+ request = getMockRequest(null, realUser, user);
+ try {
+ JspHelper.getUGI(context, request, conf);
+ Assert.fail("bad request allowed");
+ } catch (IOException ioe) {
+ Assert.assertEquals(
+ "Security enabled but user not authenticated by filter",
+ ioe.getMessage());
+ }
+
+ // proxy ugi for user via remote user
+ request = getMockRequest(realUser, null, user);
+ ugi = JspHelper.getUGI(context, request, conf);
+ Assert.assertNotNull(ugi.getRealUser());
+ Assert.assertEquals(ugi.getRealUser().getShortUserName(), realUser);
+ Assert.assertEquals(ugi.getShortUserName(), user);
+ checkUgiFromAuth(ugi);
+
+ // proxy ugi for user vi a remote user = real user
+ request = getMockRequest(realUser, realUser, user);
+ ugi = JspHelper.getUGI(context, request, conf);
+ Assert.assertNotNull(ugi.getRealUser());
+ Assert.assertEquals(ugi.getRealUser().getShortUserName(), realUser);
+ Assert.assertEquals(ugi.getShortUserName(), user);
+ checkUgiFromAuth(ugi);
+
+ // proxy ugi for user via remote user != real user
+ request = getMockRequest(realUser, user, user);
+ try {
+ JspHelper.getUGI(context, request, conf);
+ Assert.fail("bad request allowed");
+ } catch (IOException ioe) {
+ Assert.assertEquals(
+ "Usernames not matched: name="+user+" != expected="+realUser,
+ ioe.getMessage());
+ }
+
+ // try to get get a proxy user with unauthorized user
+ try {
+ request = getMockRequest(user, null, realUser);
+ JspHelper.getUGI(context, request, conf);
+ Assert.fail("bad proxy request allowed");
+ } catch (AuthorizationException ae) {
+ Assert.assertEquals(
+ "User: " + user + " is not allowed to impersonate " + realUser,
+ ae.getMessage());
+ }
+ try {
+ request = getMockRequest(user, user, realUser);
+ JspHelper.getUGI(context, request, conf);
+ Assert.fail("bad proxy request allowed");
+ } catch (AuthorizationException ae) {
+ Assert.assertEquals(
+ "User: " + user + " is not allowed to impersonate " + realUser,
+ ae.getMessage());
+ }
+ }
+
+ private HttpServletRequest getMockRequest(String remoteUser, String user, String doAs) {
+ HttpServletRequest request = mock(HttpServletRequest.class);
+ when(request.getParameter(UserParam.NAME)).thenReturn(user);
+ if (doAs != null) {
+ when(request.getParameter(DoAsParam.NAME)).thenReturn(doAs);
+ }
+ when(request.getRemoteUser()).thenReturn(remoteUser);
+ return request;
+ }
+
+ private void checkUgiFromAuth(UserGroupInformation ugi) {
+ if (ugi.getRealUser() != null) {
+ Assert.assertEquals(AuthenticationMethod.PROXY,
+ ugi.getAuthenticationMethod());
+ Assert.assertEquals(AuthenticationMethod.KERBEROS_SSL,
+ ugi.getRealUser().getAuthenticationMethod());
+ } else {
+ Assert.assertEquals(AuthenticationMethod.KERBEROS_SSL,
+ ugi.getAuthenticationMethod());
+ }
+ }
+
+ private void checkUgiFromToken(UserGroupInformation ugi) {
+ if (ugi.getRealUser() != null) {
+ Assert.assertEquals(AuthenticationMethod.PROXY,
+ ugi.getAuthenticationMethod());
+ Assert.assertEquals(AuthenticationMethod.TOKEN,
+ ugi.getRealUser().getAuthenticationMethod());
+ } else {
+ Assert.assertEquals(AuthenticationMethod.TOKEN,
+ ugi.getAuthenticationMethod());
+ }
+ }
+
+ @Test
+ public void testSortNodeByFields() throws Exception {
+ DatanodeID dnId1 = new DatanodeID("127.0.0.1", "localhost1", "storage1",
+ 1234, 2345, 3456);
+ DatanodeID dnId2 = new DatanodeID("127.0.0.2", "localhost2", "storage2",
+ 1235, 2346, 3457);
+ DatanodeDescriptor dnDesc1 = new DatanodeDescriptor(dnId1, "rack1", 1024,
+ 100, 924, 100, 10, 2);
+ DatanodeDescriptor dnDesc2 = new DatanodeDescriptor(dnId2, "rack2", 2500,
+ 200, 1848, 200, 20, 1);
+ ArrayList<DatanodeDescriptor> live = new ArrayList<DatanodeDescriptor>();
+ live.add(dnDesc1);
+ live.add(dnDesc2);
+
+ // Test sorting by failed volumes
+ JspHelper.sortNodeList(live, "volfails", "ASC");
+ Assert.assertEquals(dnDesc2, live.get(0));
+ Assert.assertEquals(dnDesc1, live.get(1));
+ JspHelper.sortNodeList(live, "volfails", "DSC");
+ Assert.assertEquals(dnDesc1, live.get(0));
+ Assert.assertEquals(dnDesc2, live.get(1));
+
+ // Test sorting by Blockpool used
+ JspHelper.sortNodeList(live, "bpused", "ASC");
+ Assert.assertEquals(dnDesc1, live.get(0));
+ Assert.assertEquals(dnDesc2, live.get(1));
+ JspHelper.sortNodeList(live, "bpused", "DSC");
+ Assert.assertEquals(dnDesc2, live.get(0));
+ Assert.assertEquals(dnDesc1, live.get(1));
+
+ // Test sorting by Percentage Blockpool used
+ JspHelper.sortNodeList(live, "pcbpused", "ASC");
+ Assert.assertEquals(dnDesc2, live.get(0));
+ Assert.assertEquals(dnDesc1, live.get(1));
+ JspHelper.sortNodeList(live, "pcbpused", "DSC");
+ Assert.assertEquals(dnDesc1, live.get(0));
+ Assert.assertEquals(dnDesc2, live.get(1));
+ }
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java?rev=1399950&r1=1399949&r2=1399950&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java Fri Oct 19 02:25:55 2012
@@ -105,10 +105,19 @@ public class DataNodeTestUtils {
}
public static InterDatanodeProtocol createInterDatanodeProtocolProxy(
- DataNode dn, DatanodeID datanodeid, final Configuration conf
- ) throws IOException {
+ DataNode dn, DatanodeID datanodeid, final Configuration conf,
+ boolean connectToDnViaHostname) throws IOException {
+ if (connectToDnViaHostname != dn.getDnConf().connectToDnViaHostname) {
+ throw new AssertionError("Unexpected DN hostname configuration");
+ }
return DataNode.createInterDataNodeProtocolProxy(datanodeid, conf,
- dn.getDnConf().socketTimeout);
+ dn.getDnConf().socketTimeout, dn.getDnConf().connectToDnViaHostname);
+ }
+
+ public static void runBlockScannerForBlock(DataNode dn, ExtendedBlock b) {
+ DataBlockScanner scanner = dn.getBlockScanner();
+ BlockPoolSliceScanner bpScanner = scanner.getBPScanner(b.getBlockPoolId());
+ bpScanner.verifyBlock(b);
}
public static void shutdownBlockScanner(DataNode dn) {
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java?rev=1399950&r1=1399949&r2=1399950&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java Fri Oct 19 02:25:55 2012
@@ -37,6 +37,7 @@ import org.apache.hadoop.hdfs.protocol.B
import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
@@ -47,6 +48,7 @@ import org.apache.hadoop.hdfs.server.dat
import org.apache.hadoop.hdfs.server.datanode.metrics.FSDatasetMBean;
import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock;
import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.DiskChecker.DiskErrorException;
@@ -93,8 +95,8 @@ public class SimulatedFSDataset implemen
static final byte[] nullCrcFileData;
static {
- DataChecksum checksum = DataChecksum.newDataChecksum( DataChecksum.
- CHECKSUM_NULL, 16*1024 );
+ DataChecksum checksum = DataChecksum.newDataChecksum(
+ DataChecksum.Type.NULL, 16*1024 );
byte[] nullCrcHeader = checksum.getHeader();
nullCrcFileData = new byte[2 + nullCrcHeader.length];
nullCrcFileData[0] = (byte) ((BlockMetadataHeader.VERSION >>> 8) & 0xff);
@@ -132,10 +134,12 @@ public class SimulatedFSDataset implemen
}
}
+ @Override
synchronized public long getGenerationStamp() {
return theBlock.getGenerationStamp();
}
+ @Override
synchronized public long getNumBytes() {
if (!finalized) {
return bytesRcvd;
@@ -144,6 +148,7 @@ public class SimulatedFSDataset implemen
}
}
+ @Override
synchronized public void setNumBytes(long length) {
if (!finalized) {
bytesRcvd = length;
@@ -686,7 +691,7 @@ public class SimulatedFSDataset implemen
public synchronized InputStream getBlockInputStream(ExtendedBlock b,
long seekOffset) throws IOException {
InputStream result = getBlockInputStream(b);
- result.skip(seekOffset);
+ IOUtils.skipFully(result, seekOffset);
return result;
}
@@ -876,14 +881,17 @@ public class SimulatedFSDataset implemen
DataNode.LOG.info("Registered FSDatasetState MBean");
}
+ @Override
public void shutdown() {
if (mbeanName != null) MBeans.unregister(mbeanName);
}
+ @Override
public String getStorageInfo() {
return "Simulated FSDataset-" + storageId;
}
+ @Override
public boolean hasEnoughResource() {
return true;
}
@@ -954,6 +962,12 @@ public class SimulatedFSDataset implemen
public BlockLocalPathInfo getBlockLocalPathInfo(ExtendedBlock b) {
throw new UnsupportedOperationException();
}
+
+ @Override
+ public HdfsBlocksMetadata getHdfsBlocksMetadata(List<ExtendedBlock> blocks)
+ throws IOException {
+ throw new UnsupportedOperationException();
+ }
@Override
public String[] getBlockPoolList() {
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java?rev=1399950&r1=1399949&r2=1399950&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java Fri Oct 19 02:25:55 2012
@@ -32,7 +32,6 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics;
@@ -111,10 +110,8 @@ public class TestBPOfferService {
throws Exception {
DatanodeProtocolClientSideTranslatorPB mock =
Mockito.mock(DatanodeProtocolClientSideTranslatorPB.class);
- Mockito.doReturn(
- new NamespaceInfo(1, FAKE_CLUSTERID, FAKE_BPID,
- 0, HdfsConstants.LAYOUT_VERSION))
- .when(mock).versionRequest();
+ Mockito.doReturn(new NamespaceInfo(1, FAKE_CLUSTERID, FAKE_BPID, 0))
+ .when(mock).versionRequest();
Mockito.doReturn(DFSTestUtil.getLocalDatanodeRegistration())
.when(mock).registerDatanode(Mockito.any(DatanodeRegistration.class));
@@ -229,10 +226,9 @@ public class TestBPOfferService {
*/
@Test
public void testNNsFromDifferentClusters() throws Exception {
- Mockito.doReturn(
- new NamespaceInfo(1, "fake foreign cluster", FAKE_BPID,
- 0, HdfsConstants.LAYOUT_VERSION))
- .when(mockNN1).versionRequest();
+ Mockito
+ .doReturn(new NamespaceInfo(1, "fake foreign cluster", FAKE_BPID, 0))
+ .when(mockNN1).versionRequest();
BPOfferService bpos = setupBPOSForNNs(mockNN1, mockNN2);
bpos.start();
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolManager.java?rev=1399950&r1=1399949&r2=1399950&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolManager.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolManager.java Fri Oct 19 02:25:55 2012
@@ -17,7 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.datanode;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
import java.io.IOException;
import java.net.InetSocketAddress;
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java?rev=1399950&r1=1399949&r2=1399950&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java Fri Oct 19 02:25:55 2012
@@ -147,7 +147,7 @@ public class TestBlockRecovery {
Mockito.any(DatanodeRegistration.class));
when(namenode.versionRequest()).thenReturn(new NamespaceInfo
- (1, CLUSTER_ID, POOL_ID, 1L, 1));
+ (1, CLUSTER_ID, POOL_ID, 1L));
when(namenode.sendHeartbeat(
Mockito.any(DatanodeRegistration.class),
@@ -550,7 +550,7 @@ public class TestBlockRecovery {
ReplicaOutputStreams streams = null;
try {
streams = replicaInfo.createStreams(true,
- DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32, 512));
+ DataChecksum.newDataChecksum(DataChecksum.Type.CRC32, 512));
streams.getChecksumOut().write('a');
dn.data.initReplicaRecovery(new RecoveringBlock(block, null, RECOVERY_ID+1));
try {
@@ -595,6 +595,7 @@ public class TestBlockRecovery {
final AtomicBoolean recoveryInitResult = new AtomicBoolean(true);
Thread recoveryThread = new Thread() {
+ @Override
public void run() {
try {
DatanodeInfo[] locations = block.getLocations();
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java?rev=1399950&r1=1399949&r2=1399950&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java Fri Oct 19 02:25:55 2012
@@ -17,6 +17,10 @@
*/
package org.apache.hadoop.hdfs.server.datanode;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
@@ -47,14 +51,10 @@ import org.apache.hadoop.hdfs.protocol.p
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
-import org.apache.hadoop.hdfs.server.common.Util;
import org.apache.hadoop.hdfs.util.DataTransferThrottler;
import org.apache.hadoop.net.NetUtils;
-
+import org.apache.hadoop.util.Time;
import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
/**
* This class tests if block replacement request to data nodes work correctly.
@@ -71,7 +71,7 @@ public class TestBlockReplacement {
long bandwidthPerSec = 1024*1024L;
final long TOTAL_BYTES =6*bandwidthPerSec;
long bytesToSend = TOTAL_BYTES;
- long start = Util.now();
+ long start = Time.now();
DataTransferThrottler throttler = new DataTransferThrottler(bandwidthPerSec);
long totalBytes = 0L;
long bytesSent = 1024*512L; // 0.5MB
@@ -84,12 +84,12 @@ public class TestBlockReplacement {
Thread.sleep(1000);
} catch (InterruptedException ignored) {}
throttler.throttle(bytesToSend);
- long end = Util.now();
+ long end = Time.now();
assertTrue(totalBytes*1000/(end-start)<=bandwidthPerSec);
}
@Test
- public void testBlockReplacement() throws IOException, TimeoutException {
+ public void testBlockReplacement() throws Exception {
final Configuration CONF = new HdfsConfiguration();
final String[] INITIAL_RACKS = {"/RACK0", "/RACK1", "/RACK2"};
final String[] NEW_RACKS = {"/RACK2"};
@@ -208,7 +208,7 @@ public class TestBlockReplacement {
throws IOException, TimeoutException {
boolean notDone;
final long TIMEOUT = 20000L;
- long starttime = System.currentTimeMillis();
+ long starttime = Time.now();
long failtime = starttime + TIMEOUT;
do {
try {
@@ -233,7 +233,7 @@ public class TestBlockReplacement {
}
}
}
- if (System.currentTimeMillis() > failtime) {
+ if (Time.now() > failtime) {
String expectedNodesList = "";
String currentNodesList = "";
for (DatanodeInfo dn : includeNodes)
@@ -248,7 +248,7 @@ public class TestBlockReplacement {
}
} while(notDone);
LOG.info("Achieved expected replication values in "
- + (System.currentTimeMillis() - starttime) + " msec.");
+ + (Time.now() - starttime) + " msec.");
}
/* Copy a block from sourceProxy to destination. If the block becomes