You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by wa...@apache.org on 2013/10/17 04:14:36 UTC
svn commit: r1532952 [5/6] - in
/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs: ./
src/contrib/bkjournal/ src/main/bin/ src/main/java/
src/main/java/org/apache/hadoop/hdfs/
src/main/java/org/apache/hadoop/hdfs/protocol/ src/main/java...
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpFileSystem.java?rev=1532952&r1=1532951&r2=1532952&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpFileSystem.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpFileSystem.java Thu Oct 17 02:14:33 2013
@@ -294,11 +294,13 @@ public class TestHftpFileSystem {
HftpFileSystem fs = (HftpFileSystem) FileSystem.get(uri, conf);
assertEquals(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT, fs.getDefaultPort());
- assertEquals(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_DEFAULT, fs.getDefaultSecurePort());
assertEquals(uri, fs.getUri());
+
+ // HFTP uses http to get the token so canonical service name should
+ // return the http port.
assertEquals(
- "127.0.0.1:"+DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_DEFAULT,
+ "127.0.0.1:" + DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT,
fs.getCanonicalServiceName()
);
}
@@ -307,17 +309,18 @@ public class TestHftpFileSystem {
public void testHftpCustomDefaultPorts() throws IOException {
Configuration conf = new Configuration();
conf.setInt(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_KEY, 123);
- conf.setInt(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_KEY, 456);
URI uri = URI.create("hftp://localhost");
HftpFileSystem fs = (HftpFileSystem) FileSystem.get(uri, conf);
assertEquals(123, fs.getDefaultPort());
- assertEquals(456, fs.getDefaultSecurePort());
assertEquals(uri, fs.getUri());
+
+ // HFTP uses http to get the token so canonical service name should
+ // return the http port.
assertEquals(
- "127.0.0.1:456",
+ "127.0.0.1:123",
fs.getCanonicalServiceName()
);
}
@@ -329,11 +332,10 @@ public class TestHftpFileSystem {
HftpFileSystem fs = (HftpFileSystem) FileSystem.get(uri, conf);
assertEquals(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT, fs.getDefaultPort());
- assertEquals(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_DEFAULT, fs.getDefaultSecurePort());
assertEquals(uri, fs.getUri());
assertEquals(
- "127.0.0.1:"+DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_DEFAULT,
+ "127.0.0.1:123",
fs.getCanonicalServiceName()
);
}
@@ -342,17 +344,15 @@ public class TestHftpFileSystem {
public void testHftpCustomUriPortWithCustomDefaultPorts() throws IOException {
Configuration conf = new Configuration();
conf.setInt(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_KEY, 123);
- conf.setInt(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_KEY, 456);
URI uri = URI.create("hftp://localhost:789");
HftpFileSystem fs = (HftpFileSystem) FileSystem.get(uri, conf);
assertEquals(123, fs.getDefaultPort());
- assertEquals(456, fs.getDefaultSecurePort());
assertEquals(uri, fs.getUri());
assertEquals(
- "127.0.0.1:456",
+ "127.0.0.1:789",
fs.getCanonicalServiceName()
);
}
@@ -366,7 +366,6 @@ public class TestHftpFileSystem {
HsftpFileSystem fs = (HsftpFileSystem) FileSystem.get(uri, conf);
assertEquals(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_DEFAULT, fs.getDefaultPort());
- assertEquals(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_DEFAULT, fs.getDefaultSecurePort());
assertEquals(uri, fs.getUri());
assertEquals(
@@ -385,7 +384,6 @@ public class TestHftpFileSystem {
HsftpFileSystem fs = (HsftpFileSystem) FileSystem.get(uri, conf);
assertEquals(456, fs.getDefaultPort());
- assertEquals(456, fs.getDefaultSecurePort());
assertEquals(uri, fs.getUri());
assertEquals(
@@ -401,7 +399,6 @@ public class TestHftpFileSystem {
HsftpFileSystem fs = (HsftpFileSystem) FileSystem.get(uri, conf);
assertEquals(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_DEFAULT, fs.getDefaultPort());
- assertEquals(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_DEFAULT, fs.getDefaultSecurePort());
assertEquals(uri, fs.getUri());
assertEquals(
@@ -420,7 +417,6 @@ public class TestHftpFileSystem {
HsftpFileSystem fs = (HsftpFileSystem) FileSystem.get(uri, conf);
assertEquals(456, fs.getDefaultPort());
- assertEquals(456, fs.getDefaultSecurePort());
assertEquals(uri, fs.getUri());
assertEquals(
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPeerCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPeerCache.java?rev=1532952&r1=1532951&r2=1532952&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPeerCache.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPeerCache.java Thu Oct 17 02:14:33 2013
@@ -17,26 +17,23 @@
*/
package org.apache.hadoop.hdfs;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertSame;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.channels.ReadableByteChannel;
-
+import com.google.common.collect.HashMultiset;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.net.Peer;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.net.unix.DomainSocket;
import org.junit.Test;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
-import com.google.common.collect.HashMultiset;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.channels.ReadableByteChannel;
+
+import static org.junit.Assert.*;
public class TestPeerCache {
static final Log LOG = LogFactory.getLog(TestPeerCache.class);
@@ -150,7 +147,7 @@ public class TestPeerCache {
PeerCache cache = new PeerCache(3, 100000);
DatanodeID dnId = new DatanodeID("192.168.0.1",
"fakehostname", "fake_storage_id",
- 100, 101, 102);
+ 100, 101, 102, 103);
FakePeer peer = new FakePeer(dnId, false);
cache.put(dnId, peer);
assertTrue(!peer.isClosed());
@@ -170,7 +167,7 @@ public class TestPeerCache {
for (int i = 0; i < CAPACITY; ++i) {
dnIds[i] = new DatanodeID("192.168.0.1",
"fakehostname_" + i, "fake_storage_id",
- 100, 101, 102);
+ 100, 101, 102, 103);
peers[i] = new FakePeer(dnIds[i], false);
}
for (int i = 0; i < CAPACITY; ++i) {
@@ -201,7 +198,7 @@ public class TestPeerCache {
for (int i = 0; i < dnIds.length; ++i) {
dnIds[i] = new DatanodeID("192.168.0.1",
"fakehostname_" + i, "fake_storage_id_" + i,
- 100, 101, 102);
+ 100, 101, 102, 103);
peers[i] = new FakePeer(dnIds[i], false);
}
for (int i = 0; i < CAPACITY; ++i) {
@@ -232,7 +229,7 @@ public class TestPeerCache {
PeerCache cache = new PeerCache(CAPACITY, 100000);
DatanodeID dnId = new DatanodeID("192.168.0.1",
"fakehostname", "fake_storage_id",
- 100, 101, 102);
+ 100, 101, 102, 103);
HashMultiset<FakePeer> peers = HashMultiset.create(CAPACITY);
for (int i = 0; i < CAPACITY; ++i) {
FakePeer peer = new FakePeer(dnId, false);
@@ -257,7 +254,7 @@ public class TestPeerCache {
PeerCache cache = new PeerCache(CAPACITY, 100000);
DatanodeID dnId = new DatanodeID("192.168.0.1",
"fakehostname", "fake_storage_id",
- 100, 101, 102);
+ 100, 101, 102, 103);
HashMultiset<FakePeer> peers = HashMultiset.create(CAPACITY);
for (int i = 0; i < CAPACITY; ++i) {
FakePeer peer = new FakePeer(dnId, i == CAPACITY - 1);
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java?rev=1532952&r1=1532951&r2=1532952&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java Thu Oct 17 02:14:33 2013
@@ -577,4 +577,64 @@ public class TestShortCircuitLocalRead {
System.out.println("Iteration " + iteration + " took " + (end - start));
fs.delete(file1, false);
}
+
+ public void testReadWithRemoteBlockReader() throws IOException, InterruptedException {
+ doTestShortCircuitReadWithRemoteBlockReader(true, 3*blockSize+100, getCurrentUser(), 0, false);
+ }
+
+ /**
+ * Test that file data can be read by reading the block
+ * through RemoteBlockReader
+ * @throws IOException
+ */
+ public void doTestShortCircuitReadWithRemoteBlockReader(boolean ignoreChecksum, int size, String shortCircuitUser,
+ int readOffset, boolean shortCircuitFails) throws IOException, InterruptedException {
+ Configuration conf = new Configuration();
+ conf.setBoolean(DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADER, true);
+ conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY, true);
+
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1)
+ .format(true).build();
+ FileSystem fs = cluster.getFileSystem();
+ // check that / exists
+ Path path = new Path("/");
+ URI uri = cluster.getURI();
+ assertTrue("/ should be a directory", fs.getFileStatus(path)
+ .isDirectory() == true);
+
+ byte[] fileData = AppendTestUtil.randomBytes(seed, size);
+ Path file1 = new Path("filelocal.dat");
+ FSDataOutputStream stm = createFile(fs, file1, 1);
+
+ stm.write(fileData);
+ stm.close();
+ try {
+ checkFileContent(uri, file1, fileData, readOffset, shortCircuitUser, conf, shortCircuitFails);
+ //RemoteBlockReader have unsupported method read(ByteBuffer bf)
+ assertTrue("RemoteBlockReader unsupported method read(ByteBuffer bf) error",
+ checkUnsupportedMethod(fs, file1, fileData, readOffset));
+ } catch(IOException e) {
+ throw new IOException("doTestShortCircuitReadWithRemoteBlockReader ex error ", e);
+ } catch(InterruptedException inEx) {
+ throw inEx;
+ } finally {
+ fs.close();
+ cluster.shutdown();
+ }
+ }
+
+ private boolean checkUnsupportedMethod(FileSystem fs, Path file,
+ byte[] expected, int readOffset) throws IOException {
+ HdfsDataInputStream stm = (HdfsDataInputStream)fs.open(file);
+ ByteBuffer actual = ByteBuffer.allocateDirect(expected.length - readOffset);
+ IOUtils.skipFully(stm, readOffset);
+ try {
+ stm.read(actual);
+ } catch(UnsupportedOperationException unex) {
+ return true;
+ }
+ return false;
+ }
+
+
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java?rev=1532952&r1=1532951&r2=1532952&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java Thu Oct 17 02:14:33 2013
@@ -31,6 +31,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
@@ -39,9 +40,11 @@ import org.apache.hadoop.hdfs.NameNodePr
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyWithNodeGroup;
import org.apache.hadoop.net.NetworkTopology;
+import org.apache.hadoop.net.NetworkTopologyWithNodeGroup;
+import org.junit.Assert;
import org.junit.Test;
-import junit.framework.Assert;
/**
* This class tests if a balancer schedules tasks correctly.
@@ -75,10 +78,9 @@ public class TestBalancerWithNodeGroup {
Configuration conf = new HdfsConfiguration();
TestBalancer.initConf(conf);
conf.set(CommonConfigurationKeysPublic.NET_TOPOLOGY_IMPL_KEY,
- "org.apache.hadoop.net.NetworkTopologyWithNodeGroup");
- conf.set("dfs.block.replicator.classname",
- "org.apache.hadoop.hdfs.server.blockmanagement." +
- "BlockPlacementPolicyWithNodeGroup");
+ NetworkTopologyWithNodeGroup.class.getName());
+ conf.set(DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY,
+ BlockPlacementPolicyWithNodeGroup.class.getName());
return conf;
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCachedBlocksList.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCachedBlocksList.java?rev=1532952&r1=1532951&r2=1532952&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCachedBlocksList.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCachedBlocksList.java Thu Oct 17 02:14:33 2013
@@ -36,7 +36,7 @@ public class TestCachedBlocksList {
@Test(timeout=60000)
public void testSingleList() {
DatanodeDescriptor dn = new DatanodeDescriptor(
- new DatanodeID("127.0.0.1", "localhost", "abcd", 5000, 5001, 5002));
+ new DatanodeID("127.0.0.1", "localhost", "abcd", 5000, 5001, 5002, 5003));
CachedBlock[] blocks = new CachedBlock[] {
new CachedBlock(0L, (short)1, true),
new CachedBlock(1L, (short)1, true),
@@ -124,9 +124,9 @@ public class TestCachedBlocksList {
public void testMultipleLists() {
DatanodeDescriptor[] datanodes = new DatanodeDescriptor[] {
new DatanodeDescriptor(
- new DatanodeID("127.0.0.1", "localhost", "abcd", 5000, 5001, 5002)),
+ new DatanodeID("127.0.0.1", "localhost", "abcd", 5000, 5001, 5002, 5003)),
new DatanodeDescriptor(
- new DatanodeID("127.0.1.1", "localhost", "efgh", 6000, 6001, 6002)),
+ new DatanodeID("127.0.1.1", "localhost", "efgh", 6000, 6001, 6002, 6003)),
};
CachedBlocksList[] lists = new CachedBlocksList[] {
datanodes[0].getPendingCached(),
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestRBWBlockInvalidation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestRBWBlockInvalidation.java?rev=1532952&r1=1532951&r2=1532952&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestRBWBlockInvalidation.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestRBWBlockInvalidation.java Thu Oct 17 02:14:33 2013
@@ -157,8 +157,8 @@ public class TestRBWBlockInvalidation {
// in the context of the test, whereas a random one is more accurate
// to what is seen in real clusters (nodes have random amounts of free
// space)
- conf.setClass("dfs.block.replicator.classname", RandomDeleterPolicy.class,
- BlockPlacementPolicy.class);
+ conf.setClass(DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY,
+ RandomDeleterPolicy.class, BlockPlacementPolicy.class);
// Speed up the test a bit with faster heartbeats.
conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java?rev=1532952&r1=1532951&r2=1532952&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java Thu Oct 17 02:14:33 2013
@@ -29,9 +29,11 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Random;
+import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.ContentSummary;
@@ -137,30 +139,25 @@ public class TestReplicationPolicy {
0L, 0L, 4, 0); // overloaded
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename, 0, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(0);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename, 1, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(1);
assertEquals(targets.length, 1);
assertEquals(targets[0], dataNodes[0]);
- targets = replicator.chooseTarget(filename,
- 2, dataNodes[0], new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(2);
assertEquals(targets.length, 2);
assertEquals(targets[0], dataNodes[0]);
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
- targets = replicator.chooseTarget(filename, 3, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(3);
assertEquals(targets.length, 3);
assertEquals(targets[0], dataNodes[0]);
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
assertTrue(cluster.isOnSameRack(targets[1], targets[2]));
- targets = replicator.chooseTarget(filename, 4, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(4);
assertEquals(targets.length, 4);
assertEquals(targets[0], dataNodes[0]);
assertTrue(cluster.isOnSameRack(targets[1], targets[2]) ||
@@ -172,15 +169,38 @@ public class TestReplicationPolicy {
HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 0L, 0L, 0, 0);
}
+ private static DatanodeDescriptor[] chooseTarget(int numOfReplicas) {
+ return chooseTarget(numOfReplicas, dataNodes[0]);
+ }
+
+ private static DatanodeDescriptor[] chooseTarget(int numOfReplicas,
+ DatanodeDescriptor writer) {
+ return chooseTarget(numOfReplicas, writer,
+ new ArrayList<DatanodeDescriptor>());
+ }
+
+ private static DatanodeDescriptor[] chooseTarget(int numOfReplicas,
+ List<DatanodeDescriptor> chosenNodes) {
+ return chooseTarget(numOfReplicas, dataNodes[0], chosenNodes);
+ }
+
+ private static DatanodeDescriptor[] chooseTarget(int numOfReplicas,
+ DatanodeDescriptor writer, List<DatanodeDescriptor> chosenNodes) {
+ return chooseTarget(numOfReplicas, writer, chosenNodes, null);
+ }
+
+ private static DatanodeDescriptor[] chooseTarget(int numOfReplicas,
+ List<DatanodeDescriptor> chosenNodes, Set<Node> excludedNodes) {
+ return chooseTarget(numOfReplicas, dataNodes[0], chosenNodes, excludedNodes);
+ }
+
private static DatanodeDescriptor[] chooseTarget(
- BlockPlacementPolicyDefault policy,
int numOfReplicas,
DatanodeDescriptor writer,
List<DatanodeDescriptor> chosenNodes,
- HashMap<Node, Node> excludedNodes,
- long blocksize) {
- return policy.chooseTarget(numOfReplicas, writer, chosenNodes, false,
- excludedNodes, blocksize);
+ Set<Node> excludedNodes) {
+ return replicator.chooseTarget(filename, numOfReplicas, writer, chosenNodes,
+ false, excludedNodes, BLOCK_SIZE);
}
/**
@@ -193,39 +213,34 @@ public class TestReplicationPolicy {
*/
@Test
public void testChooseTarget2() throws Exception {
- HashMap<Node, Node> excludedNodes;
+ Set<Node> excludedNodes;
DatanodeDescriptor[] targets;
- BlockPlacementPolicyDefault repl = (BlockPlacementPolicyDefault)replicator;
List<DatanodeDescriptor> chosenNodes = new ArrayList<DatanodeDescriptor>();
- excludedNodes = new HashMap<Node, Node>();
- excludedNodes.put(dataNodes[1], dataNodes[1]);
- targets = chooseTarget(repl, 0, dataNodes[0], chosenNodes, excludedNodes,
- BLOCK_SIZE);
+ excludedNodes = new HashSet<Node>();
+ excludedNodes.add(dataNodes[1]);
+ targets = chooseTarget(0, chosenNodes, excludedNodes);
assertEquals(targets.length, 0);
excludedNodes.clear();
chosenNodes.clear();
- excludedNodes.put(dataNodes[1], dataNodes[1]);
- targets = chooseTarget(repl, 1, dataNodes[0], chosenNodes, excludedNodes,
- BLOCK_SIZE);
+ excludedNodes.add(dataNodes[1]);
+ targets = chooseTarget(1, chosenNodes, excludedNodes);
assertEquals(targets.length, 1);
assertEquals(targets[0], dataNodes[0]);
excludedNodes.clear();
chosenNodes.clear();
- excludedNodes.put(dataNodes[1], dataNodes[1]);
- targets = chooseTarget(repl, 2, dataNodes[0], chosenNodes, excludedNodes,
- BLOCK_SIZE);
+ excludedNodes.add(dataNodes[1]);
+ targets = chooseTarget(2, chosenNodes, excludedNodes);
assertEquals(targets.length, 2);
assertEquals(targets[0], dataNodes[0]);
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
excludedNodes.clear();
chosenNodes.clear();
- excludedNodes.put(dataNodes[1], dataNodes[1]);
- targets = chooseTarget(repl, 3, dataNodes[0], chosenNodes, excludedNodes,
- BLOCK_SIZE);
+ excludedNodes.add(dataNodes[1]);
+ targets = chooseTarget(3, chosenNodes, excludedNodes);
assertEquals(targets.length, 3);
assertEquals(targets[0], dataNodes[0]);
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
@@ -233,9 +248,8 @@ public class TestReplicationPolicy {
excludedNodes.clear();
chosenNodes.clear();
- excludedNodes.put(dataNodes[1], dataNodes[1]);
- targets = chooseTarget(repl, 4, dataNodes[0], chosenNodes, excludedNodes,
- BLOCK_SIZE);
+ excludedNodes.add(dataNodes[1]);
+ targets = chooseTarget(4, chosenNodes, excludedNodes);
assertEquals(targets.length, 4);
assertEquals(targets[0], dataNodes[0]);
for(int i=1; i<4; i++) {
@@ -247,9 +261,9 @@ public class TestReplicationPolicy {
excludedNodes.clear();
chosenNodes.clear();
- excludedNodes.put(dataNodes[1], dataNodes[1]);
+ excludedNodes.add(dataNodes[1]);
chosenNodes.add(dataNodes[2]);
- targets = repl.chooseTarget(1, dataNodes[0], chosenNodes, true,
+ targets = replicator.chooseTarget(filename, 1, dataNodes[0], chosenNodes, true,
excludedNodes, BLOCK_SIZE);
System.out.println("targets=" + Arrays.asList(targets));
assertEquals(2, targets.length);
@@ -276,30 +290,25 @@ public class TestReplicationPolicy {
0L, 0L, 0, 0); // no space
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename, 0, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(0);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename, 1, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(1);
assertEquals(targets.length, 1);
assertEquals(targets[0], dataNodes[1]);
- targets = replicator.chooseTarget(filename, 2, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(2);
assertEquals(targets.length, 2);
assertEquals(targets[0], dataNodes[1]);
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
- targets = replicator.chooseTarget(filename, 3, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(3);
assertEquals(targets.length, 3);
assertEquals(targets[0], dataNodes[1]);
assertTrue(cluster.isOnSameRack(targets[1], targets[2]));
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
- targets = replicator.chooseTarget(filename, 4, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(4);
assertEquals(targets.length, 4);
assertEquals(targets[0], dataNodes[1]);
for(int i=1; i<4; i++) {
@@ -332,23 +341,19 @@ public class TestReplicationPolicy {
}
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename, 0, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(0);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename, 1, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(1);
assertEquals(targets.length, 1);
assertFalse(cluster.isOnSameRack(targets[0], dataNodes[0]));
- targets = replicator.chooseTarget(filename, 2, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(2);
assertEquals(targets.length, 2);
assertFalse(cluster.isOnSameRack(targets[0], dataNodes[0]));
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
- targets = replicator.chooseTarget(filename, 3, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(3);
assertEquals(targets.length, 3);
for(int i=0; i<3; i++) {
assertFalse(cluster.isOnSameRack(targets[i], dataNodes[0]));
@@ -377,21 +382,17 @@ public class TestReplicationPolicy {
DFSTestUtil.getDatanodeDescriptor("7.7.7.7", "/d2/r4");
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename, 0, writerDesc,
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(0, writerDesc);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename, 1, writerDesc,
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(1, writerDesc);
assertEquals(targets.length, 1);
- targets = replicator.chooseTarget(filename, 2, writerDesc,
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(2, writerDesc);
assertEquals(targets.length, 2);
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
- targets = replicator.chooseTarget(filename, 3, writerDesc,
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(3, writerDesc);
assertEquals(targets.length, 3);
assertTrue(cluster.isOnSameRack(targets[1], targets[2]));
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
@@ -435,9 +436,7 @@ public class TestReplicationPolicy {
// try to choose NUM_OF_DATANODES which is more than actually available
// nodes.
- DatanodeDescriptor[] targets = replicator.chooseTarget(filename,
- NUM_OF_DATANODES, dataNodes[0], new ArrayList<DatanodeDescriptor>(),
- BLOCK_SIZE);
+ DatanodeDescriptor[] targets = chooseTarget(NUM_OF_DATANODES);
assertEquals(targets.length, NUM_OF_DATANODES - 2);
final List<LoggingEvent> log = appender.getLog();
@@ -480,17 +479,14 @@ public class TestReplicationPolicy {
DatanodeDescriptor[] targets;
// We set the datanode[0] as stale, thus should choose datanode[1] since
// datanode[1] is on the same rack with datanode[0] (writer)
- targets = replicator.chooseTarget(filename, 1, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(1);
assertEquals(targets.length, 1);
assertEquals(targets[0], dataNodes[1]);
- HashMap<Node, Node> excludedNodes = new HashMap<Node, Node>();
- excludedNodes.put(dataNodes[1], dataNodes[1]);
+ Set<Node> excludedNodes = new HashSet<Node>();
+ excludedNodes.add(dataNodes[1]);
List<DatanodeDescriptor> chosenNodes = new ArrayList<DatanodeDescriptor>();
- BlockPlacementPolicyDefault repl = (BlockPlacementPolicyDefault)replicator;
- targets = chooseTarget(repl, 1, dataNodes[0], chosenNodes, excludedNodes,
- BLOCK_SIZE);
+ targets = chooseTarget(1, chosenNodes, excludedNodes);
assertEquals(targets.length, 1);
assertFalse(cluster.isOnSameRack(targets[0], dataNodes[0]));
@@ -517,33 +513,27 @@ public class TestReplicationPolicy {
namenode.getNamesystem().getBlockManager()
.getDatanodeManager().getHeartbeatManager().heartbeatCheck();
- DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename, 0, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ DatanodeDescriptor[] targets = chooseTarget(0);
assertEquals(targets.length, 0);
// Since we have 6 datanodes total, stale nodes should
// not be returned until we ask for more than 3 targets
- targets = replicator.chooseTarget(filename, 1, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(1);
assertEquals(targets.length, 1);
assertFalse(containsWithinRange(targets[0], dataNodes, 0, 2));
- targets = replicator.chooseTarget(filename, 2, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(2);
assertEquals(targets.length, 2);
assertFalse(containsWithinRange(targets[0], dataNodes, 0, 2));
assertFalse(containsWithinRange(targets[1], dataNodes, 0, 2));
- targets = replicator.chooseTarget(filename, 3, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(3);
assertEquals(targets.length, 3);
assertTrue(containsWithinRange(targets[0], dataNodes, 3, 5));
assertTrue(containsWithinRange(targets[1], dataNodes, 3, 5));
assertTrue(containsWithinRange(targets[2], dataNodes, 3, 5));
- targets = replicator.chooseTarget(filename, 4, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(4);
assertEquals(targets.length, 4);
assertTrue(containsWithinRange(dataNodes[3], targets, 0, 3));
assertTrue(containsWithinRange(dataNodes[4], targets, 0, 3));
@@ -596,7 +586,8 @@ public class TestReplicationPolicy {
BlockPlacementPolicy replicator = miniCluster.getNameNode()
.getNamesystem().getBlockManager().getBlockPlacementPolicy();
DatanodeDescriptor[] targets = replicator.chooseTarget(filename, 3,
- staleNodeInfo, new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ staleNodeInfo, new ArrayList<DatanodeDescriptor>(), false, null, BLOCK_SIZE);
+
assertEquals(targets.length, 3);
assertFalse(cluster.isOnSameRack(targets[0], staleNodeInfo));
@@ -620,7 +611,7 @@ public class TestReplicationPolicy {
.getDatanodeManager().shouldAvoidStaleDataNodesForWrite());
// Call chooseTarget
targets = replicator.chooseTarget(filename, 3,
- staleNodeInfo, new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ staleNodeInfo, new ArrayList<DatanodeDescriptor>(), false, null, BLOCK_SIZE);
assertEquals(targets.length, 3);
assertTrue(cluster.isOnSameRack(targets[0], staleNodeInfo));
@@ -642,8 +633,7 @@ public class TestReplicationPolicy {
assertTrue(miniCluster.getNameNode().getNamesystem().getBlockManager()
.getDatanodeManager().shouldAvoidStaleDataNodesForWrite());
// Call chooseTarget
- targets = replicator.chooseTarget(filename, 3,
- staleNodeInfo, new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(3, staleNodeInfo);
assertEquals(targets.length, 3);
assertFalse(cluster.isOnSameRack(targets[0], staleNodeInfo));
} finally {
@@ -664,23 +654,19 @@ public class TestReplicationPolicy {
chosenNodes.add(dataNodes[0]);
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename,
- 0, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(0, chosenNodes);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename,
- 1, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(1, chosenNodes);
assertEquals(targets.length, 1);
assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0]));
- targets = replicator.chooseTarget(filename,
- 2, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(2, chosenNodes);
assertEquals(targets.length, 2);
assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0]));
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
- targets = replicator.chooseTarget(filename,
- 3, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(3, chosenNodes);
assertEquals(targets.length, 3);
assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0]));
assertFalse(cluster.isOnSameRack(targets[0], targets[2]));
@@ -700,17 +686,14 @@ public class TestReplicationPolicy {
chosenNodes.add(dataNodes[1]);
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename,
- 0, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(0, chosenNodes);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename,
- 1, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(1, chosenNodes);
assertEquals(targets.length, 1);
assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0]));
- targets = replicator.chooseTarget(filename,
- 2, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(2, chosenNodes);
assertEquals(targets.length, 2);
assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0]));
assertFalse(cluster.isOnSameRack(dataNodes[0], targets[1]));
@@ -730,29 +713,24 @@ public class TestReplicationPolicy {
chosenNodes.add(dataNodes[2]);
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename,
- 0, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(0, chosenNodes);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename,
- 1, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(1, chosenNodes);
assertEquals(targets.length, 1);
assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0]));
assertFalse(cluster.isOnSameRack(dataNodes[2], targets[0]));
- targets = replicator.chooseTarget(filename,
- 1, dataNodes[2], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(1, dataNodes[2], chosenNodes);
assertEquals(targets.length, 1);
assertTrue(cluster.isOnSameRack(dataNodes[2], targets[0]));
assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0]));
- targets = replicator.chooseTarget(filename,
- 2, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(2, chosenNodes);
assertEquals(targets.length, 2);
assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0]));
- targets = replicator.chooseTarget(filename,
- 2, dataNodes[2], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(2, dataNodes[2], chosenNodes);
assertEquals(targets.length, 2);
assertTrue(cluster.isOnSameRack(dataNodes[2], targets[0]));
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java?rev=1532952&r1=1532951&r2=1532952&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java Thu Oct 17 02:14:33 2013
@@ -39,6 +39,7 @@ import org.apache.hadoop.hdfs.HdfsConfig
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.net.NetworkTopology;
+import org.apache.hadoop.net.NetworkTopologyWithNodeGroup;
import org.apache.hadoop.net.Node;
import org.apache.hadoop.test.PathUtils;
import org.junit.After;
@@ -101,10 +102,10 @@ public class TestReplicationPolicyWithNo
FileSystem.setDefaultUri(CONF, "hdfs://localhost:0");
CONF.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
// Set properties to make HDFS aware of NodeGroup.
- CONF.set("dfs.block.replicator.classname",
- "org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyWithNodeGroup");
+ CONF.set(DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY,
+ BlockPlacementPolicyWithNodeGroup.class.getName());
CONF.set(CommonConfigurationKeysPublic.NET_TOPOLOGY_IMPL_KEY,
- "org.apache.hadoop.net.NetworkTopologyWithNodeGroup");
+ NetworkTopologyWithNodeGroup.class.getName());
File baseDir = PathUtils.getTestDir(TestReplicationPolicyWithNodeGroup.class);
@@ -156,6 +157,35 @@ public class TestReplicationPolicyWithNo
return true;
}
+ private DatanodeDescriptor[] chooseTarget(int numOfReplicas) {
+ return chooseTarget(numOfReplicas, dataNodes[0]);
+ }
+
+ private DatanodeDescriptor[] chooseTarget(int numOfReplicas,
+ DatanodeDescriptor writer) {
+ return chooseTarget(numOfReplicas, writer,
+ new ArrayList<DatanodeDescriptor>());
+ }
+
+ private DatanodeDescriptor[] chooseTarget(int numOfReplicas,
+ List<DatanodeDescriptor> chosenNodes) {
+ return chooseTarget(numOfReplicas, dataNodes[0], chosenNodes);
+ }
+
+ private DatanodeDescriptor[] chooseTarget(int numOfReplicas,
+ DatanodeDescriptor writer, List<DatanodeDescriptor> chosenNodes) {
+ return chooseTarget(numOfReplicas, writer, chosenNodes, null);
+ }
+
+ private DatanodeDescriptor[] chooseTarget(
+ int numOfReplicas,
+ DatanodeDescriptor writer,
+ List<DatanodeDescriptor> chosenNodes,
+ Set<Node> excludedNodes) {
+ return replicator.chooseTarget(filename, numOfReplicas, writer, chosenNodes,
+ false, excludedNodes, BLOCK_SIZE);
+ }
+
/**
* In this testcase, client is dataNodes[0]. So the 1st replica should be
* placed on dataNodes[0], the 2nd replica should be placed on
@@ -173,31 +203,26 @@ public class TestReplicationPolicyWithNo
0L, 0L, 4, 0); // overloaded
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename, 0, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(0);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename, 1, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(1);
assertEquals(targets.length, 1);
assertEquals(targets[0], dataNodes[0]);
- targets = replicator.chooseTarget(filename, 2, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(2);
assertEquals(targets.length, 2);
assertEquals(targets[0], dataNodes[0]);
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
- targets = replicator.chooseTarget(filename, 3, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(3);
assertEquals(targets.length, 3);
assertEquals(targets[0], dataNodes[0]);
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
assertTrue(cluster.isOnSameRack(targets[1], targets[2]));
assertFalse(cluster.isOnSameNodeGroup(targets[1], targets[2]));
- targets = replicator.chooseTarget(filename, 4, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(4);
assertEquals(targets.length, 4);
assertEquals(targets[0], dataNodes[0]);
assertTrue(cluster.isOnSameRack(targets[1], targets[2]) ||
@@ -228,15 +253,14 @@ public class TestReplicationPolicyWithNo
* @throws Exception
*/
@Test
- public void testChooseTarget2() throws Exception {
- HashMap<Node, Node> excludedNodes;
+ public void testChooseTarget2() throws Exception {
DatanodeDescriptor[] targets;
BlockPlacementPolicyDefault repl = (BlockPlacementPolicyDefault)replicator;
List<DatanodeDescriptor> chosenNodes = new ArrayList<DatanodeDescriptor>();
- excludedNodes = new HashMap<Node, Node>();
- excludedNodes.put(dataNodes[1], dataNodes[1]);
- targets = repl.chooseTarget(4, dataNodes[0], chosenNodes, false,
+ Set<Node> excludedNodes = new HashSet<Node>();
+ excludedNodes.add(dataNodes[1]);
+ targets = repl.chooseTarget(filename, 4, dataNodes[0], chosenNodes, false,
excludedNodes, BLOCK_SIZE);
assertEquals(targets.length, 4);
assertEquals(targets[0], dataNodes[0]);
@@ -251,9 +275,9 @@ public class TestReplicationPolicyWithNo
excludedNodes.clear();
chosenNodes.clear();
- excludedNodes.put(dataNodes[1], dataNodes[1]);
+ excludedNodes.add(dataNodes[1]);
chosenNodes.add(dataNodes[2]);
- targets = repl.chooseTarget(1, dataNodes[0], chosenNodes, true,
+ targets = repl.chooseTarget(filename, 1, dataNodes[0], chosenNodes, true,
excludedNodes, BLOCK_SIZE);
System.out.println("targets=" + Arrays.asList(targets));
assertEquals(2, targets.length);
@@ -280,30 +304,25 @@ public class TestReplicationPolicyWithNo
0L, 0L, 0, 0); // no space
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename, 0, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(0);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename, 1, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(1);
assertEquals(targets.length, 1);
assertEquals(targets[0], dataNodes[1]);
- targets = replicator.chooseTarget(filename, 2, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(2);
assertEquals(targets.length, 2);
assertEquals(targets[0], dataNodes[1]);
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
- targets = replicator.chooseTarget(filename, 3, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(3);
assertEquals(targets.length, 3);
assertEquals(targets[0], dataNodes[1]);
assertTrue(cluster.isOnSameRack(targets[1], targets[2]));
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
- targets = replicator.chooseTarget(filename, 4, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(4);
assertEquals(targets.length, 4);
assertEquals(targets[0], dataNodes[1]);
assertTrue(cluster.isNodeGroupAware());
@@ -335,23 +354,19 @@ public class TestReplicationPolicyWithNo
}
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename, 0, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(0);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename, 1, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(1);
assertEquals(targets.length, 1);
assertFalse(cluster.isOnSameRack(targets[0], dataNodes[0]));
- targets = replicator.chooseTarget(filename, 2, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(2);
assertEquals(targets.length, 2);
assertFalse(cluster.isOnSameRack(targets[0], dataNodes[0]));
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
- targets = replicator.chooseTarget(filename, 3, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(3);
assertEquals(targets.length, 3);
for(int i=0; i<3; i++) {
assertFalse(cluster.isOnSameRack(targets[i], dataNodes[0]));
@@ -373,21 +388,17 @@ public class TestReplicationPolicyWithNo
public void testChooseTarget5() throws Exception {
setupDataNodeCapacity();
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename, 0, NODE,
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(0, NODE);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename, 1, NODE,
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(1, NODE);
assertEquals(targets.length, 1);
- targets = replicator.chooseTarget(filename, 2, NODE,
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(2, NODE);
assertEquals(targets.length, 2);
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
- targets = replicator.chooseTarget(filename, 3, NODE,
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(3, NODE);
assertEquals(targets.length, 3);
assertTrue(cluster.isOnSameRack(targets[1], targets[2]));
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
@@ -408,23 +419,19 @@ public class TestReplicationPolicyWithNo
chosenNodes.add(dataNodes[0]);
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename,
- 0, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(0, chosenNodes);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename,
- 1, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(1, chosenNodes);
assertEquals(targets.length, 1);
assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0]));
- targets = replicator.chooseTarget(filename,
- 2, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(2, chosenNodes);
assertEquals(targets.length, 2);
assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0]));
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
- targets = replicator.chooseTarget(filename,
- 3, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(3, chosenNodes);
assertEquals(targets.length, 3);
assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0]));
assertFalse(cluster.isOnSameNodeGroup(dataNodes[0], targets[0]));
@@ -446,17 +453,14 @@ public class TestReplicationPolicyWithNo
chosenNodes.add(dataNodes[1]);
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename,
- 0, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(0, chosenNodes);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename,
- 1, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(1, chosenNodes);
assertEquals(targets.length, 1);
assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0]));
- targets = replicator.chooseTarget(filename,
- 2, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(2, chosenNodes);
assertEquals(targets.length, 2);
assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0]) &&
cluster.isOnSameRack(dataNodes[0], targets[1]));
@@ -477,30 +481,26 @@ public class TestReplicationPolicyWithNo
chosenNodes.add(dataNodes[3]);
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename,
- 0, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(0, chosenNodes);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename,
- 1, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(1, chosenNodes);
assertEquals(targets.length, 1);
assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0]));
assertFalse(cluster.isOnSameRack(dataNodes[3], targets[0]));
- targets = replicator.chooseTarget(filename,
- 1, dataNodes[3], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(1, dataNodes[3], chosenNodes);
assertEquals(targets.length, 1);
assertTrue(cluster.isOnSameRack(dataNodes[3], targets[0]));
assertFalse(cluster.isOnSameNodeGroup(dataNodes[3], targets[0]));
assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0]));
- targets = replicator.chooseTarget(filename,
- 2, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(2, chosenNodes);
assertEquals(targets.length, 2);
assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0]));
assertFalse(cluster.isOnSameNodeGroup(dataNodes[0], targets[0]));
- targets = replicator.chooseTarget(filename,
- 2, dataNodes[3], chosenNodes, BLOCK_SIZE);
+
+ targets = chooseTarget(2, dataNodes[3], chosenNodes);
assertEquals(targets.length, 2);
assertTrue(cluster.isOnSameRack(dataNodes[3], targets[0]));
}
@@ -586,21 +586,17 @@ public class TestReplicationPolicyWithNo
}
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename, 0, dataNodesInBoundaryCase[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(0, dataNodesInBoundaryCase[0]);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename, 1, dataNodesInBoundaryCase[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(1, dataNodesInBoundaryCase[0]);
assertEquals(targets.length, 1);
- targets = replicator.chooseTarget(filename, 2, dataNodesInBoundaryCase[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(2, dataNodesInBoundaryCase[0]);
assertEquals(targets.length, 2);
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
- targets = replicator.chooseTarget(filename, 3, dataNodesInBoundaryCase[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(3, dataNodesInBoundaryCase[0]);
assertEquals(targets.length, 3);
assertTrue(checkTargetsOnDifferentNodeGroup(targets));
}
@@ -623,8 +619,7 @@ public class TestReplicationPolicyWithNo
chosenNodes.add(dataNodesInBoundaryCase[0]);
chosenNodes.add(dataNodesInBoundaryCase[5]);
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename, 1, dataNodesInBoundaryCase[0],
- chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(1, dataNodesInBoundaryCase[0], chosenNodes);
assertFalse(cluster.isOnSameNodeGroup(targets[0],
dataNodesInBoundaryCase[0]));
assertFalse(cluster.isOnSameNodeGroup(targets[0],
@@ -663,14 +658,12 @@ public class TestReplicationPolicyWithNo
DatanodeDescriptor[] targets;
// Test normal case -- 3 replicas
- targets = replicator.chooseTarget(filename, 3, dataNodesInMoreTargetsCase[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(3, dataNodesInMoreTargetsCase[0]);
assertEquals(targets.length, 3);
assertTrue(checkTargetsOnDifferentNodeGroup(targets));
// Test special case -- replica number over node groups.
- targets = replicator.chooseTarget(filename, 10, dataNodesInMoreTargetsCase[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(10, dataNodesInMoreTargetsCase[0]);
assertTrue(checkTargetsOnDifferentNodeGroup(targets));
// Verify it only can find 6 targets for placing replicas.
assertEquals(targets.length, 6);
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java?rev=1532952&r1=1532951&r2=1532952&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java Thu Oct 17 02:14:33 2013
@@ -17,23 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.common;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-import static org.mockito.Mockito.doAnswer;
-
-import java.io.IOException;
-import java.io.StringReader;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-
-import javax.servlet.ServletContext;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.jsp.JspWriter;
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-
-
+import com.google.common.base.Strings;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
@@ -43,6 +27,8 @@ import org.apache.hadoop.hdfs.server.blo
import org.apache.hadoop.hdfs.server.namenode.NameNodeHttpServer;
import org.apache.hadoop.hdfs.web.resources.DoAsParam;
import org.apache.hadoop.hdfs.web.resources.UserParam;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
@@ -54,11 +40,27 @@ import org.apache.hadoop.security.token.
import org.junit.Assert;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
+import javax.servlet.ServletContext;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.jsp.JspWriter;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import java.io.IOException;
+import java.io.StringReader;
+import java.net.InetSocketAddress;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.*;
+
public class TestJspHelper {
@@ -446,9 +448,9 @@ public class TestJspHelper {
@Test
public void testSortNodeByFields() throws Exception {
DatanodeID dnId1 = new DatanodeID("127.0.0.1", "localhost1", "storage1",
- 1234, 2345, 3456);
+ 1234, 2345, 3456, 4567);
DatanodeID dnId2 = new DatanodeID("127.0.0.2", "localhost2", "storage2",
- 1235, 2346, 3457);
+ 1235, 2346, 3457, 4568);
DatanodeDescriptor dnDesc1 = new DatanodeDescriptor(dnId1, "rack1", 1024,
100, 924, 100, 5l, 3l, 10, 2);
DatanodeDescriptor dnDesc2 = new DatanodeDescriptor(dnId2, "rack2", 2500,
@@ -480,5 +482,136 @@ public class TestJspHelper {
JspHelper.sortNodeList(live, "pcbpused", "DSC");
Assert.assertEquals(dnDesc1, live.get(0));
Assert.assertEquals(dnDesc2, live.get(1));
+
+ //unexisted field comparition is d1.getHostName().compareTo(d2.getHostName());
+ JspHelper.sortNodeList(live, "unexists", "ASC");
+ Assert.assertEquals(dnDesc1, live.get(0));
+ Assert.assertEquals(dnDesc2, live.get(1));
+
+ JspHelper.sortNodeList(live, "unexists", "DSC");
+ Assert.assertEquals(dnDesc2, live.get(0));
+ Assert.assertEquals(dnDesc1, live.get(1));
+
+ // test sorting by capacity
+ JspHelper.sortNodeList(live, "capacity", "ASC");
+ Assert.assertEquals(dnDesc1, live.get(0));
+ Assert.assertEquals(dnDesc2, live.get(1));
+
+ JspHelper.sortNodeList(live, "capacity", "DSC");
+ Assert.assertEquals(dnDesc2, live.get(0));
+ Assert.assertEquals(dnDesc1, live.get(1));
+
+ // test sorting by used
+ JspHelper.sortNodeList(live, "used", "ASC");
+ Assert.assertEquals(dnDesc1, live.get(0));
+ Assert.assertEquals(dnDesc2, live.get(1));
+
+ JspHelper.sortNodeList(live, "used", "DSC");
+ Assert.assertEquals(dnDesc2, live.get(0));
+ Assert.assertEquals(dnDesc1, live.get(1));
+
+ // test sorting by nondfsused
+ JspHelper.sortNodeList(live, "nondfsused", "ASC");
+ Assert.assertEquals(dnDesc1, live.get(0));
+ Assert.assertEquals(dnDesc2, live.get(1));
+
+ JspHelper.sortNodeList(live, "nondfsused", "DSC");
+ Assert.assertEquals(dnDesc2, live.get(0));
+ Assert.assertEquals(dnDesc1, live.get(1));
+
+ // test sorting by remaining
+ JspHelper.sortNodeList(live, "remaining", "ASC");
+ Assert.assertEquals(dnDesc1, live.get(0));
+ Assert.assertEquals(dnDesc2, live.get(1));
+
+ JspHelper.sortNodeList(live, "remaining", "DSC");
+ Assert.assertEquals(dnDesc2, live.get(0));
+ Assert.assertEquals(dnDesc1, live.get(1));
+ }
+
+ @Test
+ public void testPrintMethods() throws IOException {
+ JspWriter out = mock(JspWriter.class);
+ HttpServletRequest req = mock(HttpServletRequest.class);
+
+ final StringBuffer buffer = new StringBuffer();
+
+ ArgumentCaptor<String> arg = ArgumentCaptor.forClass(String.class);
+ doAnswer(new Answer<Object>() {
+ @Override
+ public Object answer(InvocationOnMock invok) {
+ Object[] args = invok.getArguments();
+ buffer.append((String)args[0]);
+ return null;
+ }
+ }).when(out).print(arg.capture());
+
+
+ JspHelper.createTitle(out, req, "testfile.txt");
+ Mockito.verify(out, Mockito.times(1)).print(Mockito.anyString());
+
+ JspHelper.addTableHeader(out);
+ Mockito.verify(out, Mockito.times(1 + 2)).print(Mockito.anyString());
+
+ JspHelper.addTableRow(out, new String[] {" row11", "row12 "});
+ Mockito.verify(out, Mockito.times(1 + 2 + 4)).print(Mockito.anyString());
+
+ JspHelper.addTableRow(out, new String[] {" row11", "row12 "}, 3);
+ Mockito.verify(out, Mockito.times(1 + 2 + 4 + 4)).print(Mockito.anyString());
+
+ JspHelper.addTableRow(out, new String[] {" row21", "row22"});
+ Mockito.verify(out, Mockito.times(1 + 2 + 4 + 4 + 4)).print(Mockito.anyString());
+
+ JspHelper.addTableFooter(out);
+ Mockito.verify(out, Mockito.times(1 + 2 + 4 + 4 + 4 + 1)).print(Mockito.anyString());
+
+ assertFalse(Strings.isNullOrEmpty(buffer.toString()));
}
+
+ @Test
+ public void testReadWriteReplicaState() {
+ try {
+ DataOutputBuffer out = new DataOutputBuffer();
+ DataInputBuffer in = new DataInputBuffer();
+ for (HdfsServerConstants.ReplicaState repState : HdfsServerConstants.ReplicaState
+ .values()) {
+ repState.write(out);
+ in.reset(out.getData(), out.getLength());
+ HdfsServerConstants.ReplicaState result = HdfsServerConstants.ReplicaState
+ .read(in);
+ assertTrue("testReadWrite error !!!", repState == result);
+ out.reset();
+ in.reset();
+ }
+ } catch (Exception ex) {
+ fail("testReadWrite ex error ReplicaState");
+ }
+ }
+
+ @Test
+ public void testUpgradeStatusReport() {
+ short status = 6;
+ int version = 15;
+ String EXPECTED__NOTF_PATTERN = "Upgrade for version {0} has been completed.\nUpgrade is not finalized.";
+ String EXPECTED_PATTERN = "Upgrade for version {0} is in progress. Status = {1}%";
+
+ UpgradeStatusReport upgradeStatusReport = new UpgradeStatusReport(version,
+ status, true);
+ assertTrue(upgradeStatusReport.getVersion() == version);
+ assertTrue(upgradeStatusReport.getUpgradeStatus() == status);
+ assertTrue(upgradeStatusReport.isFinalized());
+
+ assertEquals(MessageFormat.format(EXPECTED_PATTERN, version, status),
+ upgradeStatusReport.getStatusText(true));
+
+ status += 100;
+ upgradeStatusReport = new UpgradeStatusReport(version, status, false);
+ assertFalse(upgradeStatusReport.isFinalized());
+ assertTrue(upgradeStatusReport.toString().equals(
+ MessageFormat.format(EXPECTED__NOTF_PATTERN, version)));
+ assertTrue(upgradeStatusReport.getStatusText(false).equals(
+ MessageFormat.format(EXPECTED__NOTF_PATTERN, version)));
+ assertTrue(upgradeStatusReport.getStatusText(true).equals(
+ MessageFormat.format(EXPECTED__NOTF_PATTERN, version)));
+ }
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java?rev=1532952&r1=1532951&r2=1532952&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java Thu Oct 17 02:14:33 2013
@@ -115,11 +115,22 @@ public class DataNodeTestUtils {
}
public static void runBlockScannerForBlock(DataNode dn, ExtendedBlock b) {
+ BlockPoolSliceScanner bpScanner = getBlockPoolScanner(dn, b);
+ bpScanner.verifyBlock(b);
+ }
+
+ private static BlockPoolSliceScanner getBlockPoolScanner(DataNode dn,
+ ExtendedBlock b) {
DataBlockScanner scanner = dn.getBlockScanner();
BlockPoolSliceScanner bpScanner = scanner.getBPScanner(b.getBlockPoolId());
- bpScanner.verifyBlock(b);
+ return bpScanner;
}
-
+
+ public static long getLatestScanTime(DataNode dn, ExtendedBlock b) {
+ BlockPoolSliceScanner scanner = getBlockPoolScanner(dn, b);
+ return scanner.getLastScanTime(b.getLocalBlock());
+ }
+
public static void shutdownBlockScanner(DataNode dn) {
if (dn.blockScanner != null) {
dn.blockScanner.shutdown();
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDatanodeJsp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDatanodeJsp.java?rev=1532952&r1=1532951&r2=1532952&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDatanodeJsp.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDatanodeJsp.java Thu Oct 17 02:14:33 2013
@@ -61,9 +61,10 @@ public class TestDatanodeJsp {
InetSocketAddress nnIpcAddress = cluster.getNameNode().getNameNodeAddress();
InetSocketAddress nnHttpAddress = cluster.getNameNode().getHttpAddress();
- int dnInfoPort = cluster.getDataNodes().get(0).getInfoPort();
-
- URL url = new URL("http://localhost:" + dnInfoPort + "/"
+ String base = JspHelper.Url.url("http", cluster.getDataNodes().get(0)
+ .getDatanodeId());
+
+ URL url = new URL(base + "/"
+ "browseDirectory.jsp" + JspHelper.getUrlParam("dir",
URLEncoder.encode(testPath.toString(), "UTF-8"), true)
+ JspHelper.getUrlParam("namenodeInfoPort", Integer
@@ -86,18 +87,21 @@ public class TestDatanodeJsp {
// check whether able to 'Go Back to File View' after tailing the file
regex = "<a.+href=\"(.+?)\">Go\\s*Back\\s*to\\s*File\\s*View\\<\\/a\\>";
assertFileContents(regex, "Go Back to File View");
+
+ regex = "<a href=\"///localhost:" + nnHttpAddress.getPort() + "/dfshealth.jsp\">Go back to DFS home</a>";
+ assertTrue("page should generate DFS home scheme without explicit scheme", viewFilePage.contains(regex));
}
private static void assertFileContents(String regex, String text)
throws IOException {
Pattern compile = Pattern.compile(regex);
Matcher matcher = compile.matcher(viewFilePage);
- URL hyperlink = null;
if (matcher.find()) {
// got hyperlink for Tail this file
- hyperlink = new URL(matcher.group(1));
+ String u = matcher.group(1);
+ String urlString = u.startsWith("///") ? ("http://" + u.substring(3)) : u;
viewFilePage = StringEscapeUtils.unescapeHtml(DFSTestUtil
- .urlGet(hyperlink));
+ .urlGet(new URL(urlString)));
assertTrue("page should show preview of file contents", viewFilePage
.contains(FILE_DATA));
} else {
@@ -166,6 +170,7 @@ public class TestDatanodeJsp {
Mockito.doReturn(NetUtils.getHostPortString(NameNode.getAddress(CONF)))
.when(reqMock).getParameter("nnaddr");
Mockito.doReturn(testFile.toString()).when(reqMock).getPathInfo();
+ Mockito.doReturn("http").when(reqMock).getScheme();
}
static Path writeFile(FileSystem fs, Path f) throws IOException {
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java?rev=1532952&r1=1532951&r2=1532952&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java Thu Oct 17 02:14:33 2013
@@ -17,14 +17,6 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.EnumSet;
-import java.util.List;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
@@ -33,41 +25,28 @@ import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.*;
import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataStorage;
-import org.apache.hadoop.hdfs.server.protocol.BlockCommand;
-import org.apache.hadoop.hdfs.server.protocol.CacheReport;
-import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
-import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol;
-import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
-import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
-import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
-import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
-import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
-import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport;
-import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
-import org.apache.hadoop.hdfs.server.protocol.StorageReport;
+import org.apache.hadoop.hdfs.server.protocol.*;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.net.DNS;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.security.Groups;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.util.*;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.EnumSet;
+import java.util.List;
+
/**
* Main class for a series of name-node benchmarks.
*
@@ -840,6 +819,7 @@ public class NNThroughputBenchmark imple
DNS.getDefaultHost("default", "default"),
"", getNodePort(dnIdx),
DFSConfigKeys.DFS_DATANODE_HTTP_DEFAULT_PORT,
+ DFSConfigKeys.DFS_DATANODE_HTTPS_DEFAULT_PORT,
DFSConfigKeys.DFS_DATANODE_IPC_DEFAULT_PORT),
new DataStorage(nsInfo, ""),
new ExportedBlockKeys(), VersionInfo.getVersion());
@@ -1308,7 +1288,7 @@ public class NNThroughputBenchmark imple
/**
* Main method of the benchmark.
- * @param args command line parameters
+ * @param aArgs command line parameters
*/
@Override // Tool
public int run(String[] aArgs) throws Exception {
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java?rev=1532952&r1=1532951&r2=1532952&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java Thu Oct 17 02:14:33 2013
@@ -25,7 +25,7 @@ import static org.mockito.Mockito.spy;
import java.lang.reflect.Field;
import java.util.EnumSet;
-import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import org.apache.commons.logging.Log;
@@ -119,7 +119,7 @@ public class TestAddBlockRetry {
return ret;
}
}).when(spyBM).chooseTarget(Mockito.anyString(), Mockito.anyInt(),
- Mockito.<DatanodeDescriptor>any(), Mockito.<HashMap<Node, Node>>any(),
+ Mockito.<DatanodeDescriptor>any(), Mockito.<HashSet<Node>>any(),
Mockito.anyLong(), Mockito.<List<String>>any());
// create file
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java?rev=1532952&r1=1532951&r2=1532952&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java Thu Oct 17 02:14:33 2013
@@ -18,15 +18,6 @@
package org.apache.hadoop.hdfs.server.namenode;
-import static org.junit.Assert.fail;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyBoolean;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-
-import java.io.IOException;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
@@ -37,6 +28,13 @@ import org.apache.hadoop.hdfs.server.blo
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.junit.Test;
+import java.io.IOException;
+
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyBoolean;
+import static org.mockito.Mockito.*;
+
/**
* Verify that TestCommitBlockSynchronization is idempotent.
*/
@@ -177,7 +175,7 @@ public class TestCommitBlockSynchronizat
Block block = new Block(blockId, length, genStamp);
FSNamesystem namesystemSpy = makeNameSystemSpy(block, file);
DatanodeID[] newTargets = new DatanodeID[]{
- new DatanodeID("0.0.0.0", "nonexistantHost", "1", 0, 0, 0)};
+ new DatanodeID("0.0.0.0", "nonexistantHost", "1", 0, 0, 0, 0)};
ExtendedBlock lastBlock = new ExtendedBlock();
namesystemSpy.commitBlockSynchronization(
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java?rev=1532952&r1=1532951&r2=1532952&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java Thu Oct 17 02:14:33 2013
@@ -20,8 +20,7 @@ package org.apache.hadoop.hdfs.server.na
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
import java.io.File;
import java.io.IOException;
@@ -142,4 +141,21 @@ public class TestFSNamesystem {
assertTrue("Replication queues weren't being populated after entering "
+ "safemode 2nd time", fsn.isPopulatingReplQueues());
}
+
+ @Test
+ public void testFsLockFairness() throws IOException, InterruptedException{
+ Configuration conf = new Configuration();
+
+ FSEditLog fsEditLog = Mockito.mock(FSEditLog.class);
+ FSImage fsImage = Mockito.mock(FSImage.class);
+ Mockito.when(fsImage.getEditLog()).thenReturn(fsEditLog);
+
+ conf.setBoolean("dfs.namenode.fslock.fair", true);
+ FSNamesystem fsNamesystem = new FSNamesystem(conf, fsImage);
+ assertTrue(fsNamesystem.getFsLockForTests().isFair());
+
+ conf.setBoolean("dfs.namenode.fslock.fair", false);
+ fsNamesystem = new FSNamesystem(conf, fsImage);
+ assertFalse(fsNamesystem.getFsLockForTests().isFair());
+ }
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java?rev=1532952&r1=1532951&r2=1532952&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java Thu Oct 17 02:14:33 2013
@@ -86,24 +86,27 @@ public class TestFileJournalManager {
EditLogInputStream elis = null;
try {
while ((elis = allStreams.poll()) != null) {
- elis.skipUntil(txId);
- while (true) {
- FSEditLogOp op = elis.readOp();
- if (op == null) {
- break;
- }
- if (abortOnGap && (op.getTransactionId() != txId)) {
- LOG.info("getNumberOfTransactions: detected gap at txId " +
- fromTxId);
- return numTransactions;
+ try {
+ elis.skipUntil(txId);
+ while (true) {
+ FSEditLogOp op = elis.readOp();
+ if (op == null) {
+ break;
+ }
+ if (abortOnGap && (op.getTransactionId() != txId)) {
+ LOG.info("getNumberOfTransactions: detected gap at txId "
+ + fromTxId);
+ return numTransactions;
+ }
+ txId = op.getTransactionId() + 1;
+ numTransactions++;
}
- txId = op.getTransactionId() + 1;
- numTransactions++;
+ } finally {
+ IOUtils.cleanup(LOG, elis);
}
}
} finally {
IOUtils.cleanup(LOG, allStreams.toArray(new EditLogInputStream[0]));
- IOUtils.cleanup(LOG, elis);
}
return numTransactions;
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java?rev=1532952&r1=1532951&r2=1532952&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java Thu Oct 17 02:14:33 2013
@@ -32,6 +32,7 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.DirectoryListingStartAfterNotFoundException;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -51,6 +52,7 @@ import org.apache.hadoop.hdfs.DFSTestUti
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
@@ -930,7 +932,9 @@ public class TestINodeFile {
dirNodeFromNode = fsdir.getInode(dirNode.getId());
assertSame(dirNode, dirNodeFromNode);
} finally {
- cluster.shutdown();
+ if (cluster != null) {
+ cluster.shutdown();
+ }
}
}
@@ -960,7 +964,55 @@ public class TestINodeFile {
assertTrue(parentId == status.getFileId());
} finally {
- cluster.shutdown();
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ @Test
+ public void testFilesInGetListingOps() throws Exception {
+ final Configuration conf = new Configuration();
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ cluster.waitActive();
+ final DistributedFileSystem hdfs = cluster.getFileSystem();
+ final FSDirectory fsdir = cluster.getNamesystem().getFSDirectory();
+
+ hdfs.mkdirs(new Path("/tmp"));
+ DFSTestUtil.createFile(hdfs, new Path("/tmp/f1"), 0, (short) 1, 0);
+ DFSTestUtil.createFile(hdfs, new Path("/tmp/f2"), 0, (short) 1, 0);
+ DFSTestUtil.createFile(hdfs, new Path("/tmp/f3"), 0, (short) 1, 0);
+
+ DirectoryListing dl = cluster.getNameNodeRpc().getListing("/tmp",
+ HdfsFileStatus.EMPTY_NAME, false);
+ assertTrue(dl.getPartialListing().length == 3);
+
+ String f2 = new String("f2");
+ dl = cluster.getNameNodeRpc().getListing("/tmp", f2.getBytes(), false);
+ assertTrue(dl.getPartialListing().length == 1);
+
+ INode f2INode = fsdir.getINode("/tmp/f2");
+ String f2InodePath = "/.reserved/.inodes/" + f2INode.getId();
+ dl = cluster.getNameNodeRpc().getListing("/tmp", f2InodePath.getBytes(),
+ false);
+ assertTrue(dl.getPartialListing().length == 1);
+
+ // Test the deleted startAfter file
+ hdfs.delete(new Path("/tmp/f2"), false);
+ try {
+ dl = cluster.getNameNodeRpc().getListing("/tmp",
+ f2InodePath.getBytes(), false);
+ fail("Didn't get exception for the deleted startAfter token.");
+ } catch (IOException e) {
+ assertTrue(e instanceof DirectoryListingStartAfterNotFoundException);
+ }
+
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
}
}
}