You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by vi...@apache.org on 2013/10/30 23:22:22 UTC
svn commit: r1537330 [9/11] - in
/hadoop/common/branches/YARN-321/hadoop-hdfs-project: ./
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/
hadoop-hdfs-nfs/
hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/
hadoop-hdfs-nfs/sr...
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java Wed Oct 30 22:21:59 2013
@@ -23,12 +23,16 @@ import static org.junit.Assert.assertFal
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+
+import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumSet;
+import java.util.List;
import java.util.Random;
import org.apache.commons.lang.ArrayUtils;
@@ -43,11 +47,14 @@ import org.apache.hadoop.fs.FSDataOutput
import org.apache.hadoop.fs.FileChecksum;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.VolumeId;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.web.HftpFileSystem;
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.DataChecksum;
@@ -70,6 +77,9 @@ public class TestDistributedFileSystem {
HdfsConfiguration conf;
if (noXmlDefaults) {
conf = new HdfsConfiguration(false);
+ String namenodeDir = new File(MiniDFSCluster.getBaseDirectory(), "name").getAbsolutePath();
+ conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, namenodeDir);
+ conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, namenodeDir);
} else {
conf = new HdfsConfiguration();
}
@@ -83,6 +93,19 @@ public class TestDistributedFileSystem {
}
@Test
+ public void testEmptyDelegationToken() throws IOException {
+ Configuration conf = getTestConfiguration();
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ FileSystem fileSys = cluster.getFileSystem();
+ fileSys.getDelegationToken("");
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
public void testFileSystemCloseAll() throws Exception {
Configuration conf = getTestConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
@@ -187,7 +210,7 @@ public class TestDistributedFileSystem {
final long millis = Time.now();
{
- DistributedFileSystem dfs = (DistributedFileSystem)cluster.getFileSystem();
+ final DistributedFileSystem dfs = cluster.getFileSystem();
dfs.dfs.getLeaseRenewer().setGraceSleepPeriod(grace);
assertFalse(dfs.dfs.getLeaseRenewer().isRunning());
@@ -287,7 +310,7 @@ public class TestDistributedFileSystem {
}
{
- DistributedFileSystem dfs = (DistributedFileSystem)cluster.getFileSystem();
+ final DistributedFileSystem dfs = cluster.getFileSystem();
assertFalse(dfs.dfs.getLeaseRenewer().isRunning());
//open and check the file
@@ -796,4 +819,25 @@ public class TestDistributedFileSystem {
}
}
+ @Test(timeout=60000)
+ public void testListFiles() throws IOException {
+ Configuration conf = new HdfsConfiguration();
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
+
+ try {
+ DistributedFileSystem fs = cluster.getFileSystem();
+
+ final Path relative = new Path("relative");
+ fs.create(new Path(relative, "foo")).close();
+
+ final List<LocatedFileStatus> retVal = new ArrayList<LocatedFileStatus>();
+ final RemoteIterator<LocatedFileStatus> iter = fs.listFiles(relative, true);
+ while (iter.hasNext()) {
+ retVal.add(iter.next());
+ }
+ System.out.println("retVal = " + retVal);
+ } finally {
+ cluster.shutdown();
+ }
+ }
}
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFSInputChecker.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFSInputChecker.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFSInputChecker.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFSInputChecker.java Wed Oct 30 22:21:59 2013
@@ -39,6 +39,7 @@ import org.apache.hadoop.fs.LocalFileSys
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.test.PathUtils;
import org.junit.Test;
/**
@@ -260,7 +261,7 @@ public class TestFSInputChecker {
// create a file and verify that checksum corruption results in
// a checksum exception on LocalFS
- String dir = System.getProperty("test.build.data", ".");
+ String dir = PathUtils.getTestDirName(getClass());
Path file = new Path(dir + "/corruption-test.dat");
Path crcFile = new Path(dir + "/.corruption-test.dat.crc");
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppendRestart.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppendRestart.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppendRestart.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppendRestart.java Wed Oct 30 22:21:59 2013
@@ -37,6 +37,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.util.Holder;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.PathUtils;
import org.junit.Test;
/**
@@ -145,7 +146,7 @@ public class TestFileAppendRestart {
String tarFile = System.getProperty("test.cache.data", "build/test/cache")
+ "/" + HADOOP_23_BROKEN_APPEND_TGZ;
- String testDir = System.getProperty("test.build.data", "build/test/data");
+ String testDir = PathUtils.getTestDirName(getClass());
File dfsDir = new File(testDir, "image-with-buggy-append");
if (dfsDir.exists() && !FileUtil.fullyDelete(dfsDir)) {
throw new IOException("Could not delete dfs directory '" + dfsDir + "'");
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java Wed Oct 30 22:21:59 2013
@@ -43,6 +43,7 @@ import org.apache.hadoop.hdfs.server.dat
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
+import org.apache.hadoop.test.PathUtils;
import org.apache.log4j.Level;
import org.junit.Test;
@@ -95,7 +96,7 @@ public class TestFileCorruption {
@Test
public void testLocalFileCorruption() throws Exception {
Configuration conf = new HdfsConfiguration();
- Path file = new Path(System.getProperty("test.build.data"), "corruptFile");
+ Path file = new Path(PathUtils.getTestDirName(getClass()), "corruptFile");
FileSystem fs = FileSystem.getLocal(conf);
DataOutputStream dos = fs.create(file);
dos.writeBytes("original bytes");
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileInputStreamCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileInputStreamCache.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileInputStreamCache.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileInputStreamCache.java Wed Oct 30 22:21:59 2013
@@ -17,12 +17,7 @@
*/
package org.apache.hadoop.hdfs;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-
import junit.framework.Assert;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
@@ -31,6 +26,10 @@ import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.unix.TemporarySocketDirectory;
import org.junit.Test;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
public class TestFileInputStreamCache {
static final Log LOG = LogFactory.getLog(TestFileInputStreamCache.class);
@@ -80,7 +79,7 @@ public class TestFileInputStreamCache {
public void testAddAndRetrieve() throws Exception {
FileInputStreamCache cache = new FileInputStreamCache(1, 1000000);
DatanodeID dnId = new DatanodeID("127.0.0.1", "localhost",
- "xyzzy", 8080, 9090, 7070);
+ "xyzzy", 8080, 9090, 7070, 6060);
ExtendedBlock block = new ExtendedBlock("poolid", 123);
TestFileDescriptorPair pair = new TestFileDescriptorPair();
cache.put(dnId, block, pair.getFileInputStreams());
@@ -94,7 +93,7 @@ public class TestFileInputStreamCache {
public void testExpiry() throws Exception {
FileInputStreamCache cache = new FileInputStreamCache(1, 10);
DatanodeID dnId = new DatanodeID("127.0.0.1", "localhost",
- "xyzzy", 8080, 9090, 7070);
+ "xyzzy", 8080, 9090, 7070, 6060);
ExtendedBlock block = new ExtendedBlock("poolid", 123);
TestFileDescriptorPair pair = new TestFileDescriptorPair();
cache.put(dnId, block, pair.getFileInputStreams());
@@ -109,12 +108,12 @@ public class TestFileInputStreamCache {
public void testEviction() throws Exception {
FileInputStreamCache cache = new FileInputStreamCache(1, 10000000);
DatanodeID dnId = new DatanodeID("127.0.0.1", "localhost",
- "xyzzy", 8080, 9090, 7070);
+ "xyzzy", 8080, 9090, 7070, 6060);
ExtendedBlock block = new ExtendedBlock("poolid", 123);
TestFileDescriptorPair pair = new TestFileDescriptorPair();
cache.put(dnId, block, pair.getFileInputStreams());
DatanodeID dnId2 = new DatanodeID("127.0.0.1", "localhost",
- "xyzzy", 8081, 9091, 7071);
+ "xyzzy", 8081, 9091, 7071, 6061);
TestFileDescriptorPair pair2 = new TestFileDescriptorPair();
cache.put(dnId2, block, pair2.getFileInputStreams());
FileInputStream fis[] = cache.get(dnId, block);
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java Wed Oct 30 22:21:59 2013
@@ -41,6 +41,7 @@ import org.apache.hadoop.hdfs.protocol.H
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.web.HftpFileSystem;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.log4j.Level;
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSServerPorts.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSServerPorts.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSServerPorts.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSServerPorts.java Wed Oct 30 22:21:59 2013
@@ -36,6 +36,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.net.DNS;
+import org.apache.hadoop.test.PathUtils;
import org.junit.Test;
/**
@@ -53,6 +54,9 @@ public class TestHDFSServerPorts {
// reset default 0.0.0.0 addresses in order to avoid IPv6 problem
static final String THIS_HOST = getFullHostName() + ":0";
+
+ private static final File TEST_DATA_DIR = PathUtils.getTestDir(TestHDFSServerPorts.class);
+
static {
DefaultMetricsSystem.setMiniClusterMode(true);
}
@@ -81,13 +85,6 @@ public class TestHDFSServerPorts {
}
}
- /**
- * Get base directory these tests should run in.
- */
- private String getTestingDir() {
- return System.getProperty("test.build.data", "build/test/data");
- }
-
public NameNode startNameNode() throws IOException {
return startNameNode(false);
}
@@ -95,8 +92,7 @@ public class TestHDFSServerPorts {
* Start the namenode.
*/
public NameNode startNameNode(boolean withService) throws IOException {
- String dataDir = getTestingDir();
- hdfsDir = new File(dataDir, "dfs");
+ hdfsDir = new File(TEST_DATA_DIR, "dfs");
if ( hdfsDir.exists() && !FileUtil.fullyDelete(hdfsDir) ) {
throw new IOException("Could not delete hdfs directory '" + hdfsDir + "'");
}
@@ -119,9 +115,8 @@ public class TestHDFSServerPorts {
* Start the BackupNode
*/
public BackupNode startBackupNode(Configuration conf) throws IOException {
- String dataDir = getTestingDir();
// Set up testing environment directories
- hdfsDir = new File(dataDir, "backupNode");
+ hdfsDir = new File(TEST_DATA_DIR, "backupNode");
if ( hdfsDir.exists() && !FileUtil.fullyDelete(hdfsDir) ) {
throw new IOException("Could not delete hdfs directory '" + hdfsDir + "'");
}
@@ -150,8 +145,7 @@ public class TestHDFSServerPorts {
*/
public DataNode startDataNode(int index, Configuration config)
throws IOException {
- String dataDir = getTestingDir();
- File dataNodeDir = new File(dataDir, "data-" + index);
+ File dataNodeDir = new File(TEST_DATA_DIR, "data-" + index);
config.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dataNodeDir.getPath());
String[] args = new String[] {};
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestListPathServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestListPathServlet.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestListPathServlet.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestListPathServlet.java Wed Oct 30 22:21:59 2013
@@ -29,6 +29,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.server.namenode.ListPathsServlet;
+import org.apache.hadoop.hdfs.web.HftpFileSystem;
import org.apache.hadoop.security.UserGroupInformation;
import org.junit.AfterClass;
import org.junit.Assert;
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java Wed Oct 30 22:21:59 2013
@@ -25,8 +25,8 @@ import static org.junit.Assume.assumeTru
import java.io.File;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.FSConstants;
-import org.junit.After;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.test.PathUtils;
import org.junit.Before;
import org.junit.Test;
@@ -43,20 +43,10 @@ public class TestMiniDFSCluster {
private static final String CLUSTER_3 = "cluster3";
private static final String CLUSTER_4 = "cluster4";
private static final String CLUSTER_5 = "cluster5";
- protected String testDataPath;
- protected File testDataDir;
+ protected File testDataPath;
@Before
public void setUp() {
- testDataPath = System.getProperty(MiniDFSCluster.PROP_TEST_BUILD_DATA,
- "build/test/data");
- testDataDir = new File(new File(testDataPath).getParentFile(),
- "miniclusters");
-
-
- }
- @After
- public void tearDown() {
- System.setProperty(MiniDFSCluster.PROP_TEST_BUILD_DATA, testDataPath);
+ testDataPath = new File(PathUtils.getTestDir(getClass()), "miniclusters");
}
/**
@@ -120,7 +110,7 @@ public class TestMiniDFSCluster {
MiniDFSCluster cluster4 = new MiniDFSCluster.Builder(conf).build();
try {
DistributedFileSystem dfs = (DistributedFileSystem) cluster4.getFileSystem();
- dfs.setSafeMode(FSConstants.SafeModeAction.SAFEMODE_ENTER);
+ dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER);
cluster4.shutdown();
} finally {
while(cluster4.isClusterUp()){
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPeerCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPeerCache.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPeerCache.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPeerCache.java Wed Oct 30 22:21:59 2013
@@ -17,26 +17,23 @@
*/
package org.apache.hadoop.hdfs;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertSame;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.channels.ReadableByteChannel;
-
+import com.google.common.collect.HashMultiset;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.net.Peer;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.net.unix.DomainSocket;
import org.junit.Test;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
-import com.google.common.collect.HashMultiset;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.channels.ReadableByteChannel;
+
+import static org.junit.Assert.*;
public class TestPeerCache {
static final Log LOG = LogFactory.getLog(TestPeerCache.class);
@@ -150,7 +147,7 @@ public class TestPeerCache {
PeerCache cache = new PeerCache(3, 100000);
DatanodeID dnId = new DatanodeID("192.168.0.1",
"fakehostname", "fake_storage_id",
- 100, 101, 102);
+ 100, 101, 102, 103);
FakePeer peer = new FakePeer(dnId, false);
cache.put(dnId, peer);
assertTrue(!peer.isClosed());
@@ -170,7 +167,7 @@ public class TestPeerCache {
for (int i = 0; i < CAPACITY; ++i) {
dnIds[i] = new DatanodeID("192.168.0.1",
"fakehostname_" + i, "fake_storage_id",
- 100, 101, 102);
+ 100, 101, 102, 103);
peers[i] = new FakePeer(dnIds[i], false);
}
for (int i = 0; i < CAPACITY; ++i) {
@@ -201,7 +198,7 @@ public class TestPeerCache {
for (int i = 0; i < dnIds.length; ++i) {
dnIds[i] = new DatanodeID("192.168.0.1",
"fakehostname_" + i, "fake_storage_id_" + i,
- 100, 101, 102);
+ 100, 101, 102, 103);
peers[i] = new FakePeer(dnIds[i], false);
}
for (int i = 0; i < CAPACITY; ++i) {
@@ -232,7 +229,7 @@ public class TestPeerCache {
PeerCache cache = new PeerCache(CAPACITY, 100000);
DatanodeID dnId = new DatanodeID("192.168.0.1",
"fakehostname", "fake_storage_id",
- 100, 101, 102);
+ 100, 101, 102, 103);
HashMultiset<FakePeer> peers = HashMultiset.create(CAPACITY);
for (int i = 0; i < CAPACITY; ++i) {
FakePeer peer = new FakePeer(dnId, false);
@@ -257,7 +254,7 @@ public class TestPeerCache {
PeerCache cache = new PeerCache(CAPACITY, 100000);
DatanodeID dnId = new DatanodeID("192.168.0.1",
"fakehostname", "fake_storage_id",
- 100, 101, 102);
+ 100, 101, 102, 103);
HashMultiset<FakePeer> peers = HashMultiset.create(CAPACITY);
for (int i = 0; i < CAPACITY; ++i) {
FakePeer peer = new FakePeer(dnId, i == CAPACITY - 1);
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPersistBlocks.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPersistBlocks.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPersistBlocks.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPersistBlocks.java Wed Oct 30 22:21:59 2013
@@ -43,6 +43,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.PathUtils;
import org.apache.log4j.Level;
import org.junit.Test;
@@ -304,7 +305,7 @@ public class TestPersistBlocks {
String tarFile = System.getProperty("test.cache.data", "build/test/cache")
+ "/" + HADOOP_1_0_MULTIBLOCK_TGZ;
- String testDir = System.getProperty("test.build.data", "build/test/data");
+ String testDir = PathUtils.getTestDirName(getClass());
File dfsDir = new File(testDir, "image-1.0");
if (dfsDir.exists() && !FileUtil.fullyDelete(dfsDir)) {
throw new IOException("Could not delete dfs directory '" + dfsDir + "'");
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java Wed Oct 30 22:21:59 2013
@@ -578,4 +578,64 @@ public class TestShortCircuitLocalRead {
System.out.println("Iteration " + iteration + " took " + (end - start));
fs.delete(file1, false);
}
+
+ public void testReadWithRemoteBlockReader() throws IOException, InterruptedException {
+ doTestShortCircuitReadWithRemoteBlockReader(true, 3*blockSize+100, getCurrentUser(), 0, false);
+ }
+
+ /**
+ * Test that file data can be read by reading the block
+ * through RemoteBlockReader
+ * @throws IOException
+ */
+ public void doTestShortCircuitReadWithRemoteBlockReader(boolean ignoreChecksum, int size, String shortCircuitUser,
+ int readOffset, boolean shortCircuitFails) throws IOException, InterruptedException {
+ Configuration conf = new Configuration();
+ conf.setBoolean(DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADER, true);
+ conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY, true);
+
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1)
+ .format(true).build();
+ FileSystem fs = cluster.getFileSystem();
+ // check that / exists
+ Path path = new Path("/");
+ URI uri = cluster.getURI();
+ assertTrue("/ should be a directory", fs.getFileStatus(path)
+ .isDirectory() == true);
+
+ byte[] fileData = AppendTestUtil.randomBytes(seed, size);
+ Path file1 = new Path("filelocal.dat");
+ FSDataOutputStream stm = createFile(fs, file1, 1);
+
+ stm.write(fileData);
+ stm.close();
+ try {
+ checkFileContent(uri, file1, fileData, readOffset, shortCircuitUser, conf, shortCircuitFails);
+ //RemoteBlockReader have unsupported method read(ByteBuffer bf)
+ assertTrue("RemoteBlockReader unsupported method read(ByteBuffer bf) error",
+ checkUnsupportedMethod(fs, file1, fileData, readOffset));
+ } catch(IOException e) {
+ throw new IOException("doTestShortCircuitReadWithRemoteBlockReader ex error ", e);
+ } catch(InterruptedException inEx) {
+ throw inEx;
+ } finally {
+ fs.close();
+ cluster.shutdown();
+ }
+ }
+
+ private boolean checkUnsupportedMethod(FileSystem fs, Path file,
+ byte[] expected, int readOffset) throws IOException {
+ HdfsDataInputStream stm = (HdfsDataInputStream)fs.open(file);
+ ByteBuffer actual = ByteBuffer.allocateDirect(expected.length - readOffset);
+ IOUtils.skipFully(stm, readOffset);
+ try {
+ stm.read(actual);
+ } catch(UnsupportedOperationException unex) {
+ return true;
+ }
+ return false;
+ }
+
+
}
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java Wed Oct 30 22:21:59 2013
@@ -174,6 +174,7 @@ public class TestNNWithQJM {
public void testMismatchedNNIsRejected() throws Exception {
conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
MiniDFSCluster.getBaseDirectory() + "/TestNNWithQJM/image");
+ String defaultEditsDir = conf.get(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY);
conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY,
mjc.getQuorumJournalURI("myjournal").toString());
@@ -187,7 +188,7 @@ public class TestNNWithQJM {
// Reformat just the on-disk portion
Configuration onDiskOnly = new Configuration(conf);
- onDiskOnly.unset(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY);
+ onDiskOnly.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, defaultEditsDir);
NameNode.format(onDiskOnly);
// Start the NN - should fail because the JNs are still formatted
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java Wed Oct 30 22:21:59 2013
@@ -46,6 +46,7 @@ import org.apache.hadoop.metrics2.Metric
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.MetricsAsserts;
+import org.apache.hadoop.test.PathUtils;
import org.apache.hadoop.util.Shell;
import org.junit.After;
import org.junit.Before;
@@ -61,13 +62,13 @@ public class TestJournalNode {
private static final NamespaceInfo FAKE_NSINFO = new NamespaceInfo(
12345, "mycluster", "my-bp", 0L);
+ private static File TEST_BUILD_DATA = PathUtils.getTestDir(TestJournalNode.class);
+
private JournalNode jn;
private Journal journal;
private Configuration conf = new Configuration();
private IPCLoggerChannel ch;
private String journalId;
- private File TEST_BUILD_DATA =
- new File(System.getProperty("test.build.data", "build/test/data"));
static {
// Avoid an error when we double-initialize JvmMetrics
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java Wed Oct 30 22:21:59 2013
@@ -31,6 +31,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
@@ -39,9 +40,11 @@ import org.apache.hadoop.hdfs.NameNodePr
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyWithNodeGroup;
import org.apache.hadoop.net.NetworkTopology;
+import org.apache.hadoop.net.NetworkTopologyWithNodeGroup;
+import org.junit.Assert;
import org.junit.Test;
-import junit.framework.Assert;
/**
* This class tests if a balancer schedules tasks correctly.
@@ -75,10 +78,9 @@ public class TestBalancerWithNodeGroup {
Configuration conf = new HdfsConfiguration();
TestBalancer.initConf(conf);
conf.set(CommonConfigurationKeysPublic.NET_TOPOLOGY_IMPL_KEY,
- "org.apache.hadoop.net.NetworkTopologyWithNodeGroup");
- conf.set("dfs.block.replicator.classname",
- "org.apache.hadoop.hdfs.server.blockmanagement." +
- "BlockPlacementPolicyWithNodeGroup");
+ NetworkTopologyWithNodeGroup.class.getName());
+ conf.set(DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY,
+ BlockPlacementPolicyWithNodeGroup.class.getName());
return conf;
}
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java Wed Oct 30 22:21:59 2013
@@ -22,7 +22,6 @@ import static org.junit.Assert.assertEqu
import static org.junit.Assert.assertTrue;
import java.io.File;
-import java.io.IOException;
import java.util.Collection;
import org.apache.hadoop.conf.Configuration;
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestRBWBlockInvalidation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestRBWBlockInvalidation.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestRBWBlockInvalidation.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestRBWBlockInvalidation.java Wed Oct 30 22:21:59 2013
@@ -157,8 +157,8 @@ public class TestRBWBlockInvalidation {
// in the context of the test, whereas a random one is more accurate
// to what is seen in real clusters (nodes have random amounts of free
// space)
- conf.setClass("dfs.block.replicator.classname", RandomDeleterPolicy.class,
- BlockPlacementPolicy.class);
+ conf.setClass(DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY,
+ RandomDeleterPolicy.class, BlockPlacementPolicy.class);
// Speed up the test a bit with faster heartbeats.
conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java Wed Oct 30 22:21:59 2013
@@ -26,9 +26,11 @@ import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Random;
+import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -45,6 +47,7 @@ import org.apache.hadoop.hdfs.server.dat
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.net.Node;
+import org.apache.hadoop.test.PathUtils;
import org.apache.hadoop.util.Time;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
@@ -84,8 +87,7 @@ public class TestReplicationPolicy {
FileSystem.setDefaultUri(conf, "hdfs://localhost:0");
conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
- File baseDir = new File(System.getProperty(
- "test.build.data", "build/test/data"), "dfs/");
+ File baseDir = PathUtils.getTestDir(TestReplicationPolicy.class);
conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
new File(baseDir, "name").getPath());
@@ -128,30 +130,25 @@ public class TestReplicationPolicy {
HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 4, 0); // overloaded
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename, 0, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(0);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename, 1, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(1);
assertEquals(targets.length, 1);
assertEquals(targets[0], dataNodes[0]);
- targets = replicator.chooseTarget(filename,
- 2, dataNodes[0], new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(2);
assertEquals(targets.length, 2);
assertEquals(targets[0], dataNodes[0]);
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
- targets = replicator.chooseTarget(filename, 3, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(3);
assertEquals(targets.length, 3);
assertEquals(targets[0], dataNodes[0]);
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
assertTrue(cluster.isOnSameRack(targets[1], targets[2]));
- targets = replicator.chooseTarget(filename, 4, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(4);
assertEquals(targets.length, 4);
assertEquals(targets[0], dataNodes[0]);
assertTrue(cluster.isOnSameRack(targets[1], targets[2]) ||
@@ -163,15 +160,38 @@ public class TestReplicationPolicy {
HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 0, 0);
}
+ private static DatanodeDescriptor[] chooseTarget(int numOfReplicas) {
+ return chooseTarget(numOfReplicas, dataNodes[0]);
+ }
+
+ private static DatanodeDescriptor[] chooseTarget(int numOfReplicas,
+ DatanodeDescriptor writer) {
+ return chooseTarget(numOfReplicas, writer,
+ new ArrayList<DatanodeDescriptor>());
+ }
+
+ private static DatanodeDescriptor[] chooseTarget(int numOfReplicas,
+ List<DatanodeDescriptor> chosenNodes) {
+ return chooseTarget(numOfReplicas, dataNodes[0], chosenNodes);
+ }
+
+ private static DatanodeDescriptor[] chooseTarget(int numOfReplicas,
+ DatanodeDescriptor writer, List<DatanodeDescriptor> chosenNodes) {
+ return chooseTarget(numOfReplicas, writer, chosenNodes, null);
+ }
+
+ private static DatanodeDescriptor[] chooseTarget(int numOfReplicas,
+ List<DatanodeDescriptor> chosenNodes, Set<Node> excludedNodes) {
+ return chooseTarget(numOfReplicas, dataNodes[0], chosenNodes, excludedNodes);
+ }
+
private static DatanodeDescriptor[] chooseTarget(
- BlockPlacementPolicyDefault policy,
int numOfReplicas,
DatanodeDescriptor writer,
List<DatanodeDescriptor> chosenNodes,
- HashMap<Node, Node> excludedNodes,
- long blocksize) {
- return policy.chooseTarget(numOfReplicas, writer, chosenNodes, false,
- excludedNodes, blocksize);
+ Set<Node> excludedNodes) {
+ return replicator.chooseTarget(filename, numOfReplicas, writer, chosenNodes,
+ false, excludedNodes, BLOCK_SIZE);
}
/**
@@ -184,39 +204,34 @@ public class TestReplicationPolicy {
*/
@Test
public void testChooseTarget2() throws Exception {
- HashMap<Node, Node> excludedNodes;
+ Set<Node> excludedNodes;
DatanodeDescriptor[] targets;
- BlockPlacementPolicyDefault repl = (BlockPlacementPolicyDefault)replicator;
List<DatanodeDescriptor> chosenNodes = new ArrayList<DatanodeDescriptor>();
- excludedNodes = new HashMap<Node, Node>();
- excludedNodes.put(dataNodes[1], dataNodes[1]);
- targets = chooseTarget(repl, 0, dataNodes[0], chosenNodes, excludedNodes,
- BLOCK_SIZE);
+ excludedNodes = new HashSet<Node>();
+ excludedNodes.add(dataNodes[1]);
+ targets = chooseTarget(0, chosenNodes, excludedNodes);
assertEquals(targets.length, 0);
excludedNodes.clear();
chosenNodes.clear();
- excludedNodes.put(dataNodes[1], dataNodes[1]);
- targets = chooseTarget(repl, 1, dataNodes[0], chosenNodes, excludedNodes,
- BLOCK_SIZE);
+ excludedNodes.add(dataNodes[1]);
+ targets = chooseTarget(1, chosenNodes, excludedNodes);
assertEquals(targets.length, 1);
assertEquals(targets[0], dataNodes[0]);
excludedNodes.clear();
chosenNodes.clear();
- excludedNodes.put(dataNodes[1], dataNodes[1]);
- targets = chooseTarget(repl, 2, dataNodes[0], chosenNodes, excludedNodes,
- BLOCK_SIZE);
+ excludedNodes.add(dataNodes[1]);
+ targets = chooseTarget(2, chosenNodes, excludedNodes);
assertEquals(targets.length, 2);
assertEquals(targets[0], dataNodes[0]);
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
excludedNodes.clear();
chosenNodes.clear();
- excludedNodes.put(dataNodes[1], dataNodes[1]);
- targets = chooseTarget(repl, 3, dataNodes[0], chosenNodes, excludedNodes,
- BLOCK_SIZE);
+ excludedNodes.add(dataNodes[1]);
+ targets = chooseTarget(3, chosenNodes, excludedNodes);
assertEquals(targets.length, 3);
assertEquals(targets[0], dataNodes[0]);
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
@@ -224,9 +239,8 @@ public class TestReplicationPolicy {
excludedNodes.clear();
chosenNodes.clear();
- excludedNodes.put(dataNodes[1], dataNodes[1]);
- targets = chooseTarget(repl, 4, dataNodes[0], chosenNodes, excludedNodes,
- BLOCK_SIZE);
+ excludedNodes.add(dataNodes[1]);
+ targets = chooseTarget(4, chosenNodes, excludedNodes);
assertEquals(targets.length, 4);
assertEquals(targets[0], dataNodes[0]);
for(int i=1; i<4; i++) {
@@ -238,9 +252,9 @@ public class TestReplicationPolicy {
excludedNodes.clear();
chosenNodes.clear();
- excludedNodes.put(dataNodes[1], dataNodes[1]);
+ excludedNodes.add(dataNodes[1]);
chosenNodes.add(dataNodes[2]);
- targets = repl.chooseTarget(1, dataNodes[0], chosenNodes, true,
+ targets = replicator.chooseTarget(filename, 1, dataNodes[0], chosenNodes, true,
excludedNodes, BLOCK_SIZE);
System.out.println("targets=" + Arrays.asList(targets));
assertEquals(2, targets.length);
@@ -266,30 +280,25 @@ public class TestReplicationPolicy {
(HdfsConstants.MIN_BLOCKS_FOR_WRITE-1)*BLOCK_SIZE, 0L, 0, 0); // no space
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename, 0, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(0);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename, 1, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(1);
assertEquals(targets.length, 1);
assertEquals(targets[0], dataNodes[1]);
- targets = replicator.chooseTarget(filename, 2, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(2);
assertEquals(targets.length, 2);
assertEquals(targets[0], dataNodes[1]);
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
- targets = replicator.chooseTarget(filename, 3, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(3);
assertEquals(targets.length, 3);
assertEquals(targets[0], dataNodes[1]);
assertTrue(cluster.isOnSameRack(targets[1], targets[2]));
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
- targets = replicator.chooseTarget(filename, 4, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(4);
assertEquals(targets.length, 4);
assertEquals(targets[0], dataNodes[1]);
for(int i=1; i<4; i++) {
@@ -322,23 +331,19 @@ public class TestReplicationPolicy {
}
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename, 0, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(0);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename, 1, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(1);
assertEquals(targets.length, 1);
assertFalse(cluster.isOnSameRack(targets[0], dataNodes[0]));
- targets = replicator.chooseTarget(filename, 2, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(2);
assertEquals(targets.length, 2);
assertFalse(cluster.isOnSameRack(targets[0], dataNodes[0]));
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
- targets = replicator.chooseTarget(filename, 3, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(3);
assertEquals(targets.length, 3);
for(int i=0; i<3; i++) {
assertFalse(cluster.isOnSameRack(targets[i], dataNodes[0]));
@@ -367,21 +372,17 @@ public class TestReplicationPolicy {
DFSTestUtil.getDatanodeDescriptor("7.7.7.7", "/d2/r4");
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename, 0, writerDesc,
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(0, writerDesc);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename, 1, writerDesc,
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(1, writerDesc);
assertEquals(targets.length, 1);
- targets = replicator.chooseTarget(filename, 2, writerDesc,
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(2, writerDesc);
assertEquals(targets.length, 2);
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
- targets = replicator.chooseTarget(filename, 3, writerDesc,
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(3, writerDesc);
assertEquals(targets.length, 3);
assertTrue(cluster.isOnSameRack(targets[1], targets[2]));
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
@@ -425,9 +426,7 @@ public class TestReplicationPolicy {
// try to choose NUM_OF_DATANODES which is more than actually available
// nodes.
- DatanodeDescriptor[] targets = replicator.chooseTarget(filename,
- NUM_OF_DATANODES, dataNodes[0], new ArrayList<DatanodeDescriptor>(),
- BLOCK_SIZE);
+ DatanodeDescriptor[] targets = chooseTarget(NUM_OF_DATANODES);
assertEquals(targets.length, NUM_OF_DATANODES - 2);
final List<LoggingEvent> log = appender.getLog();
@@ -470,17 +469,14 @@ public class TestReplicationPolicy {
DatanodeDescriptor[] targets;
// We set the datanode[0] as stale, thus should choose datanode[1] since
// datanode[1] is on the same rack with datanode[0] (writer)
- targets = replicator.chooseTarget(filename, 1, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(1);
assertEquals(targets.length, 1);
assertEquals(targets[0], dataNodes[1]);
- HashMap<Node, Node> excludedNodes = new HashMap<Node, Node>();
- excludedNodes.put(dataNodes[1], dataNodes[1]);
+ Set<Node> excludedNodes = new HashSet<Node>();
+ excludedNodes.add(dataNodes[1]);
List<DatanodeDescriptor> chosenNodes = new ArrayList<DatanodeDescriptor>();
- BlockPlacementPolicyDefault repl = (BlockPlacementPolicyDefault)replicator;
- targets = chooseTarget(repl, 1, dataNodes[0], chosenNodes, excludedNodes,
- BLOCK_SIZE);
+ targets = chooseTarget(1, chosenNodes, excludedNodes);
assertEquals(targets.length, 1);
assertFalse(cluster.isOnSameRack(targets[0], dataNodes[0]));
@@ -507,33 +503,27 @@ public class TestReplicationPolicy {
namenode.getNamesystem().getBlockManager()
.getDatanodeManager().getHeartbeatManager().heartbeatCheck();
- DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename, 0, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ DatanodeDescriptor[] targets = chooseTarget(0);
assertEquals(targets.length, 0);
// Since we have 6 datanodes total, stale nodes should
// not be returned until we ask for more than 3 targets
- targets = replicator.chooseTarget(filename, 1, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(1);
assertEquals(targets.length, 1);
assertFalse(containsWithinRange(targets[0], dataNodes, 0, 2));
- targets = replicator.chooseTarget(filename, 2, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(2);
assertEquals(targets.length, 2);
assertFalse(containsWithinRange(targets[0], dataNodes, 0, 2));
assertFalse(containsWithinRange(targets[1], dataNodes, 0, 2));
- targets = replicator.chooseTarget(filename, 3, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(3);
assertEquals(targets.length, 3);
assertTrue(containsWithinRange(targets[0], dataNodes, 3, 5));
assertTrue(containsWithinRange(targets[1], dataNodes, 3, 5));
assertTrue(containsWithinRange(targets[2], dataNodes, 3, 5));
- targets = replicator.chooseTarget(filename, 4, dataNodes[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(4);
assertEquals(targets.length, 4);
assertTrue(containsWithinRange(dataNodes[3], targets, 0, 3));
assertTrue(containsWithinRange(dataNodes[4], targets, 0, 3));
@@ -586,7 +576,8 @@ public class TestReplicationPolicy {
BlockPlacementPolicy replicator = miniCluster.getNameNode()
.getNamesystem().getBlockManager().getBlockPlacementPolicy();
DatanodeDescriptor[] targets = replicator.chooseTarget(filename, 3,
- staleNodeInfo, new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ staleNodeInfo, new ArrayList<DatanodeDescriptor>(), false, null, BLOCK_SIZE);
+
assertEquals(targets.length, 3);
assertFalse(cluster.isOnSameRack(targets[0], staleNodeInfo));
@@ -610,7 +601,7 @@ public class TestReplicationPolicy {
.getDatanodeManager().shouldAvoidStaleDataNodesForWrite());
// Call chooseTarget
targets = replicator.chooseTarget(filename, 3,
- staleNodeInfo, new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ staleNodeInfo, new ArrayList<DatanodeDescriptor>(), false, null, BLOCK_SIZE);
assertEquals(targets.length, 3);
assertTrue(cluster.isOnSameRack(targets[0], staleNodeInfo));
@@ -632,8 +623,7 @@ public class TestReplicationPolicy {
assertTrue(miniCluster.getNameNode().getNamesystem().getBlockManager()
.getDatanodeManager().shouldAvoidStaleDataNodesForWrite());
// Call chooseTarget
- targets = replicator.chooseTarget(filename, 3,
- staleNodeInfo, new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(3, staleNodeInfo);
assertEquals(targets.length, 3);
assertFalse(cluster.isOnSameRack(targets[0], staleNodeInfo));
} finally {
@@ -654,23 +644,19 @@ public class TestReplicationPolicy {
chosenNodes.add(dataNodes[0]);
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename,
- 0, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(0, chosenNodes);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename,
- 1, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(1, chosenNodes);
assertEquals(targets.length, 1);
assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0]));
- targets = replicator.chooseTarget(filename,
- 2, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(2, chosenNodes);
assertEquals(targets.length, 2);
assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0]));
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
- targets = replicator.chooseTarget(filename,
- 3, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(3, chosenNodes);
assertEquals(targets.length, 3);
assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0]));
assertFalse(cluster.isOnSameRack(targets[0], targets[2]));
@@ -690,17 +676,14 @@ public class TestReplicationPolicy {
chosenNodes.add(dataNodes[1]);
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename,
- 0, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(0, chosenNodes);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename,
- 1, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(1, chosenNodes);
assertEquals(targets.length, 1);
assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0]));
- targets = replicator.chooseTarget(filename,
- 2, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(2, chosenNodes);
assertEquals(targets.length, 2);
assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0]));
assertFalse(cluster.isOnSameRack(dataNodes[0], targets[1]));
@@ -720,29 +703,24 @@ public class TestReplicationPolicy {
chosenNodes.add(dataNodes[2]);
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename,
- 0, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(0, chosenNodes);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename,
- 1, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(1, chosenNodes);
assertEquals(targets.length, 1);
assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0]));
assertFalse(cluster.isOnSameRack(dataNodes[2], targets[0]));
- targets = replicator.chooseTarget(filename,
- 1, dataNodes[2], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(1, dataNodes[2], chosenNodes);
assertEquals(targets.length, 1);
assertTrue(cluster.isOnSameRack(dataNodes[2], targets[0]));
assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0]));
- targets = replicator.chooseTarget(filename,
- 2, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(2, chosenNodes);
assertEquals(targets.length, 2);
assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0]));
- targets = replicator.chooseTarget(filename,
- 2, dataNodes[2], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(2, dataNodes[2], chosenNodes);
assertEquals(targets.length, 2);
assertTrue(cluster.isOnSameRack(dataNodes[2], targets[0]));
}
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java Wed Oct 30 22:21:59 2013
@@ -18,7 +18,6 @@
package org.apache.hadoop.hdfs.server.blockmanagement;
import java.io.File;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -37,7 +36,11 @@ import org.apache.hadoop.hdfs.HdfsConfig
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.net.NetworkTopology;
+import org.apache.hadoop.net.NetworkTopologyWithNodeGroup;
import org.apache.hadoop.net.Node;
+import org.apache.hadoop.test.PathUtils;
+import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
public class TestReplicationPolicyWithNodeGroup extends TestCase {
@@ -45,10 +48,10 @@ public class TestReplicationPolicyWithNo
private static final int NUM_OF_DATANODES = 8;
private static final int NUM_OF_DATANODES_BOUNDARY = 6;
private static final int NUM_OF_DATANODES_MORE_TARGETS = 12;
- private static final Configuration CONF = new HdfsConfiguration();
- private static final NetworkTopology cluster;
- private static final NameNode namenode;
- private static final BlockPlacementPolicy replicator;
+ private final Configuration CONF = new HdfsConfiguration();
+ private NetworkTopology cluster;
+ private NameNode namenode;
+ private BlockPlacementPolicy replicator;
private static final String filename = "/dummyfile.txt";
private final static DatanodeDescriptor dataNodes[] = new DatanodeDescriptor[] {
@@ -91,27 +94,23 @@ public class TestReplicationPolicyWithNo
private final static DatanodeDescriptor NODE =
new DatanodeDescriptor(DFSTestUtil.getDatanodeDescriptor("9.9.9.9", "/d2/r4/n7"));
- static {
- try {
- FileSystem.setDefaultUri(CONF, "hdfs://localhost:0");
- CONF.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
- // Set properties to make HDFS aware of NodeGroup.
- CONF.set("dfs.block.replicator.classname",
- "org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyWithNodeGroup");
- CONF.set(CommonConfigurationKeysPublic.NET_TOPOLOGY_IMPL_KEY,
- "org.apache.hadoop.net.NetworkTopologyWithNodeGroup");
-
- File baseDir = new File(System.getProperty(
- "test.build.data", "build/test/data"), "dfs/");
- CONF.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
- new File(baseDir, "name").getPath());
-
- DFSTestUtil.formatNameNode(CONF);
- namenode = new NameNode(CONF);
- } catch (IOException e) {
- e.printStackTrace();
- throw (RuntimeException)new RuntimeException().initCause(e);
- }
+ @Before
+ public void setUp() throws Exception {
+ FileSystem.setDefaultUri(CONF, "hdfs://localhost:0");
+ CONF.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
+ // Set properties to make HDFS aware of NodeGroup.
+ CONF.set(DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY,
+ BlockPlacementPolicyWithNodeGroup.class.getName());
+ CONF.set(CommonConfigurationKeysPublic.NET_TOPOLOGY_IMPL_KEY,
+ NetworkTopologyWithNodeGroup.class.getName());
+
+ File baseDir = PathUtils.getTestDir(TestReplicationPolicyWithNodeGroup.class);
+
+ CONF.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
+ new File(baseDir, "name").getPath());
+
+ DFSTestUtil.formatNameNode(CONF);
+ namenode = new NameNode(CONF);
final BlockManager bm = namenode.getNamesystem().getBlockManager();
replicator = bm.getBlockPlacementPolicy();
cluster = bm.getDatanodeManager().getNetworkTopology();
@@ -122,6 +121,11 @@ public class TestReplicationPolicyWithNo
setupDataNodeCapacity();
}
+ @After
+ public void tearDown() throws Exception {
+ namenode.stop();
+ }
+
private static void setupDataNodeCapacity() {
for(int i=0; i<NUM_OF_DATANODES; i++) {
dataNodes[i].updateHeartbeat(
@@ -150,6 +154,35 @@ public class TestReplicationPolicyWithNo
return true;
}
+ private DatanodeDescriptor[] chooseTarget(int numOfReplicas) {
+ return chooseTarget(numOfReplicas, dataNodes[0]);
+ }
+
+ private DatanodeDescriptor[] chooseTarget(int numOfReplicas,
+ DatanodeDescriptor writer) {
+ return chooseTarget(numOfReplicas, writer,
+ new ArrayList<DatanodeDescriptor>());
+ }
+
+ private DatanodeDescriptor[] chooseTarget(int numOfReplicas,
+ List<DatanodeDescriptor> chosenNodes) {
+ return chooseTarget(numOfReplicas, dataNodes[0], chosenNodes);
+ }
+
+ private DatanodeDescriptor[] chooseTarget(int numOfReplicas,
+ DatanodeDescriptor writer, List<DatanodeDescriptor> chosenNodes) {
+ return chooseTarget(numOfReplicas, writer, chosenNodes, null);
+ }
+
+ private DatanodeDescriptor[] chooseTarget(
+ int numOfReplicas,
+ DatanodeDescriptor writer,
+ List<DatanodeDescriptor> chosenNodes,
+ Set<Node> excludedNodes) {
+ return replicator.chooseTarget(filename, numOfReplicas, writer, chosenNodes,
+ false, excludedNodes, BLOCK_SIZE);
+ }
+
/**
* In this testcase, client is dataNodes[0]. So the 1st replica should be
* placed on dataNodes[0], the 2nd replica should be placed on
@@ -165,31 +198,26 @@ public class TestReplicationPolicyWithNo
HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 4, 0); // overloaded
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename,
- 0, dataNodes[0], new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(0);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename,
- 1, dataNodes[0], new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(1);
assertEquals(targets.length, 1);
assertEquals(targets[0], dataNodes[0]);
- targets = replicator.chooseTarget(filename,
- 2, dataNodes[0], new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(2);
assertEquals(targets.length, 2);
assertEquals(targets[0], dataNodes[0]);
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
- targets = replicator.chooseTarget(filename,
- 3, dataNodes[0], new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(3);
assertEquals(targets.length, 3);
assertEquals(targets[0], dataNodes[0]);
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
assertTrue(cluster.isOnSameRack(targets[1], targets[2]));
assertFalse(cluster.isOnSameNodeGroup(targets[1], targets[2]));
- targets = replicator.chooseTarget(filename,
- 4, dataNodes[0], new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(4);
assertEquals(targets.length, 4);
assertEquals(targets[0], dataNodes[0]);
assertTrue(cluster.isOnSameRack(targets[1], targets[2]) ||
@@ -219,15 +247,14 @@ public class TestReplicationPolicyWithNo
* node group, and the rest should be placed on a third rack.
* @throws Exception
*/
- public void testChooseTarget2() throws Exception {
- HashMap<Node, Node> excludedNodes;
+ public void testChooseTarget2() throws Exception {
DatanodeDescriptor[] targets;
BlockPlacementPolicyDefault repl = (BlockPlacementPolicyDefault)replicator;
List<DatanodeDescriptor> chosenNodes = new ArrayList<DatanodeDescriptor>();
- excludedNodes = new HashMap<Node, Node>();
- excludedNodes.put(dataNodes[1], dataNodes[1]);
- targets = repl.chooseTarget(4, dataNodes[0], chosenNodes, false,
+ Set<Node> excludedNodes = new HashSet<Node>();
+ excludedNodes.add(dataNodes[1]);
+ targets = repl.chooseTarget(filename, 4, dataNodes[0], chosenNodes, false,
excludedNodes, BLOCK_SIZE);
assertEquals(targets.length, 4);
assertEquals(targets[0], dataNodes[0]);
@@ -242,9 +269,9 @@ public class TestReplicationPolicyWithNo
excludedNodes.clear();
chosenNodes.clear();
- excludedNodes.put(dataNodes[1], dataNodes[1]);
+ excludedNodes.add(dataNodes[1]);
chosenNodes.add(dataNodes[2]);
- targets = repl.chooseTarget(1, dataNodes[0], chosenNodes, true,
+ targets = repl.chooseTarget(filename, 1, dataNodes[0], chosenNodes, true,
excludedNodes, BLOCK_SIZE);
System.out.println("targets=" + Arrays.asList(targets));
assertEquals(2, targets.length);
@@ -269,30 +296,25 @@ public class TestReplicationPolicyWithNo
(HdfsConstants.MIN_BLOCKS_FOR_WRITE-1)*BLOCK_SIZE, 0L, 0, 0); // no space
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename,
- 0, dataNodes[0], new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(0);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename,
- 1, dataNodes[0], new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(1);
assertEquals(targets.length, 1);
assertEquals(targets[0], dataNodes[1]);
- targets = replicator.chooseTarget(filename,
- 2, dataNodes[0], new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(2);
assertEquals(targets.length, 2);
assertEquals(targets[0], dataNodes[1]);
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
- targets = replicator.chooseTarget(filename,
- 3, dataNodes[0], new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(3);
assertEquals(targets.length, 3);
assertEquals(targets[0], dataNodes[1]);
assertTrue(cluster.isOnSameRack(targets[1], targets[2]));
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
- targets = replicator.chooseTarget(filename,
- 4, dataNodes[0], new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(4);
assertEquals(targets.length, 4);
assertEquals(targets[0], dataNodes[1]);
assertTrue(cluster.isNodeGroupAware());
@@ -323,23 +345,19 @@ public class TestReplicationPolicyWithNo
}
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename,
- 0, dataNodes[0], new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(0);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename,
- 1, dataNodes[0], new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(1);
assertEquals(targets.length, 1);
assertFalse(cluster.isOnSameRack(targets[0], dataNodes[0]));
- targets = replicator.chooseTarget(filename,
- 2, dataNodes[0], new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(2);
assertEquals(targets.length, 2);
assertFalse(cluster.isOnSameRack(targets[0], dataNodes[0]));
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
- targets = replicator.chooseTarget(filename,
- 3, dataNodes[0], new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(3);
assertEquals(targets.length, 3);
for(int i=0; i<3; i++) {
assertFalse(cluster.isOnSameRack(targets[i], dataNodes[0]));
@@ -360,21 +378,17 @@ public class TestReplicationPolicyWithNo
public void testChooseTarget5() throws Exception {
setupDataNodeCapacity();
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename,
- 0, NODE, new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(0, NODE);
assertEquals(targets.length, 0);
-
- targets = replicator.chooseTarget(filename,
- 1, NODE, new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+
+ targets = chooseTarget(1, NODE);
assertEquals(targets.length, 1);
-
- targets = replicator.chooseTarget(filename,
- 2, NODE, new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+
+ targets = chooseTarget(2, NODE);
assertEquals(targets.length, 2);
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
-
- targets = replicator.chooseTarget(filename,
- 3, NODE, new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+
+ targets = chooseTarget(3, NODE);
assertEquals(targets.length, 3);
assertTrue(cluster.isOnSameRack(targets[1], targets[2]));
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
@@ -394,23 +408,19 @@ public class TestReplicationPolicyWithNo
chosenNodes.add(dataNodes[0]);
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename,
- 0, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(0, chosenNodes);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename,
- 1, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(1, chosenNodes);
assertEquals(targets.length, 1);
assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0]));
- targets = replicator.chooseTarget(filename,
- 2, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(2, chosenNodes);
assertEquals(targets.length, 2);
assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0]));
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
- targets = replicator.chooseTarget(filename,
- 3, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(3, chosenNodes);
assertEquals(targets.length, 3);
assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0]));
assertFalse(cluster.isOnSameNodeGroup(dataNodes[0], targets[0]));
@@ -431,17 +441,14 @@ public class TestReplicationPolicyWithNo
chosenNodes.add(dataNodes[1]);
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename,
- 0, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(0, chosenNodes);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename,
- 1, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(1, chosenNodes);
assertEquals(targets.length, 1);
assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0]));
- targets = replicator.chooseTarget(filename,
- 2, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(2, chosenNodes);
assertEquals(targets.length, 2);
assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0]) &&
cluster.isOnSameRack(dataNodes[0], targets[1]));
@@ -461,30 +468,26 @@ public class TestReplicationPolicyWithNo
chosenNodes.add(dataNodes[3]);
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename,
- 0, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(0, chosenNodes);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename,
- 1, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(1, chosenNodes);
assertEquals(targets.length, 1);
assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0]));
assertFalse(cluster.isOnSameRack(dataNodes[3], targets[0]));
- targets = replicator.chooseTarget(filename,
- 1, dataNodes[3], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(1, dataNodes[3], chosenNodes);
assertEquals(targets.length, 1);
assertTrue(cluster.isOnSameRack(dataNodes[3], targets[0]));
assertFalse(cluster.isOnSameNodeGroup(dataNodes[3], targets[0]));
assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0]));
- targets = replicator.chooseTarget(filename,
- 2, dataNodes[0], chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(2, chosenNodes);
assertEquals(targets.length, 2);
assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0]));
assertFalse(cluster.isOnSameNodeGroup(dataNodes[0], targets[0]));
- targets = replicator.chooseTarget(filename,
- 2, dataNodes[3], chosenNodes, BLOCK_SIZE);
+
+ targets = chooseTarget(2, dataNodes[3], chosenNodes);
assertEquals(targets.length, 2);
assertTrue(cluster.isOnSameRack(dataNodes[3], targets[0]));
}
@@ -570,21 +573,17 @@ public class TestReplicationPolicyWithNo
}
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename, 0, dataNodesInBoundaryCase[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(0, dataNodesInBoundaryCase[0]);
assertEquals(targets.length, 0);
- targets = replicator.chooseTarget(filename, 1, dataNodesInBoundaryCase[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(1, dataNodesInBoundaryCase[0]);
assertEquals(targets.length, 1);
- targets = replicator.chooseTarget(filename, 2, dataNodesInBoundaryCase[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(2, dataNodesInBoundaryCase[0]);
assertEquals(targets.length, 2);
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
- targets = replicator.chooseTarget(filename, 3, dataNodesInBoundaryCase[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(3, dataNodesInBoundaryCase[0]);
assertEquals(targets.length, 3);
assertTrue(checkTargetsOnDifferentNodeGroup(targets));
}
@@ -607,8 +606,7 @@ public class TestReplicationPolicyWithNo
chosenNodes.add(dataNodesInBoundaryCase[0]);
chosenNodes.add(dataNodesInBoundaryCase[5]);
DatanodeDescriptor[] targets;
- targets = replicator.chooseTarget(filename, 1, dataNodesInBoundaryCase[0],
- chosenNodes, BLOCK_SIZE);
+ targets = chooseTarget(1, dataNodesInBoundaryCase[0], chosenNodes);
assertFalse(cluster.isOnSameNodeGroup(targets[0],
dataNodesInBoundaryCase[0]));
assertFalse(cluster.isOnSameNodeGroup(targets[0],
@@ -625,7 +623,9 @@ public class TestReplicationPolicyWithNo
*/
@Test
public void testChooseMoreTargetsThanNodeGroups() throws Exception {
- // Cleanup nodes in previous tests
+ for(int i=0; i<NUM_OF_DATANODES; i++) {
+ cluster.remove(dataNodes[i]);
+ }
for(int i=0; i<NUM_OF_DATANODES_BOUNDARY; i++) {
DatanodeDescriptor node = dataNodesInBoundaryCase[i];
if (cluster.contains(node)) {
@@ -645,14 +645,12 @@ public class TestReplicationPolicyWithNo
DatanodeDescriptor[] targets;
// Test normal case -- 3 replicas
- targets = replicator.chooseTarget(filename, 3, dataNodesInMoreTargetsCase[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(3, dataNodesInMoreTargetsCase[0]);
assertEquals(targets.length, 3);
assertTrue(checkTargetsOnDifferentNodeGroup(targets));
// Test special case -- replica number over node groups.
- targets = replicator.chooseTarget(filename, 10, dataNodesInMoreTargetsCase[0],
- new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE);
+ targets = chooseTarget(10, dataNodesInMoreTargetsCase[0]);
assertTrue(checkTargetsOnDifferentNodeGroup(targets));
// Verify it only can find 6 targets for placing replicas.
assertEquals(targets.length, 6);
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java Wed Oct 30 22:21:59 2013
@@ -17,13 +17,18 @@
*/
package org.apache.hadoop.hdfs.server.common;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-import static org.mockito.Mockito.doAnswer;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.*;
import java.io.IOException;
import java.io.StringReader;
import java.net.InetSocketAddress;
+import java.text.MessageFormat;
+import java.util.ArrayList;
import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
@@ -33,13 +38,18 @@ import javax.xml.parsers.DocumentBuilder
import javax.xml.parsers.ParserConfigurationException;
+import com.google.common.base.Strings;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.namenode.NameNodeHttpServer;
import org.apache.hadoop.hdfs.web.resources.DoAsParam;
import org.apache.hadoop.hdfs.web.resources.UserParam;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
@@ -51,11 +61,17 @@ import org.apache.hadoop.security.token.
import org.junit.Assert;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
+import static com.google.common.base.Strings.*;
+
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.*;
+
public class TestJspHelper {
@@ -439,4 +455,146 @@ public class TestJspHelper {
ugi.getAuthenticationMethod());
}
}
+
+ @Test
+ public void testSortNodeByFields() throws Exception {
+ DatanodeID dnId1 = new DatanodeID("127.0.0.1", "localhost1", "storage1",
+ 1234, 2345, 3456, 4567);
+ DatanodeID dnId2 = new DatanodeID("127.0.0.2", "localhost2", "storage2",
+ 1235, 2346, 3457, 4568);
+ DatanodeDescriptor dnDesc1 = new DatanodeDescriptor(dnId1, "rack1", 1024,
+ 100, 924, 100, 10, 2);
+ DatanodeDescriptor dnDesc2 = new DatanodeDescriptor(dnId2, "rack2", 2500,
+ 200, 1848, 200, 20, 1);
+ ArrayList<DatanodeDescriptor> live = new ArrayList<DatanodeDescriptor>();
+ live.add(dnDesc1);
+ live.add(dnDesc2);
+
+ JspHelper.sortNodeList(live, "unexists", "ASC");
+ Assert.assertEquals(dnDesc1, live.get(0));
+ Assert.assertEquals(dnDesc2, live.get(1));
+ JspHelper.sortNodeList(live, "unexists", "DSC");
+ Assert.assertEquals(dnDesc2, live.get(0));
+ Assert.assertEquals(dnDesc1, live.get(1));
+
+ // test sorting by capacity
+ JspHelper.sortNodeList(live, "capacity", "ASC");
+ Assert.assertEquals(dnDesc1, live.get(0));
+ Assert.assertEquals(dnDesc2, live.get(1));
+ JspHelper.sortNodeList(live, "capacity", "DSC");
+ Assert.assertEquals(dnDesc2, live.get(0));
+ Assert.assertEquals(dnDesc1, live.get(1));
+
+ // test sorting by used
+ JspHelper.sortNodeList(live, "used", "ASC");
+ Assert.assertEquals(dnDesc1, live.get(0));
+ Assert.assertEquals(dnDesc2, live.get(1));
+ JspHelper.sortNodeList(live, "used", "DSC");
+ Assert.assertEquals(dnDesc2, live.get(0));
+ Assert.assertEquals(dnDesc1, live.get(1));
+
+ // test sorting by nondfsused
+ JspHelper.sortNodeList(live, "nondfsused", "ASC");
+ Assert.assertEquals(dnDesc1, live.get(0));
+ Assert.assertEquals(dnDesc2, live.get(1));
+
+ JspHelper.sortNodeList(live, "nondfsused", "DSC");
+ Assert.assertEquals(dnDesc2, live.get(0));
+ Assert.assertEquals(dnDesc1, live.get(1));
+
+ // test sorting by remaining
+ JspHelper.sortNodeList(live, "remaining", "ASC");
+ Assert.assertEquals(dnDesc1, live.get(0));
+ Assert.assertEquals(dnDesc2, live.get(1));
+
+ JspHelper.sortNodeList(live, "remaining", "DSC");
+ Assert.assertEquals(dnDesc2, live.get(0));
+ Assert.assertEquals(dnDesc1, live.get(1));
+ }
+
+ @Test
+ public void testPrintMethods() throws IOException {
+ JspWriter out = mock(JspWriter.class);
+ HttpServletRequest req = mock(HttpServletRequest.class);
+
+ final StringBuffer buffer = new StringBuffer();
+
+ ArgumentCaptor<String> arg = ArgumentCaptor.forClass(String.class);
+ doAnswer(new Answer<String>() {
+ @Override
+ public String answer(InvocationOnMock invok) {
+ Object[] args = invok.getArguments();
+ buffer.append(args[0]);
+ return null;
+ }
+ }).when(out).print(arg.capture());
+
+
+ JspHelper.createTitle(out, req, "testfile.txt");
+ verify(out, times(1)).print(Mockito.anyString());
+
+ JspHelper.addTableHeader(out);
+ verify(out, times(1 + 2)).print(anyString());
+
+ JspHelper.addTableRow(out, new String[] {" row11", "row12 "});
+ verify(out, times(1 + 2 + 4)).print(anyString());
+
+ JspHelper.addTableRow(out, new String[] {" row11", "row12 "}, 3);
+ verify(out, times(1 + 2 + 4 + 4)).print(Mockito.anyString());
+
+ JspHelper.addTableRow(out, new String[] {" row21", "row22"});
+ verify(out, times(1 + 2 + 4 + 4 + 4)).print(anyString());
+
+ JspHelper.addTableFooter(out);
+ verify(out, times(1 + 2 + 4 + 4 + 4 + 1)).print(anyString());
+
+ assertFalse(isNullOrEmpty(buffer.toString()));
+ }
+
+ @Test
+ public void testReadWriteReplicaState() {
+ try {
+ DataOutputBuffer out = new DataOutputBuffer();
+ DataInputBuffer in = new DataInputBuffer();
+ for (HdfsServerConstants.ReplicaState repState : HdfsServerConstants.ReplicaState
+ .values()) {
+ repState.write(out);
+ in.reset(out.getData(), out.getLength());
+ HdfsServerConstants.ReplicaState result = HdfsServerConstants.ReplicaState
+ .read(in);
+ assertTrue("testReadWrite error !!!", repState == result);
+ out.reset();
+ in.reset();
+ }
+ } catch (Exception ex) {
+ fail("testReadWrite ex error ReplicaState");
+ }
+ }
+
+ @Test
+ public void testUpgradeStatusReport() {
+ short status = 6;
+ int version = 15;
+ String EXPECTED_NOTF_PATTERN = "Upgrade for version {0} has been completed.\nUpgrade is not finalized.";
+ String EXPECTED_PATTERN = "Upgrade for version {0} is in progress. Status = {1}%";
+
+ UpgradeStatusReport upgradeStatusReport = new UpgradeStatusReport(version,
+ status, true);
+ assertTrue(upgradeStatusReport.getVersion() == version);
+ assertTrue(upgradeStatusReport.getUpgradeStatus() == status);
+ assertTrue(upgradeStatusReport.isFinalized());
+
+ assertEquals(MessageFormat.format(EXPECTED_PATTERN, version, status),
+ upgradeStatusReport.getStatusText(true));
+
+ status += 100;
+ upgradeStatusReport = new UpgradeStatusReport(version, status, false);
+ assertFalse(upgradeStatusReport.isFinalized());
+ assertTrue(upgradeStatusReport.toString().equals(
+ MessageFormat.format(EXPECTED_NOTF_PATTERN, version)));
+ assertTrue(upgradeStatusReport.getStatusText(false).equals(
+ MessageFormat.format(EXPECTED_NOTF_PATTERN, version)));
+ assertTrue(upgradeStatusReport.getStatusText(true).equals(
+ MessageFormat.format(EXPECTED_NOTF_PATTERN, version)));
+ }
}
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java Wed Oct 30 22:21:59 2013
@@ -115,11 +115,22 @@ public class DataNodeTestUtils {
}
public static void runBlockScannerForBlock(DataNode dn, ExtendedBlock b) {
+ BlockPoolSliceScanner bpScanner = getBlockPoolScanner(dn, b);
+ bpScanner.verifyBlock(b);
+ }
+
+ private static BlockPoolSliceScanner getBlockPoolScanner(DataNode dn,
+ ExtendedBlock b) {
DataBlockScanner scanner = dn.getBlockScanner();
BlockPoolSliceScanner bpScanner = scanner.getBPScanner(b.getBlockPoolId());
- bpScanner.verifyBlock(b);
+ return bpScanner;
}
-
+
+ public static long getLatestScanTime(DataNode dn, ExtendedBlock b) {
+ BlockPoolSliceScanner scanner = getBlockPoolScanner(dn, b);
+ return scanner.getLastScanTime(b.getLocalBlock());
+ }
+
public static void shutdownBlockScanner(DataNode dn) {
if (dn.blockScanner != null) {
dn.blockScanner.shutdown();