You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ss...@apache.org on 2012/10/16 02:03:59 UTC
svn commit: r1398581 [7/9] - in
/hadoop/common/branches/MR-3902/hadoop-hdfs-project: hadoop-hdfs-httpfs/
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/
hadoop-hdfs-h...
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java Tue Oct 16 00:02:55 2012
@@ -85,6 +85,7 @@ import org.apache.hadoop.security.UserGr
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.VersionInfo;
+import com.google.common.base.Charsets;
import com.google.common.base.Joiner;
/** Utilities for HDFS tests */
@@ -210,27 +211,40 @@ public class DFSTestUtil {
public static void createFile(FileSystem fs, Path fileName, long fileLen,
short replFactor, long seed) throws IOException {
+ createFile(fs, fileName, 1024, fileLen, fs.getDefaultBlockSize(fileName),
+ replFactor, seed);
+ }
+
+ public static void createFile(FileSystem fs, Path fileName, int bufferLen,
+ long fileLen, long blockSize, short replFactor, long seed)
+ throws IOException {
+ assert bufferLen > 0;
if (!fs.mkdirs(fileName.getParent())) {
throw new IOException("Mkdirs failed to create " +
fileName.getParent().toString());
}
FSDataOutputStream out = null;
try {
- out = fs.create(fileName, replFactor);
- byte[] toWrite = new byte[1024];
- Random rb = new Random(seed);
- long bytesToWrite = fileLen;
- while (bytesToWrite>0) {
- rb.nextBytes(toWrite);
- int bytesToWriteNext = (1024<bytesToWrite)?1024:(int)bytesToWrite;
-
- out.write(toWrite, 0, bytesToWriteNext);
- bytesToWrite -= bytesToWriteNext;
+ out = fs.create(fileName, true, fs.getConf()
+ .getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096),
+ replFactor, blockSize);
+ if (fileLen > 0) {
+ byte[] toWrite = new byte[bufferLen];
+ Random rb = new Random(seed);
+ long bytesToWrite = fileLen;
+ while (bytesToWrite>0) {
+ rb.nextBytes(toWrite);
+ int bytesToWriteNext = (bufferLen < bytesToWrite) ? bufferLen
+ : (int) bytesToWrite;
+
+ out.write(toWrite, 0, bytesToWriteNext);
+ bytesToWrite -= bytesToWriteNext;
+ }
}
- out.close();
- out = null;
} finally {
- IOUtils.closeStream(out);
+ if (out != null) {
+ out.close();
+ }
}
}
@@ -273,7 +287,7 @@ public class DFSTestUtil {
* specified target.
*/
public void waitReplication(FileSystem fs, String topdir, short value)
- throws IOException {
+ throws IOException, InterruptedException, TimeoutException {
Path root = new Path(topdir);
/** wait for the replication factor to settle down */
@@ -498,36 +512,44 @@ public class DFSTestUtil {
return fileNames;
}
}
-
- /** wait for the file's replication to be done */
- public static void waitReplication(FileSystem fs, Path fileName,
- short replFactor) throws IOException {
- boolean good;
+
+ /**
+ * Wait for the given file to reach the given replication factor.
+ * @throws TimeoutException if we fail to sufficiently replicate the file
+ */
+ public static void waitReplication(FileSystem fs, Path fileName, short replFactor)
+ throws IOException, InterruptedException, TimeoutException {
+ boolean correctReplFactor;
+ final int ATTEMPTS = 40;
+ int count = 0;
+
do {
- good = true;
+ correctReplFactor = true;
BlockLocation locs[] = fs.getFileBlockLocations(
fs.getFileStatus(fileName), 0, Long.MAX_VALUE);
+ count++;
for (int j = 0; j < locs.length; j++) {
String[] hostnames = locs[j].getNames();
if (hostnames.length != replFactor) {
- String hostNameList = "";
- for (String h : hostnames) hostNameList += h + " ";
- System.out.println("Block " + j + " of file " + fileName
- + " has replication factor " + hostnames.length + "; locations "
- + hostNameList);
- good = false;
- try {
- System.out.println("Waiting for replication factor to drain");
- Thread.sleep(100);
- } catch (InterruptedException e) {}
+ correctReplFactor = false;
+ System.out.println("Block " + j + " of file " + fileName
+ + " has replication factor " + hostnames.length
+ + " (desired " + replFactor + "); locations "
+ + Joiner.on(' ').join(hostnames));
+ Thread.sleep(1000);
break;
}
}
- if (good) {
+ if (correctReplFactor) {
System.out.println("All blocks of file " + fileName
+ " verified to have replication factor " + replFactor);
}
- } while(!good);
+ } while (!correctReplFactor && count < ATTEMPTS);
+
+ if (count == ATTEMPTS) {
+ throw new TimeoutException("Timed out waiting for " + fileName +
+ " to reach " + replFactor + " replicas");
+ }
}
/** delete directory and everything underneath it.*/
@@ -586,12 +608,21 @@ public class DFSTestUtil {
IOUtils.copyBytes(is, os, s.length(), true);
}
- // Returns url content as string.
+ /**
+ * @return url content as string (UTF-8 encoding assumed)
+ */
public static String urlGet(URL url) throws IOException {
+ return new String(urlGetBytes(url), Charsets.UTF_8);
+ }
+
+ /**
+ * @return URL contents as a byte array
+ */
+ public static byte[] urlGetBytes(URL url) throws IOException {
URLConnection conn = url.openConnection();
ByteArrayOutputStream out = new ByteArrayOutputStream();
IOUtils.copyBytes(conn.getInputStream(), out, 4096, true);
- return out.toString();
+ return out.toByteArray();
}
/**
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java Tue Oct 16 00:02:55 2012
@@ -624,14 +624,20 @@ public class MiniDFSCluster {
}
federation = nnTopology.isFederated();
- createNameNodesAndSetConf(
- nnTopology, manageNameDfsDirs, manageNameDfsSharedDirs,
- enableManagedDfsDirsRedundancy,
- format, operation, clusterId, conf);
-
+ try {
+ createNameNodesAndSetConf(
+ nnTopology, manageNameDfsDirs, manageNameDfsSharedDirs,
+ enableManagedDfsDirsRedundancy,
+ format, operation, clusterId, conf);
+ } catch (IOException ioe) {
+ LOG.error("IOE creating namenodes. Permissions dump:\n" +
+ createPermissionsDiagnosisString(data_dir));
+ throw ioe;
+ }
if (format) {
if (data_dir.exists() && !FileUtil.fullyDelete(data_dir)) {
- throw new IOException("Cannot remove data directory: " + data_dir);
+ throw new IOException("Cannot remove data directory: " + data_dir +
+ createPermissionsDiagnosisString(data_dir));
}
}
@@ -647,6 +653,27 @@ public class MiniDFSCluster {
ProxyUsers.refreshSuperUserGroupsConfiguration(conf);
}
+ /**
+ * @return a debug string which can help diagnose an error of why
+ * a given directory might have a permissions error in the context
+ * of a test case
+ */
+ private String createPermissionsDiagnosisString(File path) {
+ StringBuilder sb = new StringBuilder();
+ while (path != null) {
+ sb.append("path '" + path + "': ").append("\n");
+ sb.append("\tabsolute:").append(path.getAbsolutePath()).append("\n");
+ sb.append("\tpermissions: ");
+ sb.append(path.isDirectory() ? "d": "-");
+ sb.append(path.canRead() ? "r" : "-");
+ sb.append(path.canWrite() ? "w" : "-");
+ sb.append(path.canExecute() ? "x" : "-");
+ sb.append("\n");
+ path = path.getParentFile();
+ }
+ return sb.toString();
+ }
+
private void createNameNodesAndSetConf(MiniDFSNNTopology nnTopology,
boolean manageNameDfsDirs, boolean manageNameDfsSharedDirs,
boolean enableManagedDfsDirsRedundancy, boolean format,
@@ -857,8 +884,8 @@ public class MiniDFSCluster {
// After the NN has started, set back the bound ports into
// the conf
conf.set(DFSUtil.addKeySuffixes(
- DFS_NAMENODE_RPC_ADDRESS_KEY, nameserviceId, nnId), NetUtils
- .getHostPortString(nn.getNameNodeAddress()));
+ DFS_NAMENODE_RPC_ADDRESS_KEY, nameserviceId, nnId),
+ nn.getNameNodeAddressHostPortString());
conf.set(DFSUtil.addKeySuffixes(
DFS_NAMENODE_HTTP_ADDRESS_KEY, nameserviceId, nnId), NetUtils
.getHostPortString(nn.getHttpAddress()));
@@ -880,8 +907,8 @@ public class MiniDFSCluster {
* @return URI of the given namenode in MiniDFSCluster
*/
public URI getURI(int nnIndex) {
- InetSocketAddress addr = nameNodes[nnIndex].nameNode.getNameNodeAddress();
- String hostPort = NetUtils.getHostPortString(addr);
+ String hostPort =
+ nameNodes[nnIndex].nameNode.getNameNodeAddressHostPortString();
URI uri = null;
try {
uri = new URI("hdfs://" + hostPort);
@@ -918,7 +945,8 @@ public class MiniDFSCluster {
/**
* wait for the cluster to get out of safemode.
*/
- public void waitClusterUp() {
+ public void waitClusterUp() throws IOException {
+ int i = 0;
if (numDataNodes > 0) {
while (!isClusterUp()) {
try {
@@ -926,6 +954,9 @@ public class MiniDFSCluster {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
+ if (++i > 10) {
+ throw new IOException("Timed out waiting for Mini HDFS Cluster to start");
+ }
}
}
}
@@ -1354,6 +1385,7 @@ public class MiniDFSCluster {
if (ExitUtil.terminateCalled()) {
LOG.fatal("Test resulted in an unexpected exit",
ExitUtil.getFirstExitException());
+ ExitUtil.resetFirstExitException();
throw new AssertionError("Test resulted in an unexpected exit");
}
}
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderLocal.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderLocal.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderLocal.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderLocal.java Tue Oct 16 00:02:55 2012
@@ -61,7 +61,7 @@ public class TestBlockReaderLocal {
* of this class might immediately issue a retry on failure, so it's polite.
*/
@Test
- public void testStablePositionAfterCorruptRead() throws IOException {
+ public void testStablePositionAfterCorruptRead() throws Exception {
final short REPL_FACTOR = 1;
final long FILE_LENGTH = 512L;
cluster.waitActive();
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientReportBadBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientReportBadBlock.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientReportBadBlock.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientReportBadBlock.java Tue Oct 16 00:02:55 2012
@@ -24,6 +24,7 @@ import java.io.IOException;
import java.io.PrintStream;
import java.io.RandomAccessFile;
import java.util.Random;
+import java.util.concurrent.TimeoutException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -199,11 +200,11 @@ public class TestClientReportBadBlock {
}
/**
- * create a file with one block and corrupt some/all of the block replicas.
+ * Create a file with one block and corrupt some/all of the block replicas.
*/
private void createAFileWithCorruptedBlockReplicas(Path filePath, short repl,
int corruptBlockCount) throws IOException, AccessControlException,
- FileNotFoundException, UnresolvedLinkException {
+ FileNotFoundException, UnresolvedLinkException, InterruptedException, TimeoutException {
DFSTestUtil.createFile(dfs, filePath, BLOCK_SIZE, repl, 0);
DFSTestUtil.waitReplication(dfs, filePath, repl);
// Locate the file blocks by asking name node
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestConnCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestConnCache.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestConnCache.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestConnCache.java Tue Oct 16 00:02:55 2012
@@ -25,6 +25,7 @@ import static org.mockito.Mockito.spy;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.Socket;
+import java.security.PrivilegedExceptionAction;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -54,10 +55,12 @@ public class TestConnCache {
static final int BLOCK_SIZE = 4096;
static final int FILE_SIZE = 3 * BLOCK_SIZE;
-
+ final static int CACHE_SIZE = 4;
+ final static long CACHE_EXPIRY_MS = 200;
static Configuration conf = null;
static MiniDFSCluster cluster = null;
static FileSystem fs = null;
+ static SocketCache cache;
static final Path testFile = new Path("/testConnCache.dat");
static byte authenticData[] = null;
@@ -93,6 +96,9 @@ public class TestConnCache {
public static void setupCluster() throws Exception {
final int REPLICATION_FACTOR = 1;
+ /* create a socket cache. There is only one socket cache per jvm */
+ cache = SocketCache.getInstance(CACHE_SIZE, CACHE_EXPIRY_MS);
+
util = new BlockReaderTestUtil(REPLICATION_FACTOR);
cluster = util.getCluster();
conf = util.getConf();
@@ -142,10 +148,7 @@ public class TestConnCache {
* Test the SocketCache itself.
*/
@Test
- public void testSocketCache() throws IOException {
- final int CACHE_SIZE = 4;
- SocketCache cache = new SocketCache(CACHE_SIZE);
-
+ public void testSocketCache() throws Exception {
// Make a client
InetSocketAddress nnAddr =
new InetSocketAddress("localhost", cluster.getNameNodePort());
@@ -159,6 +162,7 @@ public class TestConnCache {
DataNode dn = util.getDataNode(block);
InetSocketAddress dnAddr = dn.getXferAddress();
+
// Make some sockets to the DN
Socket[] dnSockets = new Socket[CACHE_SIZE];
for (int i = 0; i < dnSockets.length; ++i) {
@@ -166,6 +170,7 @@ public class TestConnCache {
dnAddr.getAddress(), dnAddr.getPort());
}
+
// Insert a socket to the NN
Socket nnSock = new Socket(nnAddr.getAddress(), nnAddr.getPort());
cache.put(nnSock, null);
@@ -179,7 +184,7 @@ public class TestConnCache {
assertEquals("NN socket evicted", null, cache.get(nnAddr));
assertTrue("Evicted socket closed", nnSock.isClosed());
-
+
// Lookup the DN socks
for (Socket dnSock : dnSockets) {
assertEquals("Retrieve cached sockets", dnSock, cache.get(dnAddr).sock);
@@ -189,6 +194,51 @@ public class TestConnCache {
assertEquals("Cache is empty", 0, cache.size());
}
+
+ /**
+ * Test the SocketCache expiry.
+ * Verify that socket cache entries expire after the set
+ * expiry time.
+ */
+ @Test
+ public void testSocketCacheExpiry() throws Exception {
+ // Make a client
+ InetSocketAddress nnAddr =
+ new InetSocketAddress("localhost", cluster.getNameNodePort());
+ DFSClient client = new DFSClient(nnAddr, conf);
+
+ // Find out the DN addr
+ LocatedBlock block =
+ client.getNamenode().getBlockLocations(
+ testFile.toString(), 0, FILE_SIZE)
+ .getLocatedBlocks().get(0);
+ DataNode dn = util.getDataNode(block);
+ InetSocketAddress dnAddr = dn.getXferAddress();
+
+
+ // Make some sockets to the DN and put in cache
+ Socket[] dnSockets = new Socket[CACHE_SIZE];
+ for (int i = 0; i < dnSockets.length; ++i) {
+ dnSockets[i] = client.socketFactory.createSocket(
+ dnAddr.getAddress(), dnAddr.getPort());
+ cache.put(dnSockets[i], null);
+ }
+
+ // Client side still has the sockets cached
+ assertEquals(CACHE_SIZE, client.socketCache.size());
+
+ //sleep for a second and see if it expired
+ Thread.sleep(CACHE_EXPIRY_MS + 1000);
+
+ // Client side has no sockets cached
+ assertEquals(0, client.socketCache.size());
+
+ //sleep for another second and see if
+ //the daemon thread runs fine on empty cache
+ Thread.sleep(CACHE_EXPIRY_MS + 1000);
+ }
+
+
/**
* Read a file served entirely from one DN. Seek around and read from
* different offsets. And verify that they all use the same socket.
@@ -229,33 +279,6 @@ public class TestConnCache {
in.close();
}
-
- /**
- * Test that the socket cache can be disabled by setting the capacity to
- * 0. Regression test for HDFS-3365.
- */
- @Test
- public void testDisableCache() throws IOException {
- LOG.info("Starting testDisableCache()");
-
- // Reading with the normally configured filesystem should
- // cache a socket.
- DFSTestUtil.readFile(fs, testFile);
- assertEquals(1, ((DistributedFileSystem)fs).dfs.socketCache.size());
-
- // Configure a new instance with no caching, ensure that it doesn't
- // cache anything
- Configuration confWithoutCache = new Configuration(fs.getConf());
- confWithoutCache.setInt(
- DFSConfigKeys.DFS_CLIENT_SOCKET_CACHE_CAPACITY_KEY, 0);
- FileSystem fsWithoutCache = FileSystem.newInstance(confWithoutCache);
- try {
- DFSTestUtil.readFile(fsWithoutCache, testFile);
- assertEquals(0, ((DistributedFileSystem)fsWithoutCache).dfs.socketCache.size());
- } finally {
- fsWithoutCache.close();
- }
- }
@AfterClass
public static void teardownCluster() throws Exception {
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java Tue Oct 16 00:02:55 2012
@@ -789,8 +789,7 @@ public class TestDFSClientRetries {
* way. See HDFS-3067.
*/
@Test
- public void testRetryOnChecksumFailure()
- throws UnresolvedLinkException, IOException {
+ public void testRetryOnChecksumFailure() throws Exception {
HdfsConfiguration conf = new HdfsConfiguration();
MiniDFSCluster cluster =
new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
@@ -831,7 +830,7 @@ public class TestDFSClientRetries {
}
/** Test client retry with namenode restarting. */
- @Test
+ @Test(timeout=300000)
public void testNamenodeRestart() throws Exception {
namenodeRestartTest(new Configuration(), false);
}
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSMkdirs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSMkdirs.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSMkdirs.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSMkdirs.java Tue Oct 16 00:02:55 2012
@@ -19,7 +19,6 @@ package org.apache.hadoop.hdfs;
import static org.junit.Assert.*;
-import java.io.DataOutputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -29,9 +28,7 @@ import org.apache.hadoop.fs.InvalidPathE
import org.apache.hadoop.fs.ParentNotDirectoryException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
-import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Time;
import org.junit.Test;
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java Tue Oct 16 00:02:55 2012
@@ -35,6 +35,7 @@ import java.util.Arrays;
import java.util.List;
import java.util.Random;
import java.util.Scanner;
+import java.util.zip.DeflaterOutputStream;
import java.util.zip.GZIPOutputStream;
import org.apache.commons.logging.Log;
@@ -52,11 +53,16 @@ import org.apache.hadoop.hdfs.tools.DFSA
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.compress.BZip2Codec;
+import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.ToolRunner;
import org.junit.Test;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;
+
/**
* This class tests commands from DFSShell.
*/
@@ -575,6 +581,8 @@ public class TestDFSShell {
try {
final FileSystem fs = root.getFileSystem(conf);
fs.mkdirs(root);
+
+ // Test the gzip type of files. Magic detection.
OutputStream zout = new GZIPOutputStream(
fs.create(new Path(root, "file.gz")));
Random r = new Random();
@@ -599,7 +607,7 @@ public class TestDFSShell {
Arrays.equals(file.toByteArray(), out.toByteArray()));
// Create a sequence file with a gz extension, to test proper
- // container detection
+ // container detection. Magic detection.
SequenceFile.Writer writer = SequenceFile.createWriter(
conf,
SequenceFile.Writer.file(new Path(root, "file.gz")),
@@ -617,6 +625,45 @@ public class TestDFSShell {
assertTrue("Output doesn't match input",
Arrays.equals("Foo\tBar\n".getBytes(), out.toByteArray()));
out.reset();
+
+ // Test deflate. Extension-based detection.
+ OutputStream dout = new DeflaterOutputStream(
+ fs.create(new Path(root, "file.deflate")));
+ byte[] outbytes = "foo".getBytes();
+ dout.write(outbytes);
+ dout.close();
+ out = new ByteArrayOutputStream();
+ System.setOut(new PrintStream(out));
+ argv = new String[2];
+ argv[0] = "-text";
+ argv[1] = new Path(root, "file.deflate").toString();
+ ret = ToolRunner.run(new FsShell(conf), argv);
+ assertEquals("'-text " + argv[1] + " returned " + ret, 0, ret);
+ assertTrue("Output doesn't match input",
+ Arrays.equals(outbytes, out.toByteArray()));
+ out.reset();
+
+ // Test a simple codec. Extension based detection. We use
+ // Bzip2 cause its non-native.
+ CompressionCodec codec = (CompressionCodec)
+ ReflectionUtils.newInstance(BZip2Codec.class, conf);
+ String extension = codec.getDefaultExtension();
+ Path p = new Path(root, "file." + extension);
+ OutputStream fout = new DataOutputStream(codec.createOutputStream(
+ fs.create(p, true)));
+ byte[] writebytes = "foo".getBytes();
+ fout.write(writebytes);
+ fout.close();
+ out = new ByteArrayOutputStream();
+ System.setOut(new PrintStream(out));
+ argv = new String[2];
+ argv[0] = "-text";
+ argv[1] = new Path(root, p).toString();
+ ret = ToolRunner.run(new FsShell(conf), argv);
+ assertEquals("'-text " + argv[1] + " returned " + ret, 0, ret);
+ assertTrue("Output doesn't match input",
+ Arrays.equals(writebytes, out.toByteArray()));
+ out.reset();
} finally {
if (null != bak) {
System.setOut(bak);
@@ -1284,6 +1331,11 @@ public class TestDFSShell {
public void testGet() throws IOException {
DFSTestUtil.setLogLevel2All(FSInputChecker.LOG);
final Configuration conf = new HdfsConfiguration();
+ // Race can happen here: block scanner is reading the file when test tries
+ // to corrupt the test file, which will fail the test on Windows platform.
+ // Disable block scanner to avoid this race.
+ conf.setInt(DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, -1);
+
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
DistributedFileSystem dfs = (DistributedFileSystem)cluster.getFileSystem();
@@ -1475,4 +1527,95 @@ public class TestDFSShell {
}
+ /**
+ * Delete a file optionally configuring trash on the server and client.
+ */
+ private void deleteFileUsingTrash(
+ boolean serverTrash, boolean clientTrash) throws Exception {
+ // Run a cluster, optionally with trash enabled on the server
+ Configuration serverConf = new HdfsConfiguration();
+ if (serverTrash) {
+ serverConf.setLong(FS_TRASH_INTERVAL_KEY, 1);
+ }
+
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(serverConf)
+ .numDataNodes(1).format(true).build();
+ Configuration clientConf = new Configuration(serverConf);
+
+ // Create a client, optionally with trash enabled
+ if (clientTrash) {
+ clientConf.setLong(FS_TRASH_INTERVAL_KEY, 1);
+ } else {
+ clientConf.setLong(FS_TRASH_INTERVAL_KEY, 0);
+ }
+
+ FsShell shell = new FsShell(clientConf);
+ FileSystem fs = null;
+
+ try {
+ // Create and delete a file
+ fs = cluster.getFileSystem();
+ writeFile(fs, new Path(TEST_ROOT_DIR, "foo"));
+ final String testFile = TEST_ROOT_DIR + "/foo";
+ final String trashFile = shell.getCurrentTrashDir() + "/" + testFile;
+ String[] argv = new String[] { "-rm", testFile };
+ int res = ToolRunner.run(shell, argv);
+ assertEquals("rm failed", 0, res);
+
+ if (serverTrash) {
+ // If the server config was set we should use it unconditionally
+ assertTrue("File not in trash", fs.exists(new Path(trashFile)));
+ } else if (clientTrash) {
+ // If the server config was not set but the client config was
+ // set then we should use it
+ assertTrue("File not in trashed", fs.exists(new Path(trashFile)));
+ } else {
+ // If neither was set then we should not have trashed the file
+ assertFalse("File was not removed", fs.exists(new Path(testFile)));
+ assertFalse("File was trashed", fs.exists(new Path(trashFile)));
+ }
+ } finally {
+ if (fs != null) {
+ fs.close();
+ }
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ /**
+ * Test that the server trash configuration is respected when
+ * the client configuration is not set.
+ */
+ @Test
+ public void testServerConfigRespected() throws Exception {
+ deleteFileUsingTrash(true, false);
+ }
+
+ /**
+ * Test that server trash configuration is respected even when the
+ * client configuration is set.
+ */
+ @Test
+ public void testServerConfigRespectedWithClient() throws Exception {
+ deleteFileUsingTrash(true, true);
+ }
+
+ /**
+ * Test that the client trash configuration is respected when
+ * the server configuration is not set.
+ */
+ @Test
+ public void testClientConfigRespected() throws Exception {
+ deleteFileUsingTrash(false, true);
+ }
+
+ /**
+ * Test that trash is disabled by default.
+ */
+ @Test
+ public void testNoTrashConfig() throws Exception {
+ deleteFileUsingTrash(false, false);
+ }
}
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java Tue Oct 16 00:02:55 2012
@@ -39,7 +39,9 @@ import org.apache.hadoop.fs.FSInputStrea
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil;
import org.apache.hadoop.util.StringUtils;
@@ -49,8 +51,9 @@ import org.junit.Test;
* This tests data transfer protocol handling in the Datanode. It sends
* various forms of wrong data and verifies that Datanode handles it well.
*
- * This test uses the following two file from src/test/.../dfs directory :
- * 1) hadoop-version-dfs-dir.tgz : contains DFS directories.
+ * This test uses the following items from src/test/.../dfs directory :
+ * 1) hadoop-22-dfs-dir.tgz and other tarred pre-upgrade NN / DN
+ * directory images
* 2) hadoop-dfs-dir.txt : checksums that are compared in this test.
* Please read hadoop-dfs-dir.txt for more information.
*/
@@ -62,14 +65,23 @@ public class TestDFSUpgradeFromImage {
new File(MiniDFSCluster.getBaseDirectory());
private static final String HADOOP_DFS_DIR_TXT = "hadoop-dfs-dir.txt";
private static final String HADOOP22_IMAGE = "hadoop-22-dfs-dir.tgz";
-
- public int numDataNodes = 4;
-
+ private static final String HADOOP1_BBW_IMAGE = "hadoop1-bbw.tgz";
+
private static class ReferenceFileInfo {
String path;
long checksum;
}
+ private static final Configuration upgradeConf;
+
+ static {
+ upgradeConf = new HdfsConfiguration();
+ upgradeConf.setInt(DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, -1); // block scanning off
+ if (System.getProperty("test.build.data") == null) { // to allow test to be run outside of Maven
+ System.setProperty("test.build.data", "build/test/data");
+ }
+ }
+
LinkedList<ReferenceFileInfo> refList = new LinkedList<ReferenceFileInfo>();
Iterator<ReferenceFileInfo> refIter;
@@ -137,11 +149,33 @@ public class TestDFSUpgradeFromImage {
}
}
- CRC32 overallChecksum = new CRC32();
+ /**
+ * Try to open a file for reading several times.
+ *
+ * If we fail because lease recovery hasn't completed, retry the open.
+ */
+ private static FSInputStream dfsOpenFileWithRetries(DistributedFileSystem dfs,
+ String pathName) throws IOException {
+ IOException exc = null;
+ for (int tries = 0; tries < 10; tries++) {
+ try {
+ return dfs.dfs.open(pathName);
+ } catch (IOException e) {
+ exc = e;
+ }
+ if (!exc.getMessage().contains("Cannot obtain " +
+ "block length for LocatedBlock")) {
+ throw exc;
+ }
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException ignored) {}
+ }
+ throw exc;
+ }
- private void verifyDir(DistributedFileSystem dfs, Path dir)
- throws IOException {
-
+ private void verifyDir(DistributedFileSystem dfs, Path dir,
+ CRC32 overallChecksum) throws IOException {
FileStatus[] fileArr = dfs.listStatus(dir);
TreeMap<Path, Boolean> fileMap = new TreeMap<Path, Boolean>();
@@ -157,11 +191,11 @@ public class TestDFSUpgradeFromImage {
overallChecksum.update(pathName.getBytes());
if ( isDir ) {
- verifyDir(dfs, path);
+ verifyDir(dfs, path, overallChecksum);
} else {
// this is not a directory. Checksum the file data.
CRC32 fileCRC = new CRC32();
- FSInputStream in = dfs.dfs.open(pathName);
+ FSInputStream in = dfsOpenFileWithRetries(dfs, pathName);
byte[] buf = new byte[4096];
int nRead = 0;
while ( (nRead = in.read(buf, 0, buf.length)) > 0 ) {
@@ -175,7 +209,8 @@ public class TestDFSUpgradeFromImage {
private void verifyFileSystem(DistributedFileSystem dfs) throws IOException {
- verifyDir(dfs, new Path("/"));
+ CRC32 overallChecksum = new CRC32();
+ verifyDir(dfs, new Path("/"), overallChecksum);
verifyChecksum("overallCRC", overallChecksum.getValue());
@@ -237,7 +272,8 @@ public class TestDFSUpgradeFromImage {
@Test
public void testUpgradeFromRel22Image() throws IOException {
unpackStorage(HADOOP22_IMAGE);
- upgradeAndVerify();
+ upgradeAndVerify(new MiniDFSCluster.Builder(upgradeConf).
+ numDataNodes(4));
}
/**
@@ -259,7 +295,8 @@ public class TestDFSUpgradeFromImage {
// Upgrade should now fail
try {
- upgradeAndVerify();
+ upgradeAndVerify(new MiniDFSCluster.Builder(upgradeConf).
+ numDataNodes(4));
fail("Upgrade did not fail with bad MD5");
} catch (IOException ioe) {
String msg = StringUtils.stringifyException(ioe);
@@ -268,21 +305,34 @@ public class TestDFSUpgradeFromImage {
}
}
}
-
- private void upgradeAndVerify() throws IOException {
+
+ static void recoverAllLeases(DFSClient dfs,
+ Path path) throws IOException {
+ String pathStr = path.toString();
+ HdfsFileStatus status = dfs.getFileInfo(pathStr);
+ if (!status.isDir()) {
+ dfs.recoverLease(pathStr);
+ return;
+ }
+ byte prev[] = HdfsFileStatus.EMPTY_NAME;
+ DirectoryListing dirList;
+ do {
+ dirList = dfs.listPaths(pathStr, prev);
+ HdfsFileStatus files[] = dirList.getPartialListing();
+ for (HdfsFileStatus f : files) {
+ recoverAllLeases(dfs, f.getFullPath(path));
+ }
+ prev = dirList.getLastName();
+ } while (dirList.hasMore());
+ }
+
+ private void upgradeAndVerify(MiniDFSCluster.Builder bld)
+ throws IOException {
MiniDFSCluster cluster = null;
try {
- Configuration conf = new HdfsConfiguration();
- if (System.getProperty("test.build.data") == null) { // to allow test to be run outside of Ant
- System.setProperty("test.build.data", "build/test/data");
- }
- conf.setInt(DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, -1); // block scanning off
- cluster = new MiniDFSCluster.Builder(conf)
- .numDataNodes(numDataNodes)
- .format(false)
- .startupOption(StartupOption.UPGRADE)
- .clusterId("testClusterId")
- .build();
+ bld.format(false).startupOption(StartupOption.UPGRADE)
+ .clusterId("testClusterId");
+ cluster = bld.build();
cluster.waitActive();
DistributedFileSystem dfs = (DistributedFileSystem)cluster.getFileSystem();
DFSClient dfsClient = dfs.dfs;
@@ -293,12 +343,27 @@ public class TestDFSUpgradeFromImage {
Thread.sleep(1000);
} catch (InterruptedException ignored) {}
}
-
+ recoverAllLeases(dfsClient, new Path("/"));
verifyFileSystem(dfs);
} finally {
if (cluster != null) { cluster.shutdown(); }
}
}
-
+ /**
+ * Test upgrade from a 1.x image with some blocksBeingWritten
+ */
+ @Test
+ public void testUpgradeFromRel1BBWImage() throws IOException {
+ unpackStorage(HADOOP1_BBW_IMAGE);
+ Configuration conf = new Configuration(upgradeConf);
+ conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY,
+ System.getProperty("test.build.data") + File.separator +
+ "dfs" + File.separator +
+ "data" + File.separator +
+ "data1");
+ upgradeAndVerify(new MiniDFSCluster.Builder(conf).
+ numDataNodes(1).enableManagedDfsDirsRedundancy(false).
+ manageDataDfsDirs(false));
+ }
}
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java Tue Oct 16 00:02:55 2012
@@ -37,7 +37,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
@@ -141,13 +140,6 @@ public class TestDataTransferProtocol {
}
}
- void createFile(FileSystem fs, Path path, int fileLen) throws IOException {
- byte [] arr = new byte[fileLen];
- FSDataOutputStream out = fs.create(path);
- out.write(arr);
- out.close();
- }
-
void readFile(FileSystem fs, Path path, int fileLen) throws IOException {
byte [] arr = new byte[fileLen];
FSDataInputStream in = fs.open(path);
@@ -357,7 +349,9 @@ public class TestDataTransferProtocol {
int fileLen = Math.min(conf.getInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 4096), 4096);
- createFile(fileSys, file, fileLen);
+ DFSTestUtil.createFile(fileSys, file, fileLen, fileLen,
+ fileSys.getDefaultBlockSize(file),
+ fileSys.getDefaultReplication(file), 0L);
// get the first blockid for the file
final ExtendedBlock firstBlock = DFSTestUtil.getFirstBlock(fileSys, file);
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeBlockScanner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeBlockScanner.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeBlockScanner.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeBlockScanner.java Tue Oct 16 00:02:55 2012
@@ -34,14 +34,19 @@ import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Time;
+import org.apache.log4j.Level;
import org.junit.Test;
/**
@@ -59,6 +64,10 @@ public class TestDatanodeBlockScanner {
private static Pattern pattern_blockVerify =
Pattern.compile(".*?(SCAN_PERIOD)\\s*:\\s*(\\d+.*?)");
+
+ static {
+ ((Log4JLogger)FSNamesystem.auditLog).getLogger().setLevel(Level.WARN);
+ }
/**
* This connects to datanode and fetches block verification data.
* It repeats this until the given block has a verification time > newTime.
@@ -173,7 +182,7 @@ public class TestDatanodeBlockScanner {
}
@Test
- public void testBlockCorruptionPolicy() throws IOException {
+ public void testBlockCorruptionPolicy() throws Exception {
Configuration conf = new HdfsConfiguration();
conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 1000L);
Random random = new Random();
@@ -206,12 +215,12 @@ public class TestDatanodeBlockScanner {
assertTrue(MiniDFSCluster.corruptReplica(1, block));
assertTrue(MiniDFSCluster.corruptReplica(2, block));
- // Read the file to trigger reportBadBlocks by client
- try {
- IOUtils.copyBytes(fs.open(file1), new IOUtils.NullOutputStream(),
- conf, true);
- } catch (IOException e) {
- // Ignore exception
+ // Trigger each of the DNs to scan this block immediately.
+ // The block pool scanner doesn't run frequently enough on its own
+ // to notice these, and due to HDFS-1371, the client won't report
+ // bad blocks to the NN when all replicas are bad.
+ for (DataNode dn : cluster.getDataNodes()) {
+ DataNodeTestUtils.runBlockScannerForBlock(dn, block);
}
// We now have the blocks to be marked as corrupt and we get back all
@@ -260,6 +269,7 @@ public class TestDatanodeBlockScanner {
conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 3);
conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 3L);
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, false);
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_PENDING_TIMEOUT_SEC_KEY, 5L);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes).build();
cluster.waitActive();
@@ -267,35 +277,47 @@ public class TestDatanodeBlockScanner {
Path file1 = new Path("/tmp/testBlockCorruptRecovery/file");
DFSTestUtil.createFile(fs, file1, 1024, numReplicas, 0);
ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, file1);
+ final int ITERATIONS = 10;
// Wait until block is replicated to numReplicas
DFSTestUtil.waitReplication(fs, file1, numReplicas);
- // Corrupt numCorruptReplicas replicas of block
- int[] corruptReplicasDNIDs = new int[numCorruptReplicas];
- for (int i=0, j=0; (j != numCorruptReplicas) && (i < numDataNodes); i++) {
- if (corruptReplica(block, i)) {
- corruptReplicasDNIDs[j++] = i;
- LOG.info("successfully corrupted block " + block + " on node "
- + i + " " + cluster.getDataNodes().get(i).getDisplayName());
+ for (int k = 0; ; k++) {
+ // Corrupt numCorruptReplicas replicas of block
+ int[] corruptReplicasDNIDs = new int[numCorruptReplicas];
+ for (int i=0, j=0; (j != numCorruptReplicas) && (i < numDataNodes); i++) {
+ if (corruptReplica(block, i)) {
+ corruptReplicasDNIDs[j++] = i;
+ LOG.info("successfully corrupted block " + block + " on node "
+ + i + " " + cluster.getDataNodes().get(i).getDisplayName());
+ }
+ }
+
+ // Restart the datanodes containing corrupt replicas
+ // so they would be reported to namenode and re-replicated
+ // They MUST be restarted in reverse order from highest to lowest index,
+ // because the act of restarting them removes them from the ArrayList
+ // and causes the indexes of all nodes above them in the list to change.
+ for (int i = numCorruptReplicas - 1; i >= 0 ; i--) {
+ LOG.info("restarting node with corrupt replica: position "
+ + i + " node " + corruptReplicasDNIDs[i] + " "
+ + cluster.getDataNodes().get(corruptReplicasDNIDs[i]).getDisplayName());
+ cluster.restartDataNode(corruptReplicasDNIDs[i]);
}
- }
-
- // Restart the datanodes containing corrupt replicas
- // so they would be reported to namenode and re-replicated
- // They MUST be restarted in reverse order from highest to lowest index,
- // because the act of restarting them removes them from the ArrayList
- // and causes the indexes of all nodes above them in the list to change.
- for (int i = numCorruptReplicas - 1; i >= 0 ; i--) {
- LOG.info("restarting node with corrupt replica: position "
- + i + " node " + corruptReplicasDNIDs[i] + " "
- + cluster.getDataNodes().get(corruptReplicasDNIDs[i]).getDisplayName());
- cluster.restartDataNode(corruptReplicasDNIDs[i]);
- }
- // Loop until all corrupt replicas are reported
- DFSTestUtil.waitCorruptReplicas(fs, cluster.getNamesystem(), file1,
- block, numCorruptReplicas);
+ // Loop until all corrupt replicas are reported
+ try {
+ DFSTestUtil.waitCorruptReplicas(fs, cluster.getNamesystem(), file1,
+ block, numCorruptReplicas);
+ } catch(TimeoutException e) {
+ if (k > ITERATIONS) {
+ throw e;
+ }
+ LOG.info("Timed out waiting for corrupt replicas, trying again, iteration " + k);
+ continue;
+ }
+ break;
+ }
// Loop until the block recovers after replication
DFSTestUtil.waitReplication(fs, file1, numReplicas);
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java Tue Oct 16 00:02:55 2012
@@ -27,6 +27,7 @@ import java.net.InetSocketAddress;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
@@ -92,6 +93,58 @@ public class TestDatanodeRegistration {
}
@Test
+ public void testChangeStorageID() throws Exception {
+ final String DN_IP_ADDR = "127.0.0.1";
+ final String DN_HOSTNAME = "localhost";
+ final int DN_XFER_PORT = 12345;
+ final int DN_INFO_PORT = 12346;
+ final int DN_IPC_PORT = 12347;
+ Configuration conf = new HdfsConfiguration();
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(0)
+ .build();
+ InetSocketAddress addr = new InetSocketAddress(
+ "localhost",
+ cluster.getNameNodePort());
+ DFSClient client = new DFSClient(addr, conf);
+ NamenodeProtocols rpcServer = cluster.getNameNodeRpc();
+
+ // register a datanode
+ DatanodeID dnId = new DatanodeID(DN_IP_ADDR, DN_HOSTNAME,
+ "fake-storage-id", DN_XFER_PORT, DN_INFO_PORT, DN_IPC_PORT);
+ long nnCTime = cluster.getNamesystem().getFSImage().getStorage()
+ .getCTime();
+ StorageInfo mockStorageInfo = mock(StorageInfo.class);
+ doReturn(nnCTime).when(mockStorageInfo).getCTime();
+ doReturn(HdfsConstants.LAYOUT_VERSION).when(mockStorageInfo)
+ .getLayoutVersion();
+ DatanodeRegistration dnReg = new DatanodeRegistration(dnId,
+ mockStorageInfo, null, VersionInfo.getVersion());
+ rpcServer.registerDatanode(dnReg);
+
+ DatanodeInfo[] report = client.datanodeReport(DatanodeReportType.ALL);
+ assertEquals("Expected a registered datanode", 1, report.length);
+
+ // register the same datanode again with a different storage ID
+ dnId = new DatanodeID(DN_IP_ADDR, DN_HOSTNAME,
+ "changed-fake-storage-id", DN_XFER_PORT, DN_INFO_PORT, DN_IPC_PORT);
+ dnReg = new DatanodeRegistration(dnId,
+ mockStorageInfo, null, VersionInfo.getVersion());
+ rpcServer.registerDatanode(dnReg);
+
+ report = client.datanodeReport(DatanodeReportType.ALL);
+ assertEquals("Datanode with changed storage ID not recognized",
+ 1, report.length);
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ @Test
public void testRegistrationWithDifferentSoftwareVersions() throws Exception {
Configuration conf = new HdfsConfiguration();
conf.set(DFSConfigKeys.DFS_DATANODE_MIN_SUPPORTED_NAMENODE_VERSION_KEY, "3.0.0");
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java Tue Oct 16 00:02:55 2012
@@ -22,7 +22,9 @@ import static org.junit.Assert.assertEqu
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.mock;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
@@ -43,6 +45,7 @@ import org.apache.hadoop.fs.FSDataOutput
import org.apache.hadoop.fs.FileChecksum;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.VolumeId;
@@ -53,6 +56,7 @@ import org.apache.hadoop.util.DataChecks
import org.apache.hadoop.util.Time;
import org.apache.log4j.Level;
import org.junit.Test;
+import org.mockito.InOrder;
public class TestDistributedFileSystem {
private static final Random RAN = new Random();
@@ -116,17 +120,38 @@ public class TestDistributedFileSystem {
DFSTestUtil.readFile(fileSys, p);
DFSClient client = ((DistributedFileSystem)fileSys).dfs;
- SocketCache cache = client.socketCache;
- assertEquals(1, cache.size());
fileSys.close();
- assertEquals(0, cache.size());
} finally {
if (cluster != null) {cluster.shutdown();}
}
}
+
+ @Test
+ public void testDFSCloseOrdering() throws Exception {
+ DistributedFileSystem fs = new MyDistributedFileSystem();
+ Path path = new Path("/a");
+ fs.deleteOnExit(path);
+ fs.close();
+
+ InOrder inOrder = inOrder(fs.dfs);
+ inOrder.verify(fs.dfs).closeOutputStreams(eq(false));
+ inOrder.verify(fs.dfs).delete(eq(path.toString()), eq(true));
+ inOrder.verify(fs.dfs).close();
+ }
+ private static class MyDistributedFileSystem extends DistributedFileSystem {
+ MyDistributedFileSystem() {
+ statistics = new FileSystem.Statistics("myhdfs"); // can't mock finals
+ dfs = mock(DFSClient.class);
+ }
+ @Override
+ public boolean exists(Path p) {
+ return true; // trick out deleteOnExit
+ }
+ }
+
@Test
public void testDFSSeekExceptions() throws IOException {
Configuration conf = getTestConfiguration();
@@ -708,9 +733,16 @@ public class TestDistributedFileSystem {
out2.close();
// the two checksums must be different.
- FileChecksum sum1 = dfs.getFileChecksum(path1);
- FileChecksum sum2 = dfs.getFileChecksum(path2);
+ MD5MD5CRC32FileChecksum sum1 =
+ (MD5MD5CRC32FileChecksum)dfs.getFileChecksum(path1);
+ MD5MD5CRC32FileChecksum sum2 =
+ (MD5MD5CRC32FileChecksum)dfs.getFileChecksum(path2);
assertFalse(sum1.equals(sum2));
+
+ // check the individual params
+ assertEquals(DataChecksum.Type.CRC32C, sum1.getCrcType());
+ assertEquals(DataChecksum.Type.CRC32, sum2.getCrcType());
+
} finally {
if (cluster != null) {
cluster.getFileSystem().delete(testBasePath, true);
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java Tue Oct 16 00:02:55 2012
@@ -79,7 +79,6 @@ import org.apache.hadoop.security.UserGr
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Time;
import org.apache.log4j.Level;
-import org.junit.Ignore;
import org.junit.Test;
/**
@@ -585,12 +584,9 @@ public class TestFileCreation {
/**
* Test that file leases are persisted across namenode restarts.
- * This test is currently not triggered because more HDFS work is
- * is needed to handle persistent leases.
*/
- @Ignore
@Test
- public void xxxtestFileCreationNamenodeRestart() throws IOException {
+ public void testFileCreationNamenodeRestart() throws IOException {
Configuration conf = new HdfsConfiguration();
final int MAX_IDLE_TIME = 2000; // 2s
conf.setInt("ipc.client.connection.maxidletime", MAX_IDLE_TIME);
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java Tue Oct 16 00:02:55 2012
@@ -25,6 +25,7 @@ import static org.junit.Assert.fail;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Random;
+import java.util.concurrent.TimeoutException;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
@@ -78,7 +79,8 @@ public class TestFileStatus {
hftpfs = cluster.getHftpFileSystem(0);
dfsClient = new DFSClient(NameNode.getAddress(conf), conf);
file1 = new Path("filestatus.dat");
- writeFile(fs, file1, 1, fileSize, blockSize);
+ DFSTestUtil.createFile(fs, file1, fileSize, fileSize, blockSize, (short) 1,
+ seed);
}
@AfterClass
@@ -86,21 +88,9 @@ public class TestFileStatus {
fs.close();
cluster.shutdown();
}
-
- private static void writeFile(FileSystem fileSys, Path name, int repl,
- int fileSize, int blockSize) throws IOException {
- // Create and write a file that contains three blocks of data
- FSDataOutputStream stm = fileSys.create(name, true,
- HdfsConstants.IO_FILE_BUFFER_SIZE, (short)repl, (long)blockSize);
- byte[] buffer = new byte[fileSize];
- Random rand = new Random(seed);
- rand.nextBytes(buffer);
- stm.write(buffer);
- stm.close();
- }
private void checkFile(FileSystem fileSys, Path name, int repl)
- throws IOException {
+ throws IOException, InterruptedException, TimeoutException {
DFSTestUtil.waitReplication(fileSys, name, (short) repl);
}
@@ -129,7 +119,7 @@ public class TestFileStatus {
/** Test the FileStatus obtained calling getFileStatus on a file */
@Test
- public void testGetFileStatusOnFile() throws IOException {
+ public void testGetFileStatusOnFile() throws Exception {
checkFile(fs, file1, 1);
// test getFileStatus on a file
FileStatus status = fs.getFileStatus(file1);
@@ -217,7 +207,8 @@ public class TestFileStatus {
// create another file that is smaller than a block.
Path file2 = new Path(dir, "filestatus2.dat");
- writeFile(fs, file2, 1, blockSize/4, blockSize);
+ DFSTestUtil.createFile(fs, file2, blockSize/4, blockSize/4, blockSize,
+ (short) 1, seed);
checkFile(fs, file2, 1);
// verify file attributes
@@ -229,7 +220,8 @@ public class TestFileStatus {
// Create another file in the same directory
Path file3 = new Path(dir, "filestatus3.dat");
- writeFile(fs, file3, 1, blockSize/4, blockSize);
+ DFSTestUtil.createFile(fs, file3, blockSize/4, blockSize/4, blockSize,
+ (short) 1, seed);
checkFile(fs, file3, 1);
file3 = fs.makeQualified(file3);
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java Tue Oct 16 00:02:55 2012
@@ -17,8 +17,7 @@
*/
package org.apache.hadoop.hdfs;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
import java.io.IOException;
import java.net.InetSocketAddress;
@@ -28,76 +27,191 @@ import java.util.Map;
import java.util.Random;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.common.GenerationStamp;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.util.Time;
import org.junit.Test;
+
/**
- * This class tests if block replacement request to data nodes work correctly.
+ * This class tests if getblocks request works correctly.
*/
public class TestGetBlocks {
+ private static final int blockSize = 8192;
+ private static final String racks[] = new String[] { "/d1/r1", "/d1/r1",
+ "/d1/r2", "/d1/r2", "/d1/r2", "/d2/r3", "/d2/r3" };
+ private static final int numDatanodes = racks.length;
+
+ /**
+ * Stop the heartbeat of a datanode in the MiniDFSCluster
+ *
+ * @param cluster
+ * The MiniDFSCluster
+ * @param hostName
+ * The hostName of the datanode to be stopped
+ * @return The DataNode whose heartbeat has been stopped
+ */
+ private DataNode stopDataNodeHeartbeat(MiniDFSCluster cluster, String hostName) {
+ for (DataNode dn : cluster.getDataNodes()) {
+ if (dn.getDatanodeId().getHostName().equals(hostName)) {
+ DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);
+ return dn;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Test if the datanodes returned by
+ * {@link ClientProtocol#getBlockLocations(String, long, long)} is correct
+ * when stale nodes checking is enabled. Also test during the scenario when 1)
+ * stale nodes checking is enabled, 2) a writing is going on, 3) a datanode
+ * becomes stale happen simultaneously
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testReadSelectNonStaleDatanode() throws Exception {
+ HdfsConfiguration conf = new HdfsConfiguration();
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_CHECK_STALE_DATANODE_KEY, true);
+ long staleInterval = 30 * 1000 * 60;
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_KEY,
+ staleInterval);
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(numDatanodes).racks(racks).build();
+
+ cluster.waitActive();
+ InetSocketAddress addr = new InetSocketAddress("localhost",
+ cluster.getNameNodePort());
+ DFSClient client = new DFSClient(addr, conf);
+ List<DatanodeDescriptor> nodeInfoList = cluster.getNameNode()
+ .getNamesystem().getBlockManager().getDatanodeManager()
+ .getDatanodeListForReport(DatanodeReportType.LIVE);
+ assertEquals("Unexpected number of datanodes", numDatanodes,
+ nodeInfoList.size());
+ FileSystem fileSys = cluster.getFileSystem();
+ FSDataOutputStream stm = null;
+ try {
+ // do the writing but do not close the FSDataOutputStream
+ // in order to mimic the ongoing writing
+ final Path fileName = new Path("/file1");
+ stm = fileSys.create(fileName, true,
+ fileSys.getConf().getInt(
+ CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096),
+ (short) 3, blockSize);
+ stm.write(new byte[(blockSize * 3) / 2]);
+ // We do not close the stream so that
+ // the writing seems to be still ongoing
+ stm.hflush();
+
+ LocatedBlocks blocks = client.getNamenode().getBlockLocations(
+ fileName.toString(), 0, blockSize);
+ DatanodeInfo[] nodes = blocks.get(0).getLocations();
+ assertEquals(nodes.length, 3);
+ DataNode staleNode = null;
+ DatanodeDescriptor staleNodeInfo = null;
+ // stop the heartbeat of the first node
+ staleNode = this.stopDataNodeHeartbeat(cluster, nodes[0].getHostName());
+ assertNotNull(staleNode);
+ // set the first node as stale
+ staleNodeInfo = cluster.getNameNode().getNamesystem().getBlockManager()
+ .getDatanodeManager()
+ .getDatanode(staleNode.getDatanodeId());
+ staleNodeInfo.setLastUpdate(Time.now() - staleInterval - 1);
+
+ LocatedBlocks blocksAfterStale = client.getNamenode().getBlockLocations(
+ fileName.toString(), 0, blockSize);
+ DatanodeInfo[] nodesAfterStale = blocksAfterStale.get(0).getLocations();
+ assertEquals(nodesAfterStale.length, 3);
+ assertEquals(nodesAfterStale[2].getHostName(), nodes[0].getHostName());
+
+ // restart the staleNode's heartbeat
+ DataNodeTestUtils.setHeartbeatsDisabledForTests(staleNode, false);
+ // reset the first node as non-stale, so as to avoid two stale nodes
+ staleNodeInfo.setLastUpdate(Time.now());
+
+ LocatedBlock lastBlock = client.getLocatedBlocks(fileName.toString(), 0,
+ Long.MAX_VALUE).getLastLocatedBlock();
+ nodes = lastBlock.getLocations();
+ assertEquals(nodes.length, 3);
+ // stop the heartbeat of the first node for the last block
+ staleNode = this.stopDataNodeHeartbeat(cluster, nodes[0].getHostName());
+ assertNotNull(staleNode);
+ // set the node as stale
+ cluster.getNameNode().getNamesystem().getBlockManager()
+ .getDatanodeManager()
+ .getDatanode(staleNode.getDatanodeId())
+ .setLastUpdate(Time.now() - staleInterval - 1);
+
+ LocatedBlock lastBlockAfterStale = client.getLocatedBlocks(
+ fileName.toString(), 0, Long.MAX_VALUE).getLastLocatedBlock();
+ nodesAfterStale = lastBlockAfterStale.getLocations();
+ assertEquals(nodesAfterStale.length, 3);
+ assertEquals(nodesAfterStale[2].getHostName(), nodes[0].getHostName());
+ } finally {
+ if (stm != null) {
+ stm.close();
+ }
+ cluster.shutdown();
+ }
+ }
+
/** test getBlocks */
@Test
public void testGetBlocks() throws Exception {
final Configuration CONF = new HdfsConfiguration();
- final short REPLICATION_FACTOR = (short)2;
+ final short REPLICATION_FACTOR = (short) 2;
final int DEFAULT_BLOCK_SIZE = 1024;
- final Random r = new Random();
-
+
CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DEFAULT_BLOCK_SIZE);
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(CONF)
- .numDataNodes(REPLICATION_FACTOR)
- .build();
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(CONF).numDataNodes(
+ REPLICATION_FACTOR).build();
try {
cluster.waitActive();
-
- // create a file with two blocks
- FileSystem fs = cluster.getFileSystem();
- FSDataOutputStream out = fs.create(new Path("/tmp.txt"),
- REPLICATION_FACTOR);
- byte [] data = new byte[1024];
- long fileLen = 2*DEFAULT_BLOCK_SIZE;
- long bytesToWrite = fileLen;
- while( bytesToWrite > 0 ) {
- r.nextBytes(data);
- int bytesToWriteNext = (1024<bytesToWrite)?1024:(int)bytesToWrite;
- out.write(data, 0, bytesToWriteNext);
- bytesToWrite -= bytesToWriteNext;
- }
- out.close();
+ long fileLen = 2 * DEFAULT_BLOCK_SIZE;
+ DFSTestUtil.createFile(cluster.getFileSystem(), new Path("/tmp.txt"),
+ fileLen, REPLICATION_FACTOR, 0L);
// get blocks & data nodes
List<LocatedBlock> locatedBlocks;
- DatanodeInfo[] dataNodes=null;
+ DatanodeInfo[] dataNodes = null;
boolean notWritten;
do {
- final DFSClient dfsclient = new DFSClient(NameNode.getAddress(CONF), CONF);
- locatedBlocks = dfsclient.getNamenode().
- getBlockLocations("/tmp.txt", 0, fileLen).getLocatedBlocks();
+ final DFSClient dfsclient = new DFSClient(NameNode.getAddress(CONF),
+ CONF);
+ locatedBlocks = dfsclient.getNamenode()
+ .getBlockLocations("/tmp.txt", 0, fileLen).getLocatedBlocks();
assertEquals(2, locatedBlocks.size());
notWritten = false;
- for(int i=0; i<2; i++) {
+ for (int i = 0; i < 2; i++) {
dataNodes = locatedBlocks.get(i).getLocations();
- if(dataNodes.length != REPLICATION_FACTOR) {
+ if (dataNodes.length != REPLICATION_FACTOR) {
notWritten = true;
try {
Thread.sleep(10);
- } catch(InterruptedException e) {
+ } catch (InterruptedException e) {
}
break;
}
}
- } while(notWritten);
-
+ } while (notWritten);
+
// get RPC client to namenode
InetSocketAddress addr = new InetSocketAddress("localhost",
cluster.getNameNodePort());
@@ -122,7 +236,7 @@ public class TestGetBlocks {
assertEquals(locs[0].getStorageIDs().length, 2);
// get blocks of size 0 from dataNodes[0]
- getBlocksWithException(namenode, dataNodes[0], 0);
+ getBlocksWithException(namenode, dataNodes[0], 0);
// get blocks of size -1 from dataNodes[0]
getBlocksWithException(namenode, dataNodes[0], -1);
@@ -136,46 +250,39 @@ public class TestGetBlocks {
}
private void getBlocksWithException(NamenodeProtocol namenode,
- DatanodeInfo datanode,
- long size) throws IOException {
+ DatanodeInfo datanode, long size) throws IOException {
boolean getException = false;
try {
- namenode.getBlocks(DFSTestUtil.getLocalDatanodeInfo(), 2);
- } catch(RemoteException e) {
+ namenode.getBlocks(DFSTestUtil.getLocalDatanodeInfo(), 2);
+ } catch (RemoteException e) {
getException = true;
assertTrue(e.getClassName().contains("HadoopIllegalArgumentException"));
}
assertTrue(getException);
}
-
+
@Test
public void testBlockKey() {
Map<Block, Long> map = new HashMap<Block, Long>();
final Random RAN = new Random();
final long seed = RAN.nextLong();
- System.out.println("seed=" + seed);
+ System.out.println("seed=" + seed);
RAN.setSeed(seed);
- long[] blkids = new long[10];
- for(int i = 0; i < blkids.length; i++) {
+ long[] blkids = new long[10];
+ for (int i = 0; i < blkids.length; i++) {
blkids[i] = 1000L + RAN.nextInt(100000);
map.put(new Block(blkids[i], 0, blkids[i]), blkids[i]);
}
System.out.println("map=" + map.toString().replace(",", "\n "));
-
- for(int i = 0; i < blkids.length; i++) {
- Block b = new Block(blkids[i], 0, GenerationStamp.GRANDFATHER_GENERATION_STAMP);
+
+ for (int i = 0; i < blkids.length; i++) {
+ Block b = new Block(blkids[i], 0,
+ GenerationStamp.GRANDFATHER_GENERATION_STAMP);
Long v = map.get(b);
System.out.println(b + " => " + v);
assertEquals(blkids[i], v.longValue());
}
}
- /**
- * @param args
- */
- public static void main(String[] args) throws Exception {
- (new TestGetBlocks()).testGetBlocks();
- }
-
}
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSTrash.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSTrash.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSTrash.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSTrash.java Tue Oct 16 00:02:55 2012
@@ -23,11 +23,6 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.TestTrash;
-import org.apache.hadoop.fs.Trash;
-
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -62,53 +57,4 @@ public class TestHDFSTrash {
conf.set(DFSConfigKeys.FS_DEFAULT_NAME_KEY, fs.getUri().toString());
TestTrash.trashNonDefaultFS(conf);
}
-
- /** Clients should always use trash if enabled server side */
- @Test
- public void testTrashEnabledServerSide() throws IOException {
- Configuration serverConf = new HdfsConfiguration();
- Configuration clientConf = new Configuration();
-
- // Enable trash on the server and client
- serverConf.setLong(FS_TRASH_INTERVAL_KEY, 1);
- clientConf.setLong(FS_TRASH_INTERVAL_KEY, 1);
-
- MiniDFSCluster cluster2 = null;
- try {
- cluster2 = new MiniDFSCluster.Builder(serverConf).numDataNodes(1).build();
- FileSystem fs = cluster2.getFileSystem();
- assertTrue(new Trash(fs, clientConf).isEnabled());
-
- // Disabling trash on the client is ignored
- clientConf.setLong(FS_TRASH_INTERVAL_KEY, 0);
- assertTrue(new Trash(fs, clientConf).isEnabled());
- } finally {
- if (cluster2 != null) cluster2.shutdown();
- }
- }
-
- /** Clients should always use trash if enabled client side */
- @Test
- public void testTrashEnabledClientSide() throws IOException {
- Configuration serverConf = new HdfsConfiguration();
- Configuration clientConf = new Configuration();
-
- // Disable server side
- serverConf.setLong(FS_TRASH_INTERVAL_KEY, 0);
-
- MiniDFSCluster cluster2 = null;
- try {
- cluster2 = new MiniDFSCluster.Builder(serverConf).numDataNodes(1).build();
-
- // Client side is disabled by default
- FileSystem fs = cluster2.getFileSystem();
- assertFalse(new Trash(fs, clientConf).isEnabled());
-
- // Enabling on the client works even though its disabled on the server
- clientConf.setLong(FS_TRASH_INTERVAL_KEY, 1);
- assertTrue(new Trash(fs, clientConf).isEnabled());
- } finally {
- if (cluster2 != null) cluster2.shutdown();
- }
- }
}
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java Tue Oct 16 00:02:55 2012
@@ -19,13 +19,11 @@
package org.apache.hadoop.hdfs;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertSame;
-
+import static org.junit.Assert.*;
import java.io.IOException;
import java.lang.reflect.Field;
+import java.net.ServerSocket;
+import java.net.Socket;
import java.net.URI;
import java.security.PrivilegedExceptionAction;
@@ -43,6 +41,8 @@ public class TestHftpDelegationToken {
@Test
public void testHdfsDelegationToken() throws Exception {
+ SecurityUtilTestHelper.setTokenServiceUseIp(true);
+
final Configuration conf = new Configuration();
conf.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos");
UserGroupInformation.setConfiguration(conf);
@@ -55,7 +55,7 @@ public class TestHftpDelegationToken {
new Text("127.0.0.1:8020"));
user.addToken(token);
Token<?> token2 = new Token<TokenIdentifier>
- (null, null, new Text("other token"), new Text("127.0.0.1:8020"));
+ (null, null, new Text("other token"), new Text("127.0.0.1:8021"));
user.addToken(token2);
assertEquals("wrong tokens in user", 2, user.getTokens().size());
FileSystem fs =
@@ -138,6 +138,53 @@ public class TestHftpDelegationToken {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_KEY, 5);
}
+
+ @Test
+ public void testInsecureRemoteCluster() throws Exception {
+ final ServerSocket socket = new ServerSocket(0); // just reserve a port
+ socket.close();
+ Configuration conf = new Configuration();
+ URI fsUri = URI.create("hsftp://localhost:"+socket.getLocalPort());
+ assertNull(FileSystem.newInstance(fsUri, conf).getDelegationToken(null));
+ }
+
+ @Test
+ public void testSecureClusterError() throws Exception {
+ final ServerSocket socket = new ServerSocket(0);
+ Thread t = new Thread() {
+ @Override
+ public void run() {
+ while (true) { // fetching does a few retries
+ try {
+ Socket s = socket.accept();
+ s.getOutputStream().write(1234);
+ s.shutdownOutput();
+ } catch (Exception e) {
+ break;
+ }
+ }
+ }
+ };
+ t.start();
+
+ try {
+ Configuration conf = new Configuration();
+ URI fsUri = URI.create("hsftp://localhost:"+socket.getLocalPort());
+ Exception ex = null;
+ try {
+ FileSystem.newInstance(fsUri, conf).getDelegationToken(null);
+ } catch (Exception e) {
+ ex = e;
+ }
+ assertNotNull(ex);
+ assertNotNull(ex.getCause());
+ assertEquals("Unexpected end of file from server",
+ ex.getCause().getMessage());
+ } finally {
+ t.interrupt();
+ }
+ }
+
private void checkTokenSelection(HftpFileSystem fs,
int port,
Configuration conf) throws IOException {
@@ -220,4 +267,4 @@ public class TestHftpDelegationToken {
@Override
protected void initDelegationToken() throws IOException {}
}
-}
\ No newline at end of file
+}
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpFileSystem.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpFileSystem.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpFileSystem.java Tue Oct 16 00:02:55 2012
@@ -102,9 +102,15 @@ public class TestHftpFileSystem {
@AfterClass
public static void tearDown() throws IOException {
- hdfs.close();
- hftpFs.close();
- cluster.shutdown();
+ if (hdfs != null) {
+ hdfs.close();
+ }
+ if (hftpFs != null) {
+ hftpFs.close();
+ }
+ if (cluster != null) {
+ cluster.shutdown();
+ }
}
/**
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java Tue Oct 16 00:02:55 2012
@@ -53,19 +53,23 @@ public class TestHftpURLTimeouts {
boolean timedout = false;
HftpFileSystem fs = (HftpFileSystem)FileSystem.get(uri, conf);
- HttpURLConnection conn = fs.openConnection("/", "");
- timedout = false;
try {
- // this will consume the only slot in the backlog
- conn.getInputStream();
- } catch (SocketTimeoutException ste) {
- timedout = true;
- assertEquals("Read timed out", ste.getMessage());
+ HttpURLConnection conn = fs.openConnection("/", "");
+ timedout = false;
+ try {
+ // this will consume the only slot in the backlog
+ conn.getInputStream();
+ } catch (SocketTimeoutException ste) {
+ timedout = true;
+ assertEquals("Read timed out", ste.getMessage());
+ } finally {
+ if (conn != null) conn.disconnect();
+ }
+ assertTrue("read timedout", timedout);
+ assertTrue("connect timedout", checkConnectTimeout(fs, false));
} finally {
- if (conn != null) conn.disconnect();
+ fs.close();
}
- assertTrue("read timedout", timedout);
- assertTrue("connect timedout", checkConnectTimeout(fs, false));
}
@Test
@@ -79,20 +83,24 @@ public class TestHftpURLTimeouts {
boolean timedout = false;
HsftpFileSystem fs = (HsftpFileSystem)FileSystem.get(uri, conf);
- HttpURLConnection conn = null;
- timedout = false;
try {
- // this will consume the only slot in the backlog
- conn = fs.openConnection("/", "");
- } catch (SocketTimeoutException ste) {
- // SSL expects a negotiation, so it will timeout on read, unlike hftp
- timedout = true;
- assertEquals("Read timed out", ste.getMessage());
+ HttpURLConnection conn = null;
+ timedout = false;
+ try {
+ // this will consume the only slot in the backlog
+ conn = fs.openConnection("/", "");
+ } catch (SocketTimeoutException ste) {
+ // SSL expects a negotiation, so it will timeout on read, unlike hftp
+ timedout = true;
+ assertEquals("Read timed out", ste.getMessage());
+ } finally {
+ if (conn != null) conn.disconnect();
+ }
+ assertTrue("ssl read connect timedout", timedout);
+ assertTrue("connect timedout", checkConnectTimeout(fs, true));
} finally {
- if (conn != null) conn.disconnect();
+ fs.close();
}
- assertTrue("ssl read connect timedout", timedout);
- assertTrue("connect timedout", checkConnectTimeout(fs, true));
}
private boolean checkConnectTimeout(HftpFileSystem fs, boolean ignoreReadTimeout)
Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestInjectionForSimulatedStorage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestInjectionForSimulatedStorage.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestInjectionForSimulatedStorage.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestInjectionForSimulatedStorage.java Tue Oct 16 00:02:55 2012
@@ -52,22 +52,6 @@ public class TestInjectionForSimulatedSt
private static final Log LOG = LogFactory.getLog(
"org.apache.hadoop.hdfs.TestInjectionForSimulatedStorage");
-
- private void writeFile(FileSystem fileSys, Path name, int repl)
- throws IOException {
- // create and write a file that contains three blocks of data
- FSDataOutputStream stm = fileSys.create(name, true, fileSys.getConf()
- .getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096),
- (short) repl, blockSize);
- byte[] buffer = new byte[filesize];
- for (int i=0; i<buffer.length; i++) {
- buffer[i] = '1';
- }
- stm.write(buffer);
- stm.close();
- }
-
- // Waits for all of the blocks to have expected replication
// Waits for all of the blocks to have expected replication
private void waitForBlockReplication(String filename,
@@ -149,7 +133,8 @@ public class TestInjectionForSimulatedSt
cluster.getNameNodePort()),
conf);
- writeFile(cluster.getFileSystem(), testPath, numDataNodes);
+ DFSTestUtil.createFile(cluster.getFileSystem(), testPath, filesize,
+ filesize, blockSize, (short) numDataNodes, 0L);
waitForBlockReplication(testFile, dfsClient.getNamenode(), numDataNodes, 20);
Iterable<Block>[] blocksList = cluster.getAllBlockReports(bpid);