You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by to...@apache.org on 2012/02/10 02:49:24 UTC
svn commit: r1242635 [5/5] - in
/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project: hadoop-hdfs-httpfs/
hadoop-hdfs/ hadoop-hdfs/src/main/bin/
hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/
hadoop-hdfs/src/main/java/ hadoop-hdfs/src/main...
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java?rev=1242635&r1=1242634&r2=1242635&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java Fri Feb 10 01:49:08 2012
@@ -17,14 +17,14 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import static org.junit.Assert.*;
+
import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.Collections;
import java.util.List;
-import junit.framework.TestCase;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
@@ -42,13 +42,15 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.log4j.Level;
+import org.junit.Before;
+import org.junit.Test;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
-public class TestBackupNode extends TestCase {
+public class TestBackupNode {
public static final Log LOG = LogFactory.getLog(TestBackupNode.class);
@@ -59,8 +61,8 @@ public class TestBackupNode extends Test
static final String BASE_DIR = MiniDFSCluster.getBaseDirectory();
- protected void setUp() throws Exception {
- super.setUp();
+ @Before
+ public void setUp() throws Exception {
File baseDir = new File(BASE_DIR);
if(baseDir.exists())
if(!(FileUtil.fullyDelete(baseDir)))
@@ -91,8 +93,7 @@ public class TestBackupNode extends Test
return (BackupNode)NameNode.createNameNode(new String[]{startupOpt.getName()}, c);
}
- void waitCheckpointDone(
- MiniDFSCluster cluster, BackupNode backup, long txid) {
+ void waitCheckpointDone(MiniDFSCluster cluster, long txid) {
long thisCheckpointTxId;
do {
try {
@@ -100,16 +101,16 @@ public class TestBackupNode extends Test
"checkpoint txid should increase above " + txid);
Thread.sleep(1000);
} catch (Exception e) {}
- thisCheckpointTxId = backup.getFSImage().getStorage()
+ // The checkpoint is not done until the nn has received it from the bn
+ thisCheckpointTxId = cluster.getNameNode().getFSImage().getStorage()
.getMostRecentCheckpointTxId();
-
} while (thisCheckpointTxId < txid);
-
// Check that the checkpoint got uploaded to NN successfully
FSImageTestUtil.assertNNHasCheckpoints(cluster,
Collections.singletonList((int)thisCheckpointTxId));
}
+ @Test
public void testCheckpointNode() throws Exception {
testCheckpoint(StartupOption.CHECKPOINT);
}
@@ -119,6 +120,7 @@ public class TestBackupNode extends Test
* and keep in sync, even while the NN rolls, checkpoints
* occur, etc.
*/
+ @Test
public void testBackupNodeTailsEdits() throws Exception {
Configuration conf = new HdfsConfiguration();
HAUtil.setAllowStandbyReads(conf, true);
@@ -237,6 +239,7 @@ public class TestBackupNode extends Test
FSImageTestUtil.assertParallelFilesAreIdentical(dirs, ImmutableSet.of("VERSION"));
}
+ @Test
public void testBackupNode() throws Exception {
testCheckpoint(StartupOption.BACKUP);
}
@@ -277,7 +280,7 @@ public class TestBackupNode extends Test
//
long txid = cluster.getNameNodeRpc().getTransactionID();
backup = startBackupNode(conf, op, 1);
- waitCheckpointDone(cluster, backup, txid);
+ waitCheckpointDone(cluster, txid);
} catch(IOException e) {
LOG.error("Error in TestBackupNode:", e);
assertTrue(e.getLocalizedMessage(), false);
@@ -312,7 +315,7 @@ public class TestBackupNode extends Test
//
backup = startBackupNode(conf, op, 1);
long txid = cluster.getNameNodeRpc().getTransactionID();
- waitCheckpointDone(cluster, backup, txid);
+ waitCheckpointDone(cluster, txid);
for (int i = 0; i < 10; i++) {
fileSys.mkdirs(new Path("file_" + i));
@@ -320,11 +323,11 @@ public class TestBackupNode extends Test
txid = cluster.getNameNodeRpc().getTransactionID();
backup.doCheckpoint();
- waitCheckpointDone(cluster, backup, txid);
+ waitCheckpointDone(cluster, txid);
txid = cluster.getNameNodeRpc().getTransactionID();
backup.doCheckpoint();
- waitCheckpointDone(cluster, backup, txid);
+ waitCheckpointDone(cluster, txid);
// Try BackupNode operations
InetSocketAddress add = backup.getNameNodeAddress();
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java?rev=1242635&r1=1242634&r2=1242635&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java Fri Feb 10 01:49:08 2012
@@ -38,6 +38,9 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
import org.apache.hadoop.hdfs.server.protocol.RegisterCommand;
+import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport;
+import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
+import org.apache.hadoop.hdfs.server.protocol.StorageReport;
import org.junit.After;
import org.junit.Test;
@@ -110,19 +113,22 @@ public class TestDeadDatanode {
new Block(0),
ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK,
null) };
+ StorageReceivedDeletedBlocks[] storageBlocks = {
+ new StorageReceivedDeletedBlocks(reg.getStorageID(), blocks) };
// Ensure blockReceived call from dead datanode is rejected with IOException
try {
- dnp.blockReceivedAndDeleted(reg, poolId, blocks);
+ dnp.blockReceivedAndDeleted(reg, poolId, storageBlocks);
Assert.fail("Expected IOException is not thrown");
} catch (IOException ex) {
// Expected
}
// Ensure blockReport from dead datanode is rejected with IOException
- long[] blockReport = new long[] { 0L, 0L, 0L };
+ StorageBlockReport[] report = { new StorageBlockReport(reg.getStorageID(),
+ new long[] { 0L, 0L, 0L }) };
try {
- dnp.blockReport(reg, poolId, blockReport);
+ dnp.blockReport(reg, poolId, report);
Assert.fail("Expected IOException is not thrown");
} catch (IOException ex) {
// Expected
@@ -130,8 +136,9 @@ public class TestDeadDatanode {
// Ensure heartbeat from dead datanode is rejected with a command
// that asks datanode to register again
- DatanodeCommand[] cmd = dnp.sendHeartbeat(reg, 0, 0, 0, 0, 0, 0, 0)
- .getCommands();
+ StorageReport[] rep = { new StorageReport(reg.getStorageID(), false, 0, 0,
+ 0, 0) };
+ DatanodeCommand[] cmd = dnp.sendHeartbeat(reg, rep, 0, 0, 0).getCommands();
Assert.assertEquals(1, cmd.length);
Assert.assertEquals(cmd[0].getAction(), RegisterCommand.REGISTER
.getAction());
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java?rev=1242635&r1=1242634&r2=1242635&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java Fri Feb 10 01:49:08 2012
@@ -116,12 +116,12 @@ public class TestEditLog extends TestCas
int numTransactions;
short replication = 3;
long blockSize = 64;
- final int id;
+ int startIndex;
- Transactions(FSNamesystem ns, int num, int id) {
+ Transactions(FSNamesystem ns, int numTx, int startIdx) {
namesystem = ns;
- numTransactions = num;
- this.id = id;
+ numTransactions = numTx;
+ startIndex = startIdx;
}
// add a bunch of transactions.
@@ -133,9 +133,8 @@ public class TestEditLog extends TestCas
for (int i = 0; i < numTransactions; i++) {
INodeFileUnderConstruction inode = new INodeFileUnderConstruction(
p, replication, blockSize, 0, "", "", null);
- String fileName = "/filename-" + id + "-" + i;
- editLog.logOpenFile(fileName, inode);
- editLog.logCloseFile(fileName, inode);
+ editLog.logOpenFile("/filename" + (startIndex + i), inode);
+ editLog.logCloseFile("/filename" + (startIndex + i), inode);
editLog.logSync();
}
}
@@ -283,7 +282,8 @@ public class TestEditLog extends TestCas
// Create threads and make them run transactions concurrently.
Thread threadId[] = new Thread[NUM_THREADS];
for (int i = 0; i < NUM_THREADS; i++) {
- Transactions trans = new Transactions(namesystem, NUM_TRANSACTIONS, i);
+ Transactions trans =
+ new Transactions(namesystem, NUM_TRANSACTIONS, i*NUM_TRANSACTIONS);
threadId[i] = new Thread(trans, "TransactionThread-" + i);
threadId[i].start();
}
@@ -296,11 +296,16 @@ public class TestEditLog extends TestCas
i--; // retry
}
}
-
+
+ // Reopen some files as for append
+ Transactions trans =
+ new Transactions(namesystem, NUM_TRANSACTIONS, NUM_TRANSACTIONS / 2);
+ trans.run();
+
// Roll another time to finalize edits_inprogress_3
fsimage.rollEditLog();
- long expectedTxns = (NUM_THREADS * 2 * NUM_TRANSACTIONS) + 2; // +2 for start/end txns
+ long expectedTxns = ((NUM_THREADS+1) * 2 * NUM_TRANSACTIONS) + 2; // +2 for start/end txns
// Verify that we can read in all the transactions that we have written.
// If there were any corruptions, it is likely that the reading in
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java?rev=1242635&r1=1242634&r2=1242635&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java Fri Feb 10 01:49:08 2012
@@ -25,6 +25,7 @@ import javax.management.ObjectName;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.util.VersionInfo;
import org.junit.Test;
import junit.framework.Assert;
@@ -57,6 +58,8 @@ public class TestNameNodeMXBean {
// get attribute "Version"
String version = (String) mbs.getAttribute(mxbeanName, "Version");
Assert.assertEquals(fsn.getVersion(), version);
+ Assert.assertTrue(version.equals(VersionInfo.getVersion()
+ + ", r" + VersionInfo.getRevision()));
// get attribute "Used"
Long used = (Long) mbs.getAttribute(mxbeanName, "Used");
Assert.assertEquals(fsn.getUsed(), used.longValue());
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java?rev=1242635&r1=1242634&r2=1242635&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java Fri Feb 10 01:49:08 2012
@@ -18,9 +18,10 @@
package org.apache.hadoop.hdfs.server.namenode;
import static org.apache.hadoop.hdfs.server.common.Util.fileAsURI;
-
-import static org.junit.Assert.*;
-
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import static org.mockito.Matchers.anyObject;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doThrow;
@@ -28,6 +29,7 @@ import static org.mockito.Mockito.spy;
import java.io.File;
import java.io.IOException;
+import java.io.OutputStream;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -38,16 +40,19 @@ import org.apache.commons.logging.LogFac
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.util.MD5FileUtils;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.GenericTestUtils.DelayAnswer;
import org.apache.log4j.Level;
@@ -565,6 +570,34 @@ public class TestSaveNamespace {
}
}
}
+
+ /**
+ * Test for save namespace should succeed when parent directory renamed with
+ * open lease and destination directory exist.
+ * This test is a regression for HDFS-2827
+ */
+ @Test
+ public void testSaveNamespaceWithRenamedLease() throws Exception {
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(new Configuration())
+ .numDataNodes(1).build();
+ cluster.waitActive();
+ DistributedFileSystem fs = (DistributedFileSystem) cluster.getFileSystem();
+ OutputStream out = null;
+ try {
+ fs.mkdirs(new Path("/test-target"));
+ out = fs.create(new Path("/test-source/foo")); // don't close
+ fs.rename(new Path("/test-source/"), new Path("/test-target/"));
+
+ fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+ cluster.getNameNodeRpc().saveNamespace();
+ fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
+ } finally {
+ IOUtils.cleanup(LOG, out, fs);
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
private void doAnEdit(FSNamesystem fsn, int id) throws IOException {
// Make an edit
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java?rev=1242635&r1=1242634&r2=1242635&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java Fri Feb 10 01:49:08 2012
@@ -53,6 +53,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
+import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.GenericTestUtils.DelayAnswer;
@@ -501,7 +502,7 @@ public class TestDNFencing {
.when(spy).blockReport(
Mockito.<DatanodeRegistration>anyObject(),
Mockito.anyString(),
- Mockito.<long[]>anyObject());
+ Mockito.<StorageBlockReport[]>anyObject());
dn.scheduleAllBlockReport(0);
delayer.waitForCall();