You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by su...@apache.org on 2012/12/03 19:05:06 UTC
svn commit: r1416603 [2/2] - in
/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/ src/main/java/org/apache/hadoop/hdfs/server/namenode/
src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/
src/main/native/ s...
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java Mon Dec 3 18:04:51 2012
@@ -30,7 +30,9 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Options;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
@@ -49,6 +51,10 @@ public class TestLease {
).getLeaseByPath(src.toString()) != null;
}
+ static int leaseCount(MiniDFSCluster cluster) {
+ return NameNodeAdapter.getLeaseManager(cluster.getNamesystem()).countLease();
+ }
+
static final String dirString = "/test/lease";
final Path dir = new Path(dirString);
static final Log LOG = LogFactory.getLog(TestLease.class);
@@ -127,6 +133,96 @@ public class TestLease {
}
@Test
+ public void testLeaseAfterRename() throws Exception {
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
+ try {
+ Path p = new Path("/test-file");
+ Path d = new Path("/test-d");
+ Path d2 = new Path("/test-d-other");
+
+ // open a file to get a lease
+ FileSystem fs = cluster.getFileSystem();
+ FSDataOutputStream out = fs.create(p);
+ out.writeBytes("something");
+ //out.hsync();
+ Assert.assertTrue(hasLease(cluster, p));
+ Assert.assertEquals(1, leaseCount(cluster));
+
+ // just to ensure first fs doesn't have any logic to twiddle leases
+ DistributedFileSystem fs2 = (DistributedFileSystem) FileSystem.newInstance(fs.getUri(), fs.getConf());
+
+ // rename the file into an existing dir
+ LOG.info("DMS: rename file into dir");
+ Path pRenamed = new Path(d, p.getName());
+ fs2.mkdirs(d);
+ fs2.rename(p, pRenamed);
+ Assert.assertFalse(p+" exists", fs2.exists(p));
+ Assert.assertTrue(pRenamed+" not found", fs2.exists(pRenamed));
+ Assert.assertFalse("has lease for "+p, hasLease(cluster, p));
+ Assert.assertTrue("no lease for "+pRenamed, hasLease(cluster, pRenamed));
+ Assert.assertEquals(1, leaseCount(cluster));
+
+ // rename the parent dir to a new non-existent dir
+ LOG.info("DMS: rename parent dir");
+ Path pRenamedAgain = new Path(d2, pRenamed.getName());
+ fs2.rename(d, d2);
+ // src gone
+ Assert.assertFalse(d+" exists", fs2.exists(d));
+ Assert.assertFalse("has lease for "+pRenamed, hasLease(cluster, pRenamed));
+ // dst checks
+ Assert.assertTrue(d2+" not found", fs2.exists(d2));
+ Assert.assertTrue(pRenamedAgain+" not found", fs2.exists(pRenamedAgain));
+ Assert.assertTrue("no lease for "+pRenamedAgain, hasLease(cluster, pRenamedAgain));
+ Assert.assertEquals(1, leaseCount(cluster));
+
+ // rename the parent dir to existing dir
+ // NOTE: rename w/o options moves paths into existing dir
+ LOG.info("DMS: rename parent again");
+ pRenamed = pRenamedAgain;
+ pRenamedAgain = new Path(new Path(d, d2.getName()), p.getName());
+ fs2.mkdirs(d);
+ fs2.rename(d2, d);
+ // src gone
+ Assert.assertFalse(d2+" exists", fs2.exists(d2));
+ Assert.assertFalse("no lease for "+pRenamed, hasLease(cluster, pRenamed));
+ // dst checks
+ Assert.assertTrue(d+" not found", fs2.exists(d));
+ Assert.assertTrue(pRenamedAgain +" not found", fs2.exists(pRenamedAgain));
+ Assert.assertTrue("no lease for "+pRenamedAgain, hasLease(cluster, pRenamedAgain));
+ Assert.assertEquals(1, leaseCount(cluster));
+
+ // rename with opts to non-existent dir
+ pRenamed = pRenamedAgain;
+ pRenamedAgain = new Path(d2, p.getName());
+ fs2.rename(pRenamed.getParent(), d2, Options.Rename.OVERWRITE);
+ // src gone
+ Assert.assertFalse(pRenamed.getParent() +" not found", fs2.exists(pRenamed.getParent()));
+ Assert.assertFalse("has lease for "+pRenamed, hasLease(cluster, pRenamed));
+ // dst checks
+ Assert.assertTrue(d2+" not found", fs2.exists(d2));
+ Assert.assertTrue(pRenamedAgain+" not found", fs2.exists(pRenamedAgain));
+ Assert.assertTrue("no lease for "+pRenamedAgain, hasLease(cluster, pRenamedAgain));
+ Assert.assertEquals(1, leaseCount(cluster));
+
+ // rename with opts to existing dir
+ // NOTE: rename with options will not move paths into the existing dir
+ pRenamed = pRenamedAgain;
+ pRenamedAgain = new Path(d, p.getName());
+ fs2.rename(pRenamed.getParent(), d, Options.Rename.OVERWRITE);
+ // src gone
+ Assert.assertFalse(pRenamed.getParent() +" not found", fs2.exists(pRenamed.getParent()));
+ Assert.assertFalse("has lease for "+pRenamed, hasLease(cluster, pRenamed));
+ // dst checks
+ Assert.assertTrue(d+" not found", fs2.exists(d));
+ Assert.assertTrue(pRenamedAgain+" not found", fs2.exists(pRenamedAgain));
+ Assert.assertTrue("no lease for "+pRenamedAgain, hasLease(cluster, pRenamedAgain));
+ Assert.assertEquals(1, leaseCount(cluster));
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
public void testLease() throws Exception {
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
try {
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java Mon Dec 3 18:04:51 2012
@@ -35,6 +35,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.HAUtil;
@@ -103,6 +104,9 @@ public class TestBackupNode {
BackupNode bn = (BackupNode)NameNode.createNameNode(
new String[]{startupOpt.getName()}, c);
assertTrue(bn.getRole() + " must be in SafeMode.", bn.isInSafeMode());
+ assertTrue(bn.getRole() + " must be in StandbyState",
+ bn.getNamesystem().getHAState()
+ .equalsIgnoreCase(HAServiceState.STANDBY.name()));
return bn;
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsLimits.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsLimits.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsLimits.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsLimits.java Mon Dec 3 18:04:51 2012
@@ -158,7 +158,7 @@ public class TestFsLimits {
Class<?> generated = null;
try {
fs.verifyFsLimits(inodes, 1, child);
- rootInode.addChild(child, false);
+ rootInode.addChild(child, false, null);
} catch (QuotaExceededException e) {
generated = e.getClass();
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java Mon Dec 3 18:04:51 2012
@@ -144,11 +144,11 @@ public class TestINodeFile {
assertEquals("f", inf.getFullPathName());
assertEquals("", inf.getLocalParentDir());
- dir.addChild(inf, false);
+ dir.addChild(inf, false, null);
assertEquals("d"+Path.SEPARATOR+"f", inf.getFullPathName());
assertEquals("d", inf.getLocalParentDir());
- root.addChild(dir, false);
+ root.addChild(dir, false, null);
assertEquals(Path.SEPARATOR+"d"+Path.SEPARATOR+"f", inf.getFullPathName());
assertEquals(Path.SEPARATOR+"d", dir.getFullPathName());
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java Mon Dec 3 18:04:51 2012
@@ -92,7 +92,8 @@ public class TestMetaSave {
DataInputStream in = new DataInputStream(fstream);
BufferedReader reader = new BufferedReader(new InputStreamReader(in));
String line = reader.readLine();
- assertTrue(line.equals("3 files and directories, 2 blocks = 5 total"));
+ assertTrue(line.equals(
+ "3 files and directories, 2 blocks = 5 total filesystem objects"));
line = reader.readLine();
assertTrue(line.equals("Live Datanodes: 1"));
line = reader.readLine();
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java Mon Dec 3 18:04:51 2012
@@ -603,6 +603,24 @@ public class TestSaveNamespace {
}
}
+ @Test
+ public void testSaveNamespaceWithDanglingLease() throws Exception {
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(new Configuration())
+ .numDataNodes(1).build();
+ cluster.waitActive();
+ DistributedFileSystem fs = (DistributedFileSystem) cluster.getFileSystem();
+ try {
+ cluster.getNamesystem().leaseManager.addLease("me", "/non-existent");
+ fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+ cluster.getNameNodeRpc().saveNamespace();
+ fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
private void doAnEdit(FSNamesystem fsn, int id) throws IOException {
// Make an edit
fsn.mkdirs(
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java Mon Dec 3 18:04:51 2012
@@ -430,10 +430,10 @@ public class TestSnapshotPathINodes {
// The number of inodes should be equal to components.length
assertEquals(newInodes.length, components.length);
// The last INode should be associated with file1
- assertEquals(newInodes[components.length - 1].getFullPathName(),
- file1.toString());
+ final int last = components.length - 1;
+ assertEquals(newInodes[last].getFullPathName(), file1.toString());
// The modification time of the INode for file3 should have been changed
- Assert.assertFalse(inodes[components.length - 1].getModificationTime() ==
- newInodes[components.length - 1].getModificationTime());
+ Assert.assertFalse(inodes[last].getModificationTime()
+ == newInodes[last].getModificationTime());
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeDirectoryWithSnapshot.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeDirectoryWithSnapshot.java?rev=1416603&r1=1416602&r2=1416603&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeDirectoryWithSnapshot.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeDirectoryWithSnapshot.java Mon Dec 3 18:04:51 2012
@@ -221,7 +221,7 @@ public class TestINodeDirectoryWithSnaps
Assert.assertTrue(i >= 0);
final INodeDirectory oldinode = (INodeDirectory)current.get(i);
final INodeDirectory newinode = new INodeDirectory(oldinode);
- newinode.setModificationTime(oldinode.getModificationTime() + 1);
+ newinode.updateModificationTime(oldinode.getModificationTime() + 1);
current.set(i, newinode);
if (diff != null) {