You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/05/12 22:49:21 UTC
[04/36] hadoop git commit: HDFS-6757. Simplify lease manager with
INodeID. Contributed by Haohui Mai.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12fdc447/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java
index 184cd87..255a607 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java
@@ -213,11 +213,12 @@ public class TestDiskspaceQuotaUpdate {
// ignore
}
+ LeaseManager lm = cluster.getNamesystem().getLeaseManager();
// check that the file exists, isn't UC, and has no dangling lease
INodeFile inode = fsdir.getINode(file.toString()).asFile();
Assert.assertNotNull(inode);
Assert.assertFalse("should not be UC", inode.isUnderConstruction());
- Assert.assertNull("should not have a lease", cluster.getNamesystem().getLeaseManager().getLeaseByPath(file.toString()));
+ Assert.assertNull("should not have a lease", lm.getLease(inode));
// make sure the quota usage is unchanged
final long newSpaceUsed = dirNode.getDirectoryWithQuotaFeature()
.getSpaceConsumed().getStorageSpace();
@@ -256,11 +257,11 @@ public class TestDiskspaceQuotaUpdate {
}
// check that the file exists, isn't UC, and has no dangling lease
+ LeaseManager lm = cluster.getNamesystem().getLeaseManager();
INodeFile inode = fsdir.getINode(file.toString()).asFile();
Assert.assertNotNull(inode);
Assert.assertFalse("should not be UC", inode.isUnderConstruction());
- Assert.assertNull("should not have a lease", cluster.getNamesystem()
- .getLeaseManager().getLeaseByPath(file.toString()));
+ Assert.assertNull("should not have a lease", lm.getLease(inode));
// make sure the quota usage is unchanged
final long newSpaceUsed = dirNode.getDirectoryWithQuotaFeature()
.getSpaceConsumed().getStorageSpace();
@@ -296,11 +297,11 @@ public class TestDiskspaceQuotaUpdate {
}
// check that the file exists, isn't UC, and has no dangling lease
+ LeaseManager lm = cluster.getNamesystem().getLeaseManager();
INodeFile inode = fsdir.getINode(file.toString()).asFile();
Assert.assertNotNull(inode);
Assert.assertFalse("should not be UC", inode.isUnderConstruction());
- Assert.assertNull("should not have a lease", cluster.getNamesystem()
- .getLeaseManager().getLeaseByPath(file.toString()));
+ Assert.assertNull("should not have a lease", lm.getLease(inode));
// make sure the quota usage is unchanged
final long newSpaceUsed = dirNode.getDirectoryWithQuotaFeature()
.getSpaceConsumed().getStorageSpace();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12fdc447/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
index 7b9ea93..5653df5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
@@ -109,7 +109,7 @@ public class TestFSImage {
assertEquals(1, blks.length);
assertEquals(BlockUCState.UNDER_CONSTRUCTION, blks[0].getBlockUCState());
// check lease manager
- Lease lease = fsn.leaseManager.getLeaseByPath(file2.toString());
+ Lease lease = fsn.leaseManager.getLease(file2Node);
Assert.assertNotNull(lease);
} finally {
if (cluster != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12fdc447/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
index 3d0259e..8b0662c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
@@ -76,7 +76,7 @@ public class TestFSNamesystem {
DFSTestUtil.formatNameNode(conf);
FSNamesystem fsn = FSNamesystem.loadFromDisk(conf);
LeaseManager leaseMan = fsn.getLeaseManager();
- leaseMan.addLease("client1", "importantFile");
+ leaseMan.addLease("client1", fsn.getFSDirectory().allocateNewInodeId());
assertEquals(1, leaseMan.countLease());
fsn.clear();
leaseMan = fsn.getLeaseManager();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12fdc447/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java
index a19eb1d..4d0f994 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java
@@ -70,7 +70,8 @@ public class TestGetBlockLocations {
public Void answer(InvocationOnMock invocation) throws Throwable {
INodesInPath iip = fsd.getINodesInPath(FILE_PATH, true);
FSDirDeleteOp.delete(fsd, iip, new INode.BlocksMapUpdateInfo(),
- new ArrayList<INode>(), now());
+ new ArrayList<INode>(), new ArrayList<Long>(),
+ now());
invocation.callRealMethod();
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12fdc447/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
index 2f114a7..96907f8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
@@ -21,39 +21,31 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
+import com.google.common.collect.Lists;
import org.junit.Test;
import org.mockito.Mockito;
+import java.util.ArrayList;
+
+import static org.mockito.Mockito.*;
public class TestLeaseManager {
- final Configuration conf = new HdfsConfiguration();
-
@Test
- public void testRemoveLeaseWithPrefixPath() throws Exception {
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
- cluster.waitActive();
-
- LeaseManager lm = NameNodeAdapter.getLeaseManager(cluster.getNamesystem());
- lm.addLease("holder1", "/a/b");
- lm.addLease("holder2", "/a/c");
- assertNotNull(lm.getLeaseByPath("/a/b"));
- assertNotNull(lm.getLeaseByPath("/a/c"));
-
- lm.removeLeaseWithPrefixPath("/a");
-
- assertNull(lm.getLeaseByPath("/a/b"));
- assertNull(lm.getLeaseByPath("/a/c"));
-
- lm.addLease("holder1", "/a/b");
- lm.addLease("holder2", "/a/c");
-
- lm.removeLeaseWithPrefixPath("/a/");
+ public void testRemoveLeases() throws Exception {
+ FSNamesystem fsn = mock(FSNamesystem.class);
+ LeaseManager lm = new LeaseManager(fsn);
+ ArrayList<Long> ids = Lists.newArrayList(INodeId.ROOT_INODE_ID + 1,
+ INodeId.ROOT_INODE_ID + 2, INodeId.ROOT_INODE_ID + 3,
+ INodeId.ROOT_INODE_ID + 4);
+ for (long id : ids) {
+ lm.addLease("foo", id);
+ }
- assertNull(lm.getLeaseByPath("/a/b"));
- assertNull(lm.getLeaseByPath("/a/c"));
+ assertEquals(4, lm.getINodeIdWithLeases().size());
+ synchronized (lm) {
+ lm.removeLeases(ids);
+ }
+ assertEquals(0, lm.getINodeIdWithLeases().size());
}
/** Check that even if LeaseManager.checkLease is not able to relinquish
@@ -70,13 +62,13 @@ public class TestLeaseManager {
LeaseManager lm = new LeaseManager(fsn);
//Make sure the leases we are going to add exceed the hard limit
- lm.setLeasePeriod(0,0);
+ lm.setLeasePeriod(0, 0);
//Add some leases to the LeaseManager
- lm.addLease("holder1", "src1");
- lm.addLease("holder2", "src2");
- lm.addLease("holder3", "src3");
- assertEquals(lm.getNumSortedLeases(), 3);
+ lm.addLease("holder1", INodeId.ROOT_INODE_ID + 1);
+ lm.addLease("holder2", INodeId.ROOT_INODE_ID + 2);
+ lm.addLease("holder3", INodeId.ROOT_INODE_ID + 3);
+ assertEquals(lm.countLease(), 3);
//Initiate a call to checkLease. This should exit within the test timeout
lm.checkLeases();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12fdc447/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
index f43edfb..5be1cef 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
@@ -615,7 +615,8 @@ public class TestSaveNamespace {
cluster.waitActive();
DistributedFileSystem fs = cluster.getFileSystem();
try {
- cluster.getNamesystem().leaseManager.addLease("me", "/non-existent");
+ cluster.getNamesystem().leaseManager.addLease("me",
+ INodeId.ROOT_INODE_ID + 1);
fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
cluster.getNameNodeRpc().saveNamespace(0, 0);
fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12fdc447/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeFileUnderConstructionWithSnapshot.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeFileUnderConstructionWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeFileUnderConstructionWithSnapshot.java
index aba3bb3..e716d6d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeFileUnderConstructionWithSnapshot.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeFileUnderConstructionWithSnapshot.java
@@ -40,11 +40,13 @@ import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream.SyncFlag;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
+import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiff;
import org.apache.log4j.Level;
import org.junit.After;
@@ -279,4 +281,31 @@ public class TestINodeFileUnderConstructionWithSnapshot {
assertEquals(BLOCKSIZE - 1, lastBlock.getBlockSize());
out.close();
}
+
+ @Test
+ public void testLease() throws Exception {
+ try {
+ NameNodeAdapter.setLeasePeriod(fsn, 100, 200);
+ final Path foo = new Path(dir, "foo");
+ final Path bar = new Path(foo, "bar");
+ DFSTestUtil.createFile(hdfs, bar, BLOCKSIZE, REPLICATION, 0);
+ HdfsDataOutputStream out = appendFileWithoutClosing(bar, 100);
+ out.hsync(EnumSet.of(SyncFlag.UPDATE_LENGTH));
+ SnapshotTestHelper.createSnapshot(hdfs, dir, "s0");
+
+ hdfs.delete(foo, true);
+ Thread.sleep(1000);
+ try {
+ fsn.writeLock();
+ NameNodeAdapter.getLeaseManager(fsn).runLeaseChecks();
+ } finally {
+ fsn.writeUnlock();
+ }
+ } finally {
+ NameNodeAdapter.setLeasePeriod(
+ fsn,
+ HdfsServerConstants.LEASE_SOFTLIMIT_PERIOD,
+ HdfsServerConstants.LEASE_HARDLIMIT_PERIOD);
+ }
+ }
}
\ No newline at end of file