You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by su...@apache.org on 2009/11/12 01:29:35 UTC
svn commit: r835169 - in /hadoop/hdfs/branches/branch-0.21: CHANGES.txt
src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
src/test/hdfs/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java
Author: suresh
Date: Thu Nov 12 00:29:32 2009
New Revision: 835169
URL: http://svn.apache.org/viewvc?rev=835169&view=rev
Log:
HDFS-761. Fix failure to process rename operation from edits log due to quota verification. Contributed by Suresh Srinivas.
Modified:
hadoop/hdfs/branches/branch-0.21/CHANGES.txt
hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
hadoop/hdfs/branches/branch-0.21/src/test/hdfs/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java
Modified: hadoop/hdfs/branches/branch-0.21/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.21/CHANGES.txt?rev=835169&r1=835168&r2=835169&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.21/CHANGES.txt (original)
+++ hadoop/hdfs/branches/branch-0.21/CHANGES.txt Thu Nov 12 00:29:32 2009
@@ -487,3 +487,6 @@
HDFS-525. The SimpleDateFormat object in ListPathsServlet is not thread
safe. (Suresh Srinivas and cdouglas)
+
+ HDFS-761. Fix failure to process rename operation from edits log due to
+ quota verification. (suresh)
Modified: hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=835169&r1=835168&r2=835169&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java (original)
+++ hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java Thu Nov 12 00:29:32 2009
@@ -1084,6 +1084,10 @@
*/
private void verifyQuota(INode[] inodes, int pos, long nsDelta, long dsDelta,
INode commonAncestor) throws QuotaExceededException {
+ if (!ready) {
+ // Do not check quota if edits log is still being processed
+ return;
+ }
if (pos>inodes.length) {
pos = inodes.length;
}
@@ -1117,6 +1121,10 @@
*/
private void verifyQuotaForRename(INode[] srcInodes, INode[]dstInodes)
throws QuotaExceededException {
+ if (!ready) {
+ // Do not check quota if edits log is still being processed
+ return;
+ }
INode srcInode = srcInodes[srcInodes.length - 1];
INode commonAncestor = null;
for(int i =0;srcInodes[i] == dstInodes[i]; i++) {
Modified: hadoop/hdfs/branches/branch-0.21/src/test/hdfs/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.21/src/test/hdfs/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java?rev=835169&r1=835168&r2=835169&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.21/src/test/hdfs/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java (original)
+++ hadoop/hdfs/branches/branch-0.21/src/test/hdfs/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java Thu Nov 12 00:29:32 2009
@@ -19,6 +19,7 @@
package org.apache.hadoop.fs;
import java.io.IOException;
+import java.net.URISyntaxException;
import javax.security.auth.login.LoginException;
@@ -26,7 +27,6 @@
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.FSConstants;
-import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.security.UnixUserGroupInformation;
import org.junit.After;
import org.junit.AfterClass;
@@ -39,17 +39,31 @@
FileContextMainOperationsBaseTest {
private static MiniDFSCluster cluster;
private static Path defaultWorkingDirectory;
+ private static HdfsConfiguration CONF = new HdfsConfiguration();
@BeforeClass
- public static void clusterSetupAtBegining()
- throws IOException, LoginException {
- cluster = new MiniDFSCluster(new HdfsConfiguration(), 2, true, null);
+ public static void clusterSetupAtBegining() throws IOException,
+ LoginException, URISyntaxException {
+ cluster = new MiniDFSCluster(CONF, 2, true, null);
+ cluster.waitClusterUp();
fc = FileContext.getFileContext(cluster.getFileSystem());
defaultWorkingDirectory = fc.makeQualified( new Path("/user/" +
UnixUserGroupInformation.login().getUserName()));
fc.mkdir(defaultWorkingDirectory, FileContext.DEFAULT_PERM, true);
}
+ private static void restartCluster() throws IOException, LoginException {
+ if (cluster != null) {
+ cluster.shutdown();
+ cluster = null;
+ }
+ cluster = new MiniDFSCluster(CONF, 1, false, null);
+ cluster.waitClusterUp();
+ fc = FileContext.getFileContext(cluster.getFileSystem());
+ defaultWorkingDirectory = fc.makeQualified( new Path("/user/" +
+ UnixUserGroupInformation.login().getUserName()));
+ fc.mkdir(defaultWorkingDirectory, FileContext.DEFAULT_PERM, true);
+ }
@AfterClass
public static void ClusterShutdownAtEnd() throws Exception {
@@ -79,45 +93,79 @@
}
@Test
- public void testRenameWithQuota() throws Exception {
+ public void testOldRenameWithQuota() throws Exception {
DistributedFileSystem fs = (DistributedFileSystem) cluster.getFileSystem();
- Path src1 = getTestRootPath("testRenameWithQuota/srcdir/src1");
- Path src2 = getTestRootPath("testRenameWithQuota/srcdir/src2");
- Path dst1 = getTestRootPath("testRenameWithQuota/dstdir/dst1");
- Path dst2 = getTestRootPath("testRenameWithQuota/dstdir/dst2");
+ Path src1 = getTestRootPath("test/testOldRenameWithQuota/srcdir/src1");
+ Path src2 = getTestRootPath("test/testOldRenameWithQuota/srcdir/src2");
+ Path dst1 = getTestRootPath("test/testOldRenameWithQuota/dstdir/dst1");
+ Path dst2 = getTestRootPath("test/testOldRenameWithQuota/dstdir/dst2");
createFile(src1);
createFile(src2);
fs.setQuota(src1.getParent(), FSConstants.QUOTA_DONT_SET,
FSConstants.QUOTA_DONT_SET);
fc.mkdir(dst1.getParent(), FileContext.DEFAULT_PERM, true);
- fs.setQuota(dst1.getParent(), FSConstants.QUOTA_DONT_SET,
- FSConstants.QUOTA_DONT_SET);
-
- // Test1: src does not exceed quota and dst has no quota check and hence
- // accommodates rename
- rename(src1, dst1, true, false);
-
- // Test2: src does not exceed quota and dst has *no* quota to accommodate
- // rename.
-
- // testRenameWithQuota/dstDir now has quota = 1 and dst1 already uses it
- fs.setQuota(dst1.getParent(), 1, FSConstants.QUOTA_DONT_SET);
- rename(src2, dst2, false, true);
- // Test3: src exceeds quota and dst has *no* quota to accommodate rename
-
+ fs.setQuota(dst1.getParent(), 2, FSConstants.QUOTA_DONT_SET);
+ /*
+ * Test1: src does not exceed quota and dst has no quota check and hence
+ * accommodates rename
+ */
+ oldRename(src1, dst1, true, false);
+
+ /*
+ * Test2: src does not exceed quota and dst has *no* quota to accommodate
+ * rename.
+ */
+ // dstDir quota = 1 and dst1 already uses it
+ oldRename(src2, dst2, false, true);
+
+ /*
+ * Test3: src exceeds quota and dst has *no* quota to accommodate rename
+ */
// src1 has no quota to accommodate new rename node
fs.setQuota(src1.getParent(), 1, FSConstants.QUOTA_DONT_SET);
- rename(dst1, src1, false, true);
+ oldRename(dst1, src1, false, true);
+
}
- private void rename(Path src, Path dst, boolean renameSucceeds,
- boolean quotaException) throws Exception {
+
+ /**
+ * Perform operations such as setting quota, deletion of files, rename and
+ * ensure system can apply edits log during startup.
+ */
+ @Test
+ public void testEditsLogOldRename() throws Exception {
+ DistributedFileSystem fs = (DistributedFileSystem) cluster.getFileSystem();
+ Path src1 = getTestRootPath("testEditsLogOldRename/srcdir/src1");
+ Path dst1 = getTestRootPath("testEditsLogOldRename/dstdir/dst1");
+ createFile(src1);
+ fs.mkdirs(dst1.getParent());
+ createFile(dst1);
+
+ // Set quota so that dst1 parent cannot allow under it new files/directories
+ fs.setQuota(dst1.getParent(), 2, FSConstants.QUOTA_DONT_SET);
+ // Free up quota for a subsequent rename
+ fs.delete(dst1, true);
+ oldRename(src1, dst1, true, false);
+
+ // Restart the cluster and ensure the above operations can be
+ // loaded from the edits log
+ restartCluster();
+ fs = (DistributedFileSystem)cluster.getFileSystem();
+ src1 = getTestRootPath("testEditsLogOldRename/srcdir/src1");
+ dst1 = getTestRootPath("testEditsLogOldRename/dstdir/dst1");
+ Assert.assertFalse(fs.exists(src1)); // ensure src1 is already renamed
+ Assert.assertTrue(fs.exists(dst1)); // ensure rename dst exists
+ }
+
+
+ private void oldRename(Path src, Path dst, boolean renameSucceeds,
+ boolean exception) throws Exception {
DistributedFileSystem fs = (DistributedFileSystem) cluster.getFileSystem();
try {
Assert.assertEquals(renameSucceeds, fs.rename(src, dst));
- } catch (QuotaExceededException ex) {
- Assert.assertTrue(quotaException);
+ } catch (Exception ex) {
+ Assert.assertTrue(exception);
}
Assert.assertEquals(renameSucceeds, !fc.exists(src));
Assert.assertEquals(renameSucceeds, fc.exists(dst));