You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by we...@apache.org on 2017/04/03 14:35:34 UTC
hadoop git commit: HDFS-11515. -du throws
ConcurrentModificationException. Contributed by Istvan Fajth, Wei-Chiu Chuang.
Repository: hadoop
Updated Branches:
refs/heads/trunk 845529b3a -> bc7aff7ce
HDFS-11515. -du throws ConcurrentModificationException. Contributed by Istvan Fajth, Wei-Chiu Chuang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bc7aff7c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bc7aff7c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bc7aff7c
Branch: refs/heads/trunk
Commit: bc7aff7cec07bbc3fed63a00c8f1584c34670998
Parents: 845529b
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Mon Apr 3 07:32:27 2017 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Mon Apr 3 07:35:09 2017 -0700
----------------------------------------------------------------------
.../snapshot/DirectoryWithSnapshotFeature.java | 5 ++
.../snapshot/TestRenameWithSnapshots.java | 6 +-
.../namenode/snapshot/TestSnapshotDeletion.java | 75 ++++++++++++++++++++
3 files changed, 84 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc7aff7c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
index 9addbfa..9840679 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
@@ -633,6 +633,11 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
for(DirectoryDiff d : diffs) {
for(INode deletedNode : d.getChildrenDiff().getList(ListType.DELETED)) {
context.reportDeletedSnapshottedNode(deletedNode);
+ if (deletedNode.isDirectory()){
+ DirectoryWithSnapshotFeature sf =
+ deletedNode.asDirectory().getDirectoryWithSnapshotFeature();
+ sf.computeContentSummary4Snapshot(context);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc7aff7c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
index d1b3aa6..d06c384 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
@@ -26,6 +26,7 @@ import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.anyObject;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.spy;
+import static org.apache.hadoop.test.GenericTestUtils.getTestDir;
import java.io.File;
import java.io.IOException;
@@ -2429,7 +2430,7 @@ public class TestRenameWithSnapshots {
*/
@Test (timeout=300000)
public void testDu() throws Exception {
- File tempFile = File.createTempFile("testDu-", ".tmp");
+ File tempFile = File.createTempFile("testDu-", ".tmp", getTestDir());
tempFile.deleteOnExit();
final FileSystem localfs = FileSystem.getLocal(conf);
@@ -2539,7 +2540,8 @@ public class TestRenameWithSnapshots {
*/
@Test (timeout=300000)
public void testDuMultipleDirs() throws Exception {
- File tempFile = File.createTempFile("testDuMultipleDirs-", "" + ".tmp");
+ File tempFile = File.createTempFile("testDuMultipleDirs-", ".tmp",
+ getTestDir());
tempFile.deleteOnExit();
final FileSystem localfs = FileSystem.getLocal(conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc7aff7c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
index ca53788..7926e44 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
@@ -27,6 +27,7 @@ import java.io.PrintStream;
import java.security.PrivilegedAction;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FsShell;
import org.apache.hadoop.fs.Path;
@@ -1232,4 +1233,78 @@ public class TestSnapshotDeletion {
// make sure bar has been cleaned from inodeMap
Assert.assertNull(fsdir.getInode(fileId));
}
+
+ /**
+ * Test for HDFS-11515.
+ * In a scenario where a directory with subdirectories is removed from the
+ * file system after taking a snapshot on one of its ancestors, du command
+ * fails with a ConcurrentModificationException until a new snapshot is taken,
+ * or the old snapshots are removed.
+ * This test is testing this scenario with checks on the space consumed
+ * calculation.
+ *
+ * @throws Exception
+ */
+ @Test(timeout = 180000)
+ public void testDuWithRmdirInSnapshots() throws Exception {
+ final Path parent = new Path("/testDuWithRmdirInSnapshots");
+ final Path snapshotDir = new Path(parent, "snapshotDir");
+ final Path dir1 = new Path(snapshotDir, "d1"); //snapshotDir/d1
+ final Path dir2 = new Path(snapshotDir, "d2"); //snapshotDir/d2
+ final Path dir4 = new Path(dir2, "d4"); //snapshotDir/d2/d4
+ final Path dir3 = new Path(snapshotDir, "d3"); //snapshotDir/d3
+ final Path dir5 = new Path(dir3, "d5"); //snapshotDir/d3/d5
+ final Path aFileOutsideSnapshots = new Path(parent, "aFile");
+ final Path aFileInsideSnapshots = new Path(dir5, "aFile");
+
+ final String snapshotName = "s1";
+ final String snapshotName2 = "s2";
+
+ final long spaceConsumed = BLOCKSIZE * REPLICATION;
+ final long spaceConsumed2 = 2 * spaceConsumed;
+ ContentSummary summary = null;
+
+ DFSTestUtil.createFile(hdfs, aFileOutsideSnapshots,
+ BLOCKSIZE, REPLICATION, 0);
+ summary = hdfs.getContentSummary(parent);
+ assertEquals("Du is wrong even with one file without further ado.",
+ spaceConsumed, summary.getSpaceConsumed());
+
+ hdfs.mkdirs(snapshotDir);
+ hdfs.allowSnapshot(snapshotDir);
+ hdfs.mkdirs(dir1);
+
+ hdfs.createSnapshot(snapshotDir, snapshotName);
+
+ hdfs.mkdirs(dir4);
+ hdfs.mkdirs(dir5);
+ DFSTestUtil.createFile(hdfs, aFileInsideSnapshots,
+ BLOCKSIZE, REPLICATION, 0);
+ summary = hdfs.getContentSummary(parent);
+ assertEquals("Du is wrong with 2 files added to the file system.",
+ spaceConsumed2, summary.getSpaceConsumed());
+
+ hdfs.createSnapshot(snapshotDir, snapshotName2);
+
+ hdfs.delete(dir2, true);
+ hdfs.delete(dir3, true);
+
+ summary = hdfs.getContentSummary(parent);
+ assertEquals("Snapshot file count is not matching expected value.",
+ 1, summary.getSnapshotFileCount());
+ assertEquals("Snapshot directory count is not matching expected value.",
+ 4, summary.getSnapshotDirectoryCount());
+ assertEquals("Consumed space does not matching expected value.",
+ spaceConsumed, summary.getSnapshotSpaceConsumed());
+ assertEquals("Snapshot length is not matching expected value.",
+ BLOCKSIZE, summary.getSnapshotLength());
+ assertEquals("File count is not matching expected value.",
+ 2, summary.getFileCount());
+ assertEquals("Directory count is not matching expected value.",
+ 7, summary.getDirectoryCount());
+ assertEquals("Consumed space is not matching expected value.",
+ spaceConsumed2, summary.getSpaceConsumed());
+ assertEquals("Length is not matching expected value.",
+ 2 * BLOCKSIZE, summary.getLength());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org