You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2014/08/27 19:37:28 UTC
[28/29] git commit: HDFS-6908. Incorrect snapshot directory diff
generated by snapshot deletion. Contributed by Juan Yu and Jing Zhao.
HDFS-6908. Incorrect snapshot directory diff generated by snapshot deletion. Contributed by Juan Yu and Jing Zhao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6b441d22
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6b441d22
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6b441d22
Branch: refs/heads/HDFS-6584
Commit: 6b441d227a8806e87224106a81361bd61f0b3d0b
Parents: e2d0ff3
Author: Jing Zhao <ji...@apache.org>
Authored: Wed Aug 27 10:26:22 2014 -0700
Committer: Jing Zhao <ji...@apache.org>
Committed: Wed Aug 27 10:26:22 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../snapshot/DirectoryWithSnapshotFeature.java | 10 ++-
.../namenode/snapshot/TestSnapshotDeletion.java | 77 +++++++++++++++++++-
3 files changed, 85 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b441d22/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index fb3906a..63c434d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -642,6 +642,9 @@ Release 2.6.0 - UNRELEASED
HDFS-4852. libhdfs documentation is out of date. (cnauroth)
+ HDFS-6908. Incorrect snapshot directory diff generated by snapshot deletion.
+ (Juan Yu and jing9 via jing9)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b441d22/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
index 9893bba..9c9d435 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
@@ -722,6 +722,8 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
counts.add(lastDiff.diff.destroyCreatedList(currentINode,
collectedBlocks, removedINodes));
}
+ counts.add(currentINode.cleanSubtreeRecursively(snapshot, prior,
+ collectedBlocks, removedINodes, priorDeleted, countDiffChange));
} else {
// update prior
prior = getDiffs().updatePrior(snapshot, prior);
@@ -739,7 +741,9 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
counts.add(getDiffs().deleteSnapshotDiff(snapshot, prior,
currentINode, collectedBlocks, removedINodes, countDiffChange));
-
+ counts.add(currentINode.cleanSubtreeRecursively(snapshot, prior,
+ collectedBlocks, removedINodes, priorDeleted, countDiffChange));
+
// check priorDiff again since it may be created during the diff deletion
if (prior != Snapshot.NO_SNAPSHOT_ID) {
DirectoryDiff priorDiff = this.getDiffs().getDiffById(prior);
@@ -778,9 +782,7 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
}
}
}
- counts.add(currentINode.cleanSubtreeRecursively(snapshot, prior,
- collectedBlocks, removedINodes, priorDeleted, countDiffChange));
-
+
if (currentINode.isQuotaSet()) {
currentINode.getDirectoryWithQuotaFeature().addSpaceConsumed2Cache(
-counts.get(Quota.NAMESPACE), -counts.get(Quota.DISKSPACE));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b441d22/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
index 77fa2a2..1450a7d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.namenode.snapshot;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -558,7 +559,81 @@ public class TestSnapshotDeletion {
+ toDeleteFileInSnapshot.toString(), e);
}
}
-
+
+ /**
+ * Delete a snapshot that is taken before a directory deletion,
+ * directory diff list should be combined correctly.
+ */
+ @Test (timeout=60000)
+ public void testDeleteSnapshot1() throws Exception {
+ final Path root = new Path("/");
+
+ Path dir = new Path("/dir1");
+ Path file1 = new Path(dir, "file1");
+ DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, REPLICATION, seed);
+
+ hdfs.allowSnapshot(root);
+ hdfs.createSnapshot(root, "s1");
+
+ Path file2 = new Path(dir, "file2");
+ DFSTestUtil.createFile(hdfs, file2, BLOCKSIZE, REPLICATION, seed);
+
+ hdfs.createSnapshot(root, "s2");
+
+ // delete file
+ hdfs.delete(file1, true);
+ hdfs.delete(file2, true);
+
+ // delete directory
+ assertTrue(hdfs.delete(dir, false));
+
+ // delete second snapshot
+ hdfs.deleteSnapshot(root, "s2");
+
+ NameNodeAdapter.enterSafeMode(cluster.getNameNode(), false);
+ NameNodeAdapter.saveNamespace(cluster.getNameNode());
+
+ // restart NN
+ cluster.restartNameNodes();
+ }
+
+ /**
+ * Delete a snapshot that is taken before a directory deletion (recursively),
+ * directory diff list should be combined correctly.
+ */
+ @Test (timeout=60000)
+ public void testDeleteSnapshot2() throws Exception {
+ final Path root = new Path("/");
+
+ Path dir = new Path("/dir1");
+ Path file1 = new Path(dir, "file1");
+ DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, REPLICATION, seed);
+
+ hdfs.allowSnapshot(root);
+ hdfs.createSnapshot(root, "s1");
+
+ Path file2 = new Path(dir, "file2");
+ DFSTestUtil.createFile(hdfs, file2, BLOCKSIZE, REPLICATION, seed);
+ INodeFile file2Node = fsdir.getINode(file2.toString()).asFile();
+ long file2NodeId = file2Node.getId();
+
+ hdfs.createSnapshot(root, "s2");
+
+ // delete directory recursively
+ assertTrue(hdfs.delete(dir, true));
+ assertNotNull(fsdir.getInode(file2NodeId));
+
+ // delete second snapshot
+ hdfs.deleteSnapshot(root, "s2");
+ assertTrue(fsdir.getInode(file2NodeId) == null);
+
+ NameNodeAdapter.enterSafeMode(cluster.getNameNode(), false);
+ NameNodeAdapter.saveNamespace(cluster.getNameNode());
+
+ // restart NN
+ cluster.restartNameNodes();
+ }
+
/**
* Test deleting snapshots in a more complicated scenario: need to combine
* snapshot diffs, but no need to handle diffs distributed in a dir tree