You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by we...@apache.org on 2019/08/17 10:19:02 UTC
[hadoop] branch branch-3.1 updated: HDFS-13101. Yet another fsimage
corruption related to snapshot. Contributed by Shashikant Banerjee.
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new d18b720 HDFS-13101. Yet another fsimage corruption related to snapshot. Contributed by Shashikant Banerjee.
d18b720 is described below
commit d18b720c7050f2ba3a6a7a46a994e1203f476577
Author: Shashikant Banerjee <sh...@apache.org>
AuthorDate: Thu Aug 15 10:16:25 2019 +0530
HDFS-13101. Yet another fsimage corruption related to snapshot. Contributed by Shashikant Banerjee.
(cherry picked from commit 0a85af959ce505f0659e5c69d0ca83a5dce0a7c2)
(cherry picked from commit e89413da882e155d8654d5265e180409ccb31e34)
---
.../apache/hadoop/hdfs/server/namenode/INode.java | 13 +++
.../hdfs/server/namenode/INodeDirectory.java | 8 ++
.../hadoop/hdfs/server/namenode/INodeFile.java | 12 +++
.../namenode/snapshot/AbstractINodeDiffList.java | 15 +++-
.../snapshot/DirectoryWithSnapshotFeature.java | 21 +++--
.../namenode/snapshot/FileWithSnapshotFeature.java | 5 ++
.../server/namenode/TestFSImageWithSnapshot.java | 100 ++++++++++++++++++++-
.../namenode/snapshot/SnapshotTestHelper.java | 3 +-
8 files changed, 168 insertions(+), 9 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
index 207d977..9bef64a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockUnderConstructionFeature;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.server.namenode.INodeReference.DstReference;
+import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithCount;
import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithName;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.util.Diff;
@@ -646,6 +647,18 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
return parent == null || !parent.isReference()? null: (INodeReference)parent;
}
+ /**
+ * @return true if this is a reference and the reference count is 1;
+ * otherwise, return false.
+ */
+ public boolean isLastReference() {
+ final INodeReference ref = getParentReference();
+ if (!(ref instanceof WithCount)) {
+ return false;
+ }
+ return ((WithCount)ref).getReferenceCount() == 1;
+ }
+
/** Set parent directory */
public final void setParent(INodeDirectory parent) {
this.parent = parent;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
index 433abcb..8fa9bcf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
@@ -887,6 +887,14 @@ public class INodeDirectory extends INodeWithAdditionalFields
prefix.setLength(prefix.length() - 2);
prefix.append(" ");
}
+
+ final DirectoryWithSnapshotFeature snapshotFeature =
+ getDirectoryWithSnapshotFeature();
+ if (snapshotFeature != null) {
+ out.print(prefix);
+ out.print(snapshotFeature);
+ }
+ out.println();
dumpTreeRecursively(out, prefix, new Iterable<SnapshotAndINode>() {
final Iterator<INode> i = getChildrenList(snapshot).iterator();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
index 6693297..7b6f1e3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
@@ -1046,6 +1046,18 @@ public class INodeFile extends INodeWithAdditionalFields
out.print(", blocks=");
out.print(blocks.length == 0 ? null: blocks[0]);
out.println();
+
+ final FileWithSnapshotFeature snapshotFeature =
+ getFileWithSnapshotFeature();
+ if (snapshotFeature != null) {
+ if (prefix.length() >= 2) {
+ prefix.setLength(prefix.length() - 2);
+ prefix.append(" ");
+ }
+ out.print(prefix);
+ out.print(snapshotFeature);
+ }
+ out.println();
}
/**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
index d115656..4a00d20 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
@@ -314,6 +314,19 @@ abstract class AbstractINodeDiffList<N extends INode,
@Override
public String toString() {
- return getClass().getSimpleName() + ": " + (diffs != null ? diffs : "[]");
+ if (diffs != null) {
+ final StringBuilder b =
+ new StringBuilder(getClass().getSimpleName()).append("@")
+ .append(Integer.toHexString(hashCode())).append(": ");
+ b.append("[");
+ for (D d : diffs) {
+ b.append(d).append(", ");
+ }
+ b.setLength(b.length() - 2);
+ b.append("]");
+ return b.toString();
+ } else {
+ return "";
+ }
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
index 18c0bff..7fb639c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
@@ -739,14 +739,20 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
// were created before "prior" will be covered by the later
// cleanSubtreeRecursively call.
if (priorCreated != null) {
- // we only check the node originally in prior's created list
- for (INode cNode : priorDiff.diff.getCreatedUnmodifiable()) {
- if (priorCreated.containsKey(cNode)) {
- cNode.cleanSubtree(reclaimContext, snapshot, NO_SNAPSHOT_ID);
+ if (currentINode.isLastReference()) {
+ // if this is the last reference, the created list can be
+ // destroyed.
+ priorDiff.getChildrenDiff().destroyCreatedList(
+ reclaimContext, currentINode);
+ } else {
+ // we only check the node originally in prior's created list
+ for (INode cNode : priorDiff.diff.getCreatedUnmodifiable()) {
+ if (priorCreated.containsKey(cNode)) {
+ cNode.cleanSubtree(reclaimContext, snapshot, NO_SNAPSHOT_ID);
+ }
}
}
}
-
// When a directory is moved from the deleted list of the posterior
// diff to the deleted list of this diff, we need to destroy its
// descendants that were 1) created after taking this diff and 2)
@@ -770,4 +776,9 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
reclaimContext.quotaDelta().addQuotaDirUpdate(currentINode, current);
}
}
+
+ @Override
+ public String toString() {
+ return "" + diffs;
+ }
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
index 80061c3..cf69962 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
@@ -225,4 +225,9 @@ public class FileWithSnapshotFeature implements INode.Feature {
file.collectBlocksBeyondSnapshot(snapshotBlocks,
reclaimContext.collectedBlocks());
}
+
+ @Override
+ public String toString() {
+ return "" + diffs;
+ }
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java
index 58ecc8a..243e16c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.IOException;
+import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
@@ -64,7 +65,7 @@ public class TestFSImageWithSnapshot {
}
static final long seed = 0;
- static final short NUM_DATANODES = 3;
+ static final short NUM_DATANODES = 1;
static final int BLOCKSIZE = 1024;
static final long txid = 1;
@@ -511,4 +512,101 @@ public class TestFSImageWithSnapshot {
fsn = cluster.getNamesystem();
hdfs = cluster.getFileSystem();
}
+
+ void rename(Path src, Path dst) throws Exception {
+ printTree("Before rename " + src + " -> " + dst);
+ hdfs.rename(src, dst);
+ printTree("After rename " + src + " -> " + dst);
+ }
+
+ void createFile(Path directory, String filename) throws Exception {
+ final Path f = new Path(directory, filename);
+ DFSTestUtil.createFile(hdfs, f, 0, NUM_DATANODES, seed);
+ }
+
+ void appendFile(Path directory, String filename) throws Exception {
+ final Path f = new Path(directory, filename);
+ DFSTestUtil.appendFile(hdfs, f, "more data");
+ printTree("appended " + f);
+ }
+
+ void deleteSnapshot(Path directory, String snapshotName) throws Exception {
+ hdfs.deleteSnapshot(directory, snapshotName);
+ printTree("deleted snapshot " + snapshotName);
+ }
+
+ @Test (timeout=60000)
+ public void testDoubleRename() throws Exception {
+ final Path parent = new Path("/parent");
+ hdfs.mkdirs(parent);
+ final Path sub1 = new Path(parent, "sub1");
+ final Path sub1foo = new Path(sub1, "foo");
+ hdfs.mkdirs(sub1);
+ hdfs.mkdirs(sub1foo);
+ createFile(sub1foo, "file0");
+
+ printTree("before s0");
+ hdfs.allowSnapshot(parent);
+ hdfs.createSnapshot(parent, "s0");
+
+ createFile(sub1foo, "file1");
+ createFile(sub1foo, "file2");
+
+ final Path sub2 = new Path(parent, "sub2");
+ hdfs.mkdirs(sub2);
+ final Path sub2foo = new Path(sub2, "foo");
+ // mv /parent/sub1/foo to /parent/sub2/foo
+ rename(sub1foo, sub2foo);
+
+ hdfs.createSnapshot(parent, "s1");
+ hdfs.createSnapshot(parent, "s2");
+ printTree("created snapshots: s1, s2");
+
+ appendFile(sub2foo, "file1");
+ createFile(sub2foo, "file3");
+
+ final Path sub3 = new Path(parent, "sub3");
+ hdfs.mkdirs(sub3);
+ // mv /parent/sub2/foo to /parent/sub3/foo
+ rename(sub2foo, sub3);
+
+ hdfs.delete(sub3, true);
+ printTree("deleted " + sub3);
+
+ deleteSnapshot(parent, "s1");
+ restartCluster();
+
+ deleteSnapshot(parent, "s2");
+ restartCluster();
+ }
+
+ void restartCluster() throws Exception {
+ final File before = dumpTree2File("before.txt");
+
+ hdfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+ hdfs.saveNamespace();
+ hdfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
+
+ cluster.shutdown();
+ cluster = new MiniDFSCluster.Builder(conf).format(false)
+ .numDataNodes(NUM_DATANODES).build();
+ cluster.waitActive();
+ fsn = cluster.getNamesystem();
+ hdfs = cluster.getFileSystem();
+ final File after = dumpTree2File("after.txt");
+ SnapshotTestHelper.compareDumpedTreeInFile(before, after, true);
+ }
+
+ private final PrintWriter output = new PrintWriter(System.out, true);
+ private int printTreeCount = 0;
+
+ String printTree(String label) throws Exception {
+ output.println();
+ output.println();
+ output.println("***** " + printTreeCount++ + ": " + label);
+ final String b =
+ fsn.getFSDirectory().getINode("/").dumpTreeRecursively().toString();
+ output.println(b);
+ return b;
+ }
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java
index 28cb757..d13cc38 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java
@@ -245,8 +245,7 @@ public class SnapshotTestHelper {
"\\{blockUCState=\\w+, primaryNodeIndex=[-\\d]+, replicas=\\[\\]\\}",
"");
}
-
- assertEquals(line1, line2);
+ assertEquals(line1.trim(), line2.trim());
}
Assert.assertNull(reader1.readLine());
Assert.assertNull(reader2.readLine());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org