You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2020/08/31 06:00:06 UTC
[hadoop] branch trunk updated: HDFS-15542. Add identified snapshot
corruption tests for ordered snapshot deletion (#2251)
This is an automated email from the ASF dual-hosted git repository.
shashikant pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 2d03209 HDFS-15542. Add identified snapshot corruption tests for ordered snapshot deletion (#2251)
2d03209 is described below
commit 2d03209a06df361e4d224795ed4dce8f9414d70f
Author: bshashikant <sh...@apache.org>
AuthorDate: Mon Aug 31 11:29:48 2020 +0530
HDFS-15542. Add identified snapshot corruption tests for ordered snapshot deletion (#2251)
---
.../TestFSImageWithOrderedSnapshotDeletion.java | 495 +++++++++++++++++++++
1 file changed, 495 insertions(+)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFSImageWithOrderedSnapshotDeletion.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFSImageWithOrderedSnapshotDeletion.java
new file mode 100644
index 0000000..dec28e4
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFSImageWithOrderedSnapshotDeletion.java
@@ -0,0 +1,495 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode.snapshot;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.*;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.hdfs.server.namenode.INode;
+import org.apache.hadoop.hdfs.server.namenode.visitor.NamespacePrintVisitor;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.event.Level;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+
+import static org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager.DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test FSImage correctness with ordered snapshot deletion.
+ */
+public class TestFSImageWithOrderedSnapshotDeletion {
+ {
+ SnapshotTestHelper.disableLogs();
+ GenericTestUtils.setLogLevel(INode.LOG, Level.TRACE);
+ }
+
+ static final long SEED = 0;
+ static final short NUM_DATANODES = 1;
+ static final int BLOCKSIZE = 1024;
+
+ private final Path dir = new Path("/TestSnapshot");
+ private static final String TEST_DIR =
+ GenericTestUtils.getTestDir().getAbsolutePath();
+
+ Configuration conf;
+ MiniDFSCluster cluster;
+ FSNamesystem fsn;
+ DistributedFileSystem hdfs;
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new Configuration();
+ conf.setBoolean(DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED, true);
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES)
+ .build();
+ cluster.waitActive();
+ fsn = cluster.getNamesystem();
+ hdfs = cluster.getFileSystem();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (cluster != null) {
+ cluster.shutdown();
+ cluster = null;
+ }
+ }
+
+ void rename(Path src, Path dst) throws Exception {
+ printTree("Before rename " + src + " -> " + dst);
+ hdfs.rename(src, dst);
+ printTree("After rename " + src + " -> " + dst);
+ }
+
+ void createFile(Path directory, String filename) throws Exception {
+ final Path f = new Path(directory, filename);
+ DFSTestUtil.createFile(hdfs, f, 0, NUM_DATANODES, SEED);
+ }
+
+ void appendFile(Path directory, String filename) throws Exception {
+ final Path f = new Path(directory, filename);
+ DFSTestUtil.appendFile(hdfs, f, "more data");
+ printTree("appended " + f);
+ }
+
+ void deleteSnapshot(Path directory, String snapshotName) throws Exception {
+ hdfs.deleteSnapshot(directory, snapshotName);
+ printTree("deleted snapshot " + snapshotName);
+ }
+
+ @Test (timeout=60000)
+ public void testDoubleRename() throws Exception {
+ final Path parent = new Path("/parent");
+ hdfs.mkdirs(parent);
+ final Path sub1 = new Path(parent, "sub1");
+ final Path sub1foo = new Path(sub1, "foo");
+ hdfs.mkdirs(sub1);
+ hdfs.mkdirs(sub1foo);
+ createFile(sub1foo, "file0");
+
+ printTree("before s0");
+ hdfs.allowSnapshot(parent);
+ hdfs.createSnapshot(parent, "s0");
+
+ createFile(sub1foo, "file1");
+ createFile(sub1foo, "file2");
+
+ final Path sub2 = new Path(parent, "sub2");
+ hdfs.mkdirs(sub2);
+ final Path sub2foo = new Path(sub2, "foo");
+ // mv /parent/sub1/foo to /parent/sub2/foo
+ rename(sub1foo, sub2foo);
+
+ hdfs.createSnapshot(parent, "s1");
+ hdfs.createSnapshot(parent, "s2");
+ printTree("created snapshots: s1, s2");
+
+ appendFile(sub2foo, "file1");
+ createFile(sub2foo, "file3");
+
+ final Path sub3 = new Path(parent, "sub3");
+ hdfs.mkdirs(sub3);
+ // mv /parent/sub2/foo to /parent/sub3/foo
+ rename(sub2foo, sub3);
+
+ hdfs.delete(sub3, true);
+ printTree("deleted " + sub3);
+
+ deleteSnapshot(parent, "s1");
+ restartCluster();
+
+ deleteSnapshot(parent, "s2");
+ restartCluster();
+ }
+
+ private File getDumpTreeFile(String directory, String suffix) {
+ return new File(directory, String.format("dumpTree_%s", suffix));
+ }
+ /**
+ * Dump the fsdir tree to a temp file.
+ * @param fileSuffix suffix of the temp file for dumping
+ * @return the temp file
+ */
+ private File dumpTree2File(String fileSuffix) throws IOException {
+ File file = getDumpTreeFile(TEST_DIR, fileSuffix);
+ SnapshotTestHelper.dumpTree2File(fsn.getFSDirectory(), file);
+ return file;
+ }
+
+ void restartCluster() throws Exception {
+ final File before = dumpTree2File("before.txt");
+
+ hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER);
+ hdfs.saveNamespace();
+ hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
+
+ cluster.shutdown();
+ cluster = new MiniDFSCluster.Builder(conf).format(false)
+ .numDataNodes(NUM_DATANODES).build();
+ cluster.waitActive();
+ fsn = cluster.getNamesystem();
+ hdfs = cluster.getFileSystem();
+ final File after = dumpTree2File("after.txt");
+ SnapshotTestHelper.compareDumpedTreeInFile(before, after, true);
+ }
+
+ private final PrintWriter output = new PrintWriter(System.out, true);
+ private int printTreeCount = 0;
+
+ String printTree(String label) throws Exception {
+ output.println();
+ output.println();
+ output.println("***** " + printTreeCount++ + ": " + label);
+ final String b =
+ fsn.getFSDirectory().getINode("/").dumpTreeRecursively().toString();
+ output.println(b);
+
+ final String s = NamespacePrintVisitor.print2Sting(fsn);
+ Assert.assertEquals(b, s);
+ return b;
+ }
+
+ @Test (timeout=60000)
+ public void testFSImageWithDoubleRename() throws Exception {
+ final Path dir1 = new Path("/dir1");
+ final Path dir2 = new Path("/dir2");
+ hdfs.mkdirs(dir1);
+ hdfs.mkdirs(dir2);
+ Path dira = new Path(dir1, "dira");
+ Path dirx = new Path(dir1, "dirx");
+ Path dirb = new Path(dira, "dirb");
+ hdfs.mkdirs(dira);
+ hdfs.mkdirs(dirb);
+ hdfs.mkdirs(dirx);
+ hdfs.allowSnapshot(dir1);
+ hdfs.createSnapshot(dir1, "s0");
+ Path file1 = new Path(dirb, "file1");
+ DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, (short) 1, SEED);
+ Path rennamePath = new Path(dirx, "dirb");
+ // mv /dir1/dira/dirb to /dir1/dirx/dirb
+ hdfs.rename(dirb, rennamePath);
+ hdfs.createSnapshot(dir1, "s1");
+ DFSTestUtil.appendFile(hdfs, new Path("/dir1/dirx/dirb/file1"),
+ "more data");
+ Path renamePath1 = new Path(dir2, "dira");
+ hdfs.mkdirs(renamePath1);
+ //mv dirx/dirb to /dir2/dira/dirb
+ hdfs.rename(rennamePath, renamePath1);
+ hdfs.delete(renamePath1, true);
+ hdfs.deleteSnapshot(dir1, "s1");
+ // save namespace and restart cluster
+ hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER);
+ hdfs.saveNamespace();
+ hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
+ cluster.shutdown();
+ cluster = new MiniDFSCluster.Builder(conf).format(false)
+ .numDataNodes(NUM_DATANODES).build();
+ cluster.waitActive();
+ fsn = cluster.getNamesystem();
+ hdfs = cluster.getFileSystem();
+ }
+
+
+ @Test (timeout=60000)
+ public void testFSImageWithRename1() throws Exception {
+ final Path dir1 = new Path("/dir1");
+ final Path dir2 = new Path("/dir2");
+ hdfs.mkdirs(dir1);
+ hdfs.mkdirs(dir2);
+ Path dira = new Path(dir1, "dira");
+ Path dirx = new Path(dir1, "dirx");
+ Path dirb = new Path(dirx, "dirb");
+ hdfs.mkdirs(dira);
+ hdfs.mkdirs(dirx);
+ hdfs.allowSnapshot(dir1);
+ hdfs.createSnapshot(dir1, "s0");
+ hdfs.mkdirs(dirb);
+ hdfs.createSnapshot(dir1, "s1");
+ Path rennamePath = new Path(dira, "dirb");
+ // mv /dir1/dirx/dirb to /dir1/dira/dirb
+ hdfs.rename(dirb, rennamePath);
+ hdfs.createSnapshot(dir1, "s2");
+ Path diry = new Path("/dir1/dira/dirb/diry");
+ hdfs.mkdirs(diry);
+ hdfs.createSnapshot(dir1, "s3");
+ Path file1 = new Path("/dir1/dira/dirb/diry/file1");
+ DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, (short) 1, SEED);
+ hdfs.createSnapshot(dir1, "s4");
+ hdfs.delete(new Path("/dir1/dira/dirb"), true);
+ hdfs.deleteSnapshot(dir1, "s1");
+ hdfs.deleteSnapshot(dir1, "s3");
+ // file1 should exist in the last snapshot
+ assertTrue(hdfs.exists(
+ new Path("/dir1/.snapshot/s4/dira/dirb/diry/file1")));
+
+ // save namespace and restart cluster
+ hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER);
+ hdfs.saveNamespace();
+ hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
+
+ cluster.shutdown();
+ cluster = new MiniDFSCluster.Builder(conf).format(false)
+ .numDataNodes(NUM_DATANODES).build();
+ cluster.waitActive();
+ fsn = cluster.getNamesystem();
+ hdfs = cluster.getFileSystem();
+ }
+
+ @Test (timeout=60000)
+ public void testFSImageWithRename2() throws Exception {
+ final Path dir1 = new Path("/dir1");
+ final Path dir2 = new Path("/dir2");
+ hdfs.mkdirs(dir1);
+ hdfs.mkdirs(dir2);
+ Path dira = new Path(dir1, "dira");
+ Path dirx = new Path(dir1, "dirx");
+ Path dirb = new Path(dirx, "dirb");
+ hdfs.mkdirs(dira);
+ hdfs.mkdirs(dirx);
+ hdfs.allowSnapshot(dir1);
+ hdfs.createSnapshot(dir1, "s0");
+ hdfs.mkdirs(dirb);
+ hdfs.createSnapshot(dir1, "s1");
+ Path rennamePath = new Path(dira, "dirb");
+ // mv /dir1/dirx/dirb to /dir1/dira/dirb
+ hdfs.rename(dirb, rennamePath);
+ hdfs.createSnapshot(dir1, "s2");
+ Path file1 = new Path("/dir1/dira/dirb/file1");
+ DFSTestUtil.createFile(hdfs,
+ new Path(
+ "/dir1/dira/dirb/file1"), BLOCKSIZE, (short) 1, SEED);
+ hdfs.createSnapshot(dir1, "s3");
+ hdfs.deleteSnapshot(dir1, "s1");
+ hdfs.deleteSnapshot(dir1, "s3");
+ assertTrue(hdfs.exists(file1));
+
+ // save namespace and restart cluster
+ hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER);
+ hdfs.saveNamespace();
+ hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
+
+ cluster.shutdown();
+ cluster = new MiniDFSCluster.Builder(conf).format(false)
+ .numDataNodes(NUM_DATANODES).build();
+ cluster.waitActive();
+ fsn = cluster.getNamesystem();
+ hdfs = cluster.getFileSystem();
+ }
+
+ @Test(timeout = 60000)
+ public void testFSImageWithRename3() throws Exception {
+ final Path dir1 = new Path("/dir1");
+ final Path dir2 = new Path("/dir2");
+ hdfs.mkdirs(dir1);
+ hdfs.mkdirs(dir2);
+ Path dira = new Path(dir1, "dira");
+ Path dirx = new Path(dir1, "dirx");
+ Path dirb = new Path(dirx, "dirb");
+ hdfs.mkdirs(dira);
+ hdfs.mkdirs(dirx);
+ hdfs.allowSnapshot(dir1);
+ hdfs.createSnapshot(dir1, "s0");
+ hdfs.mkdirs(dirb);
+ hdfs.createSnapshot(dir1, "s1");
+ Path rennamePath = new Path(dira, "dirb");
+ // mv /dir1/dirx/dirb to /dir1/dira/dirb
+ hdfs.rename(dirb, rennamePath);
+ hdfs.createSnapshot(dir1, "s2");
+ Path diry = new Path("/dir1/dira/dirb/diry");
+ hdfs.mkdirs(diry);
+ hdfs.createSnapshot(dir1, "s3");
+ Path file1 = new Path("/dir1/dira/dirb/diry/file1");
+ DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, (short) 1, SEED);
+ hdfs.createSnapshot(dir1, "s4");
+ hdfs.delete(new Path("/dir1/dira/dirb"), true);
+ hdfs.deleteSnapshot(dir1, "s1");
+ hdfs.deleteSnapshot(dir1, "s3");
+ // file1 should exist in the last snapshot
+ assertTrue(hdfs.exists(new Path(
+ "/dir1/.snapshot/s4/dira/dirb/diry/file1")));
+
+ // save namespace and restart cluster
+ hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER);
+ hdfs.saveNamespace();
+ hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
+
+ cluster.shutdown();
+ cluster = new MiniDFSCluster.Builder(conf).format(false)
+ .numDataNodes(NUM_DATANODES).build();
+ cluster.waitActive();
+ fsn = cluster.getNamesystem();
+ hdfs = cluster.getFileSystem();
+ }
+
+ @Test (timeout=60000)
+ public void testFSImageWithRename4() throws Exception {
+ final Path dir1 = new Path("/dir1");
+ final Path dir2 = new Path("/dir2");
+ hdfs.mkdirs(dir1);
+ hdfs.mkdirs(dir2);
+ Path dira = new Path(dir1, "dira");
+ Path dirx = new Path(dir1, "dirx");
+ Path dirb = new Path(dirx, "dirb");
+ hdfs.mkdirs(dira);
+ hdfs.mkdirs(dirx);
+ hdfs.allowSnapshot(dir1);
+ hdfs.createSnapshot(dir1, "s0");
+ hdfs.mkdirs(dirb);
+ hdfs.createSnapshot(dir1, "s1");
+ Path renamePath = new Path(dira, "dirb");
+ // mv /dir1/dirx/dirb to /dir1/dira/dirb
+ hdfs.rename(dirb, renamePath);
+ hdfs.createSnapshot(dir1, "s2");
+ Path diry = new Path("/dir1/dira/dirb/diry");
+ hdfs.mkdirs(diry);
+ hdfs.createSnapshot(dir1, "s3");
+ Path file1 = new Path("/dir1/dira/dirb/diry/file1");
+ DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, (short) 1, SEED);
+ hdfs.createSnapshot(dir1, "s4");
+ hdfs.delete(new Path("/dir1/dira/dirb/diry/file1"), false);
+ hdfs.deleteSnapshot(dir1, "s1");
+ hdfs.deleteSnapshot(dir1, "s3");
+ // file1 should exist in the last snapshot
+ assertTrue(hdfs.exists(
+ new Path("/dir1/.snapshot/s4/dira/dirb/diry/file1")));
+
+ // save namespace and restart cluster
+ hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER);
+ hdfs.saveNamespace();
+ hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
+
+ cluster.shutdown();
+ cluster = new MiniDFSCluster.Builder(conf).format(false)
+ .numDataNodes(NUM_DATANODES).build();
+ cluster.waitActive();
+ fsn = cluster.getNamesystem();
+ hdfs = cluster.getFileSystem();
+ }
+
+ @Test
+ public void testFSImageWithRename5() throws Exception {
+ final Path dir1 = new Path("/dir1");
+ final Path dir2 = new Path("/dir2");
+ hdfs.mkdirs(dir1);
+ hdfs.mkdirs(dir2);
+ Path dira = new Path(dir1, "dira");
+ Path dirx = new Path(dir1, "dirx");
+ Path dirb = new Path(dira, "dirb");
+ Path dirc = new Path(dirb, "dirc");
+ hdfs.mkdirs(dira);
+ hdfs.mkdirs(dirb);
+ hdfs.mkdirs(dirc);
+ hdfs.mkdirs(dirx);
+ hdfs.allowSnapshot(dir1);
+ hdfs.createSnapshot(dir1, "s0");
+ Path dird = new Path(dirc, "dird");
+ Path dire = new Path(dird, "dire");
+ Path file1 = new Path(dire, "file1");
+ DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, (short) 1, SEED);
+ Path rennamePath = new Path(dirx, "dirb");
+ // mv /dir1/dira/dirb to /dir1/dirx/dirb
+ hdfs.rename(dirb, rennamePath);
+ hdfs.createSnapshot(dir1, "s1");
+ DFSTestUtil.appendFile(hdfs,
+ new Path("/dir1/dirx/dirb/dirc/dird/dire/file1"), "more data");
+ Path renamePath1 = new Path(dir2, "dira");
+ hdfs.mkdirs(renamePath1);
+ //mv dirx/dirb to /dir2/dira/dirb
+ hdfs.rename(rennamePath, renamePath1);
+ hdfs.delete(renamePath1, true);
+ hdfs.deleteSnapshot(dir1, "s1");
+ // save namespace and restart cluster
+ hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER);
+ hdfs.saveNamespace();
+ hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
+ cluster.shutdown();
+ cluster = new MiniDFSCluster.Builder(conf).format(false)
+ .numDataNodes(NUM_DATANODES).build();
+ cluster.waitActive();
+ fsn = cluster.getNamesystem();
+ hdfs = cluster.getFileSystem();
+ }
+
+ @Test (timeout=60000)
+ public void testDoubleRenamesWithSnapshotDelete() throws Exception {
+ final Path sub1 = new Path(dir, "sub1");
+ hdfs.mkdirs(sub1);
+ hdfs.allowSnapshot(sub1);
+ final Path dir1 = new Path(sub1, "dir1");
+ final Path dir2 = new Path(sub1, "dir2");
+ final Path dir3 = new Path(sub1, "dir3");
+ final String snap3 = "snap3";
+ final String snap4 = "snap4";
+ final String snap5 = "snap5";
+ final String snap6 = "snap6";
+ final Path foo = new Path(dir2, "foo");
+ final Path bar = new Path(dir2, "bar");
+ hdfs.createSnapshot(sub1, "snap1");
+ hdfs.mkdirs(dir1, new FsPermission((short) 0777));
+ rename(dir1, dir2);
+ hdfs.createSnapshot(sub1, "snap2");
+ DFSTestUtil.createFile(hdfs, foo, BLOCKSIZE, (short) 1, SEED);
+ DFSTestUtil.createFile(hdfs, bar, BLOCKSIZE, (short) 1, SEED);
+ hdfs.createSnapshot(sub1, snap3);
+ hdfs.delete(foo, false);
+ DFSTestUtil.createFile(hdfs, foo, BLOCKSIZE, (short) 1, SEED);
+ hdfs.createSnapshot(sub1, snap4);
+ hdfs.delete(foo, false);
+ DFSTestUtil.createFile(hdfs, foo, BLOCKSIZE, (short) 1, SEED);
+ hdfs.createSnapshot(sub1, snap5);
+ rename(dir2, dir3);
+ hdfs.createSnapshot(sub1, snap6);
+ hdfs.delete(dir3, true);
+ deleteSnapshot(sub1, snap6);
+ deleteSnapshot(sub1, snap3);
+ // save namespace and restart Namenode
+ hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER);
+ hdfs.saveNamespace();
+ hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
+ cluster.restartNameNode(true);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org