You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ay...@apache.org on 2020/07/04 07:36:03 UTC
[hadoop] branch branch-3.1 updated: HDFS-15446. CreateSnapshotOp
fails during edit log loading for /.reserved/raw/path with error
java.io.FileNotFoundException: Directory does not exist:
/.reserved/raw/path. Contributed by Stephen O'Donnell.
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 1e0424f HDFS-15446. CreateSnapshotOp fails during edit log loading for /.reserved/raw/path with error java.io.FileNotFoundException: Directory does not exist: /.reserved/raw/path. Contributed by Stephen O'Donnell.
1e0424f is described below
commit 1e0424f39a5ed405910751d4311b5352de9bf8ca
Author: Ayush Saxena <ay...@apache.org>
AuthorDate: Sat Jul 4 12:24:49 2020 +0530
HDFS-15446. CreateSnapshotOp fails during edit log loading for /.reserved/raw/path with error java.io.FileNotFoundException: Directory does not exist: /.reserved/raw/path. Contributed by Stephen O'Donnell.
---
.../hadoop/hdfs/server/namenode/FSDirectory.java | 20 ++++++++
.../hdfs/server/namenode/FSEditLogLoader.java | 6 +--
.../server/namenode/snapshot/TestSnapshot.java | 53 ++++++++++++++++++++++
3 files changed, 76 insertions(+), 3 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index a6664f0..9e99e67 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -686,6 +686,26 @@ public class FSDirectory implements Closeable {
return iip;
}
+ /**
+ * This method should only be used from internal paths and not those provided
+ * directly by a user. It resolves a given path into an INodesInPath in a
+ * similar way to resolvePath(...), only traversal and permissions are not
+ * checked.
+ * @param src The path to resolve.
+ * @return if the path indicates an inode, return path after replacing up to
+ * {@code <inodeid>} with the corresponding path of the inode, else
+ * the path in {@code src} as is. If the path refers to a path in
+ * the "raw" directory, return the non-raw pathname.
+ * @throws FileNotFoundException
+ */
+ public INodesInPath unprotectedResolvePath(String src)
+ throws FileNotFoundException {
+ byte[][] components = INode.getPathComponents(src);
+ boolean isRaw = isReservedRawName(components);
+ components = resolveComponents(components, this);
+ return INodesInPath.resolve(rootDir, components, isRaw);
+ }
+
INodesInPath resolvePath(FSPermissionChecker pc, String src, long fileId)
throws UnresolvedLinkException, FileNotFoundException,
AccessControlException, ParentNotDirectoryException {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index 91506d8..7eb413d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -797,7 +797,7 @@ public class FSEditLogLoader {
final String snapshotRoot =
renameReservedPathsOnUpgrade(createSnapshotOp.snapshotRoot,
logVersion);
- INodesInPath iip = fsDir.getINodesInPath(snapshotRoot, DirOp.WRITE);
+ INodesInPath iip = fsDir.unprotectedResolvePath(snapshotRoot);
String path = fsNamesys.getSnapshotManager().createSnapshot(
fsDir.getFSNamesystem().getLeaseManager(),
iip, snapshotRoot, createSnapshotOp.snapshotName);
@@ -814,7 +814,7 @@ public class FSEditLogLoader {
final String snapshotRoot =
renameReservedPathsOnUpgrade(deleteSnapshotOp.snapshotRoot,
logVersion);
- INodesInPath iip = fsDir.getINodesInPath(snapshotRoot, DirOp.WRITE);
+ INodesInPath iip = fsDir.unprotectedResolvePath(snapshotRoot);
fsNamesys.getSnapshotManager().deleteSnapshot(iip,
deleteSnapshotOp.snapshotName,
new INode.ReclaimContext(fsNamesys.dir.getBlockStoragePolicySuite(),
@@ -836,7 +836,7 @@ public class FSEditLogLoader {
final String snapshotRoot =
renameReservedPathsOnUpgrade(renameSnapshotOp.snapshotRoot,
logVersion);
- INodesInPath iip = fsDir.getINodesInPath(snapshotRoot, DirOp.WRITE);
+ INodesInPath iip = fsDir.unprotectedResolvePath(snapshotRoot);
fsNamesys.getSnapshotManager().renameSnapshot(iip,
snapshotRoot, renameSnapshotOp.snapshotOldName,
renameSnapshotOp.snapshotNewName);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java
index 1c01ece..7a5eb8d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java
@@ -457,6 +457,59 @@ public class TestSnapshot {
}
/**
+ * HDFS-15446 - ensure that snapshot operations on /.reserved/raw
+ * paths work and the NN can load the resulting edits.
+ */
+ @Test(timeout = 60000)
+ public void testSnapshotOpsOnReservedPath() throws Exception {
+ Path dir = new Path("/dir");
+ Path nestedDir = new Path("/nested/dir");
+ Path sub = new Path(dir, "sub");
+ Path subFile = new Path(sub, "file");
+ Path nestedFile = new Path(nestedDir, "file");
+ DFSTestUtil.createFile(hdfs, subFile, BLOCKSIZE, REPLICATION, seed);
+ DFSTestUtil.createFile(hdfs, nestedFile, BLOCKSIZE, REPLICATION, seed);
+
+ hdfs.allowSnapshot(dir);
+ hdfs.allowSnapshot(nestedDir);
+ Path reservedDir = new Path("/.reserved/raw/dir");
+ Path reservedNestedDir = new Path("/.reserved/raw/nested/dir");
+ hdfs.createSnapshot(reservedDir, "s1");
+ hdfs.createSnapshot(reservedNestedDir, "s1");
+ hdfs.renameSnapshot(reservedDir, "s1", "s2");
+ hdfs.renameSnapshot(reservedNestedDir, "s1", "s2");
+ hdfs.deleteSnapshot(reservedDir, "s2");
+ hdfs.deleteSnapshot(reservedNestedDir, "s2");
+ // The original problem with reserved path, is that the NN was unable to
+ // replay the edits, therefore restarting the NN to ensure it starts
+ // and no exceptions are raised.
+ cluster.restartNameNode(true);
+ }
+
+ /**
+ * HDFS-15446 - ensure that snapshot operations on /.reserved/raw
+ * paths work and the NN can load the resulting edits. This test if for
+ * snapshots at the root level.
+ */
+ @Test(timeout = 60000)
+ public void testSnapshotOpsOnRootReservedPath() throws Exception {
+ Path dir = new Path("/");
+ Path sub = new Path(dir, "sub");
+ Path subFile = new Path(sub, "file");
+ DFSTestUtil.createFile(hdfs, subFile, BLOCKSIZE, REPLICATION, seed);
+
+ hdfs.allowSnapshot(dir);
+ Path reservedDir = new Path("/.reserved/raw");
+ hdfs.createSnapshot(reservedDir, "s1");
+ hdfs.renameSnapshot(reservedDir, "s1", "s2");
+ hdfs.deleteSnapshot(reservedDir, "s2");
+ // The original problem with reserved path, is that the NN was unable to
+ // replay the edits, therefore restarting the NN to ensure it starts
+ // and no exceptions are raised.
+ cluster.restartNameNode(true);
+ }
+
+ /**
* Prepare a list of modifications. A modification may be a file creation,
* file deletion, or a modification operation such as appending to an existing
* file.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org