You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2015/05/15 00:03:14 UTC
[1/2] hadoop git commit: HADOOP-11713. ViewFileSystem should support
snapshot methods. Contributed by Rakesh R.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 bc13c7d84 -> fa082e1a0
refs/heads/trunk 15ccd967e -> 09fe16f16
HADOOP-11713. ViewFileSystem should support snapshot methods. Contributed by Rakesh R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/09fe16f1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/09fe16f1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/09fe16f1
Branch: refs/heads/trunk
Commit: 09fe16f166392a99e1e54001a9112c6a4632dfc8
Parents: 15ccd96
Author: cnauroth <cn...@apache.org>
Authored: Thu May 14 14:55:58 2015 -0700
Committer: cnauroth <cn...@apache.org>
Committed: Thu May 14 14:55:58 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../hadoop/fs/viewfs/ChRootedFileSystem.java | 17 +++++++
.../org/apache/hadoop/fs/viewfs/ChRootedFs.java | 19 ++++++-
.../apache/hadoop/fs/viewfs/ViewFileSystem.java | 48 +++++++++++++++++-
.../org/apache/hadoop/fs/viewfs/ViewFs.java | 51 +++++++++++++++++--
.../fs/viewfs/TestChRootedFileSystem.java | 52 ++++++++++++++++++++
.../apache/hadoop/fs/viewfs/TestChRootedFs.java | 41 +++++++++++++++
.../fs/viewfs/ViewFileSystemBaseTest.java | 20 ++++++++
.../apache/hadoop/fs/viewfs/ViewFsBaseTest.java | 21 ++++++++
9 files changed, 267 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09fe16f1/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 359a38b..2f8acb0 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -566,6 +566,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-9723. Improve error message when hadoop archive output path already
exists. (Jean-Baptiste Onofré and Yongjun Zhang via aajisak)
+ HADOOP-11713. ViewFileSystem should support snapshot methods.
+ (Rakesh R via cnauroth)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09fe16f1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
index 18e2391..f7a93e7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
@@ -364,6 +364,23 @@ class ChRootedFileSystem extends FilterFileSystem {
}
@Override
+ public Path createSnapshot(Path path, String name) throws IOException {
+ return super.createSnapshot(fullPath(path), name);
+ }
+
+ @Override
+ public void renameSnapshot(Path path, String snapshotOldName,
+ String snapshotNewName) throws IOException {
+ super.renameSnapshot(fullPath(path), snapshotOldName, snapshotNewName);
+ }
+
+ @Override
+ public void deleteSnapshot(Path snapshotDir, String snapshotName)
+ throws IOException {
+ super.deleteSnapshot(fullPath(snapshotDir), snapshotName);
+ }
+
+ @Override
public Path resolvePath(final Path p) throws IOException {
return super.resolvePath(fullPath(p));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09fe16f1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
index 68e756a8..a05a700 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
@@ -361,7 +361,24 @@ class ChRootedFs extends AbstractFileSystem {
}
@Override
- public void setVerifyChecksum(final boolean verifyChecksum)
+ public Path createSnapshot(Path path, String name) throws IOException {
+ return myFs.createSnapshot(fullPath(path), name);
+ }
+
+ @Override
+ public void renameSnapshot(Path path, String snapshotOldName,
+ String snapshotNewName) throws IOException {
+ myFs.renameSnapshot(fullPath(path), snapshotOldName, snapshotNewName);
+ }
+
+ @Override
+ public void deleteSnapshot(Path snapshotDir, String snapshotName)
+ throws IOException {
+ myFs.deleteSnapshot(fullPath(snapshotDir), snapshotName);
+ }
+
+ @Override
+ public void setVerifyChecksum(final boolean verifyChecksum)
throws IOException, UnresolvedLinkException {
myFs.setVerifyChecksum(verifyChecksum);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09fe16f1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
index 43fe23f..8605064 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
@@ -726,7 +726,32 @@ public class ViewFileSystem extends FileSystem {
}
return result;
}
-
+
+ @Override
+ public Path createSnapshot(Path path, String snapshotName)
+ throws IOException {
+ InodeTree.ResolveResult<FileSystem> res = fsState.resolve(getUriPath(path),
+ true);
+ return res.targetFileSystem.createSnapshot(res.remainingPath, snapshotName);
+ }
+
+ @Override
+ public void renameSnapshot(Path path, String snapshotOldName,
+ String snapshotNewName) throws IOException {
+ InodeTree.ResolveResult<FileSystem> res = fsState.resolve(getUriPath(path),
+ true);
+ res.targetFileSystem.renameSnapshot(res.remainingPath, snapshotOldName,
+ snapshotNewName);
+ }
+
+ @Override
+ public void deleteSnapshot(Path path, String snapshotName)
+ throws IOException {
+ InodeTree.ResolveResult<FileSystem> res = fsState.resolve(getUriPath(path),
+ true);
+ res.targetFileSystem.deleteSnapshot(res.remainingPath, snapshotName);
+ }
+
/*
* An instance of this class represents an internal dir of the viewFs
* that is internal dir of the mount table.
@@ -1020,5 +1045,26 @@ public class ViewFileSystem extends FileSystem {
checkPathIsSlash(path);
throw readOnlyMountTable("removeXAttr", path);
}
+
+ @Override
+ public Path createSnapshot(Path path, String snapshotName)
+ throws IOException {
+ checkPathIsSlash(path);
+ throw readOnlyMountTable("createSnapshot", path);
+ }
+
+ @Override
+ public void renameSnapshot(Path path, String snapshotOldName,
+ String snapshotNewName) throws IOException {
+ checkPathIsSlash(path);
+ throw readOnlyMountTable("renameSnapshot", path);
+ }
+
+ @Override
+ public void deleteSnapshot(Path path, String snapshotName)
+ throws IOException {
+ checkPathIsSlash(path);
+ throw readOnlyMountTable("deleteSnapshot", path);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09fe16f1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
index 975496a..a23aa86 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
@@ -621,7 +621,8 @@ public class ViewFs extends AbstractFileSystem {
@Override
public boolean isValidName(String src) {
- // Prefix validated at mount time and rest of path validated by mount target.
+ // Prefix validated at mount time and rest of path validated by mount
+ // target.
return true;
}
@@ -714,8 +715,31 @@ public class ViewFs extends AbstractFileSystem {
fsState.resolve(getUriPath(path), true);
res.targetFileSystem.removeXAttr(res.remainingPath, name);
}
-
-
+
+ @Override
+ public Path createSnapshot(Path path, String snapshotName)
+ throws IOException {
+ InodeTree.ResolveResult<AbstractFileSystem> res = fsState.resolve(
+ getUriPath(path), true);
+ return res.targetFileSystem.createSnapshot(res.remainingPath, snapshotName);
+ }
+
+ @Override
+ public void renameSnapshot(Path path, String snapshotOldName,
+ String snapshotNewName) throws IOException {
+ InodeTree.ResolveResult<AbstractFileSystem> res = fsState.resolve(
+ getUriPath(path), true);
+ res.targetFileSystem.renameSnapshot(res.remainingPath, snapshotOldName,
+ snapshotNewName);
+ }
+
+ @Override
+ public void deleteSnapshot(Path path, String snapshotName) throws IOException {
+ InodeTree.ResolveResult<AbstractFileSystem> res = fsState.resolve(
+ getUriPath(path), true);
+ res.targetFileSystem.deleteSnapshot(res.remainingPath, snapshotName);
+ }
+
/*
* An instance of this class represents an internal dir of the viewFs
* ie internal dir of the mount table.
@@ -1025,5 +1049,26 @@ public class ViewFs extends AbstractFileSystem {
checkPathIsSlash(path);
throw readOnlyMountTable("removeXAttr", path);
}
+
+ @Override
+ public Path createSnapshot(Path path, String snapshotName)
+ throws IOException {
+ checkPathIsSlash(path);
+ throw readOnlyMountTable("createSnapshot", path);
+ }
+
+ @Override
+ public void renameSnapshot(Path path, String snapshotOldName,
+ String snapshotNewName) throws IOException {
+ checkPathIsSlash(path);
+ throw readOnlyMountTable("renameSnapshot", path);
+ }
+
+ @Override
+ public void deleteSnapshot(Path path, String snapshotName)
+ throws IOException {
+ checkPathIsSlash(path);
+ throw readOnlyMountTable("deleteSnapshot", path);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09fe16f1/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java
index a13ee8d..e4a1ac9 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java
@@ -416,4 +416,56 @@ public class TestChRootedFileSystem {
@Override
public void initialize(URI name, Configuration conf) throws IOException {}
}
+
+ @Test(timeout = 30000)
+ public void testCreateSnapshot() throws Exception {
+ Path snapRootPath = new Path("/snapPath");
+ Path chRootedSnapRootPath = new Path("/a/b/snapPath");
+
+ Configuration conf = new Configuration();
+ conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
+
+ URI chrootUri = URI.create("mockfs://foo/a/b");
+ ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf);
+ FileSystem mockFs = ((FilterFileSystem) chrootFs.getRawFileSystem())
+ .getRawFileSystem();
+
+ chrootFs.createSnapshot(snapRootPath, "snap1");
+ verify(mockFs).createSnapshot(chRootedSnapRootPath, "snap1");
+ }
+
+ @Test(timeout = 30000)
+ public void testDeleteSnapshot() throws Exception {
+ Path snapRootPath = new Path("/snapPath");
+ Path chRootedSnapRootPath = new Path("/a/b/snapPath");
+
+ Configuration conf = new Configuration();
+ conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
+
+ URI chrootUri = URI.create("mockfs://foo/a/b");
+ ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf);
+ FileSystem mockFs = ((FilterFileSystem) chrootFs.getRawFileSystem())
+ .getRawFileSystem();
+
+ chrootFs.deleteSnapshot(snapRootPath, "snap1");
+ verify(mockFs).deleteSnapshot(chRootedSnapRootPath, "snap1");
+ }
+
+ @Test(timeout = 30000)
+ public void testRenameSnapshot() throws Exception {
+ Path snapRootPath = new Path("/snapPath");
+ Path chRootedSnapRootPath = new Path("/a/b/snapPath");
+
+ Configuration conf = new Configuration();
+ conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
+
+ URI chrootUri = URI.create("mockfs://foo/a/b");
+ ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf);
+ FileSystem mockFs = ((FilterFileSystem) chrootFs.getRawFileSystem())
+ .getRawFileSystem();
+
+ chrootFs.renameSnapshot(snapRootPath, "snapOldName", "snapNewName");
+ verify(mockFs).renameSnapshot(chRootedSnapRootPath, "snapOldName",
+ "snapNewName");
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09fe16f1/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java
index e639291..20825e3 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java
@@ -327,4 +327,45 @@ public class TestChRootedFs {
Assert.assertFalse(chRootedFs.isValidName("/test"));
Mockito.verify(baseFs).isValidName("/chroot/test");
}
+
+ @Test(timeout = 30000)
+ public void testCreateSnapshot() throws Exception {
+ Path snapRootPath = new Path("/snapPath");
+ Path chRootedSnapRootPath = new Path(
+ Path.getPathWithoutSchemeAndAuthority(chrootedTo), "snapPath");
+ AbstractFileSystem baseFs = Mockito.spy(fc.getDefaultFileSystem());
+ ChRootedFs chRootedFs = new ChRootedFs(baseFs, chrootedTo);
+ Mockito.doReturn(snapRootPath).when(baseFs)
+ .createSnapshot(chRootedSnapRootPath, "snap1");
+ Assert.assertEquals(snapRootPath,
+ chRootedFs.createSnapshot(snapRootPath, "snap1"));
+ Mockito.verify(baseFs).createSnapshot(chRootedSnapRootPath, "snap1");
+ }
+
+ @Test(timeout = 30000)
+ public void testDeleteSnapshot() throws Exception {
+ Path snapRootPath = new Path("/snapPath");
+ Path chRootedSnapRootPath = new Path(
+ Path.getPathWithoutSchemeAndAuthority(chrootedTo), "snapPath");
+ AbstractFileSystem baseFs = Mockito.spy(fc.getDefaultFileSystem());
+ ChRootedFs chRootedFs = new ChRootedFs(baseFs, chrootedTo);
+ Mockito.doNothing().when(baseFs)
+ .deleteSnapshot(chRootedSnapRootPath, "snap1");
+ chRootedFs.deleteSnapshot(snapRootPath, "snap1");
+ Mockito.verify(baseFs).deleteSnapshot(chRootedSnapRootPath, "snap1");
+ }
+
+ @Test(timeout = 30000)
+ public void testRenameSnapshot() throws Exception {
+ Path snapRootPath = new Path("/snapPath");
+ Path chRootedSnapRootPath = new Path(
+ Path.getPathWithoutSchemeAndAuthority(chrootedTo), "snapPath");
+ AbstractFileSystem baseFs = Mockito.spy(fc.getDefaultFileSystem());
+ ChRootedFs chRootedFs = new ChRootedFs(baseFs, chrootedTo);
+ Mockito.doNothing().when(baseFs)
+ .renameSnapshot(chRootedSnapRootPath, "snapOldName", "snapNewName");
+ chRootedFs.renameSnapshot(snapRootPath, "snapOldName", "snapNewName");
+ Mockito.verify(baseFs).renameSnapshot(chRootedSnapRootPath, "snapOldName",
+ "snapNewName");
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09fe16f1/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
index 18769c2..7fad990 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
@@ -865,4 +865,24 @@ public class ViewFileSystemBaseTest {
fsView.removeXAttr(new Path("/internalDir"), "xattrName");
}
+ @Test(expected = AccessControlException.class)
+ public void testInternalCreateSnapshot1() throws IOException {
+ fsView.createSnapshot(new Path("/internalDir"));
+ }
+
+ @Test(expected = AccessControlException.class)
+ public void testInternalCreateSnapshot2() throws IOException {
+ fsView.createSnapshot(new Path("/internalDir"), "snap1");
+ }
+
+ @Test(expected = AccessControlException.class)
+ public void testInternalRenameSnapshot() throws IOException {
+ fsView.renameSnapshot(new Path("/internalDir"), "snapOldName",
+ "snapNewName");
+ }
+
+ @Test(expected = AccessControlException.class)
+ public void testInternalDeleteSnapshot() throws IOException {
+ fsView.deleteSnapshot(new Path("/internalDir"), "snap1");
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09fe16f1/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
index 035b280..d8ab539 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
@@ -777,4 +777,25 @@ public class ViewFsBaseTest {
public void testInternalRemoveXAttr() throws IOException {
fcView.removeXAttr(new Path("/internalDir"), "xattrName");
}
+
+ @Test(expected = AccessControlException.class)
+ public void testInternalCreateSnapshot1() throws IOException {
+ fcView.createSnapshot(new Path("/internalDir"));
+ }
+
+ @Test(expected = AccessControlException.class)
+ public void testInternalCreateSnapshot2() throws IOException {
+ fcView.createSnapshot(new Path("/internalDir"), "snap1");
+ }
+
+ @Test(expected = AccessControlException.class)
+ public void testInternalRenameSnapshot() throws IOException {
+ fcView.renameSnapshot(new Path("/internalDir"), "snapOldName",
+ "snapNewName");
+ }
+
+ @Test(expected = AccessControlException.class)
+ public void testInternalDeleteSnapshot() throws IOException {
+ fcView.deleteSnapshot(new Path("/internalDir"), "snap1");
+ }
}
[2/2] hadoop git commit: HADOOP-11713. ViewFileSystem should support
snapshot methods. Contributed by Rakesh R.
Posted by cn...@apache.org.
HADOOP-11713. ViewFileSystem should support snapshot methods. Contributed by Rakesh R.
(cherry picked from commit 09fe16f166392a99e1e54001a9112c6a4632dfc8)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fa082e1a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fa082e1a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fa082e1a
Branch: refs/heads/branch-2
Commit: fa082e1a0a32012de1befd38e34c18c7af303006
Parents: bc13c7d
Author: cnauroth <cn...@apache.org>
Authored: Thu May 14 14:55:58 2015 -0700
Committer: cnauroth <cn...@apache.org>
Committed: Thu May 14 14:56:13 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../hadoop/fs/viewfs/ChRootedFileSystem.java | 17 +++++++
.../org/apache/hadoop/fs/viewfs/ChRootedFs.java | 19 ++++++-
.../apache/hadoop/fs/viewfs/ViewFileSystem.java | 48 +++++++++++++++++-
.../org/apache/hadoop/fs/viewfs/ViewFs.java | 51 +++++++++++++++++--
.../fs/viewfs/TestChRootedFileSystem.java | 52 ++++++++++++++++++++
.../apache/hadoop/fs/viewfs/TestChRootedFs.java | 41 +++++++++++++++
.../fs/viewfs/ViewFileSystemBaseTest.java | 20 ++++++++
.../apache/hadoop/fs/viewfs/ViewFsBaseTest.java | 21 ++++++++
9 files changed, 267 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa082e1a/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index e43540b..412719d 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -95,6 +95,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-9723. Improve error message when hadoop archive output path already
exists. (Jean-Baptiste Onofré and Yongjun Zhang via aajisak)
+ HADOOP-11713. ViewFileSystem should support snapshot methods.
+ (Rakesh R via cnauroth)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa082e1a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
index 18e2391..f7a93e7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
@@ -364,6 +364,23 @@ class ChRootedFileSystem extends FilterFileSystem {
}
@Override
+ public Path createSnapshot(Path path, String name) throws IOException {
+ return super.createSnapshot(fullPath(path), name);
+ }
+
+ @Override
+ public void renameSnapshot(Path path, String snapshotOldName,
+ String snapshotNewName) throws IOException {
+ super.renameSnapshot(fullPath(path), snapshotOldName, snapshotNewName);
+ }
+
+ @Override
+ public void deleteSnapshot(Path snapshotDir, String snapshotName)
+ throws IOException {
+ super.deleteSnapshot(fullPath(snapshotDir), snapshotName);
+ }
+
+ @Override
public Path resolvePath(final Path p) throws IOException {
return super.resolvePath(fullPath(p));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa082e1a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
index 68e756a8..a05a700 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
@@ -361,7 +361,24 @@ class ChRootedFs extends AbstractFileSystem {
}
@Override
- public void setVerifyChecksum(final boolean verifyChecksum)
+ public Path createSnapshot(Path path, String name) throws IOException {
+ return myFs.createSnapshot(fullPath(path), name);
+ }
+
+ @Override
+ public void renameSnapshot(Path path, String snapshotOldName,
+ String snapshotNewName) throws IOException {
+ myFs.renameSnapshot(fullPath(path), snapshotOldName, snapshotNewName);
+ }
+
+ @Override
+ public void deleteSnapshot(Path snapshotDir, String snapshotName)
+ throws IOException {
+ myFs.deleteSnapshot(fullPath(snapshotDir), snapshotName);
+ }
+
+ @Override
+ public void setVerifyChecksum(final boolean verifyChecksum)
throws IOException, UnresolvedLinkException {
myFs.setVerifyChecksum(verifyChecksum);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa082e1a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
index 43fe23f..8605064 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
@@ -726,7 +726,32 @@ public class ViewFileSystem extends FileSystem {
}
return result;
}
-
+
+ @Override
+ public Path createSnapshot(Path path, String snapshotName)
+ throws IOException {
+ InodeTree.ResolveResult<FileSystem> res = fsState.resolve(getUriPath(path),
+ true);
+ return res.targetFileSystem.createSnapshot(res.remainingPath, snapshotName);
+ }
+
+ @Override
+ public void renameSnapshot(Path path, String snapshotOldName,
+ String snapshotNewName) throws IOException {
+ InodeTree.ResolveResult<FileSystem> res = fsState.resolve(getUriPath(path),
+ true);
+ res.targetFileSystem.renameSnapshot(res.remainingPath, snapshotOldName,
+ snapshotNewName);
+ }
+
+ @Override
+ public void deleteSnapshot(Path path, String snapshotName)
+ throws IOException {
+ InodeTree.ResolveResult<FileSystem> res = fsState.resolve(getUriPath(path),
+ true);
+ res.targetFileSystem.deleteSnapshot(res.remainingPath, snapshotName);
+ }
+
/*
* An instance of this class represents an internal dir of the viewFs
* that is internal dir of the mount table.
@@ -1020,5 +1045,26 @@ public class ViewFileSystem extends FileSystem {
checkPathIsSlash(path);
throw readOnlyMountTable("removeXAttr", path);
}
+
+ @Override
+ public Path createSnapshot(Path path, String snapshotName)
+ throws IOException {
+ checkPathIsSlash(path);
+ throw readOnlyMountTable("createSnapshot", path);
+ }
+
+ @Override
+ public void renameSnapshot(Path path, String snapshotOldName,
+ String snapshotNewName) throws IOException {
+ checkPathIsSlash(path);
+ throw readOnlyMountTable("renameSnapshot", path);
+ }
+
+ @Override
+ public void deleteSnapshot(Path path, String snapshotName)
+ throws IOException {
+ checkPathIsSlash(path);
+ throw readOnlyMountTable("deleteSnapshot", path);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa082e1a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
index 975496a..a23aa86 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
@@ -621,7 +621,8 @@ public class ViewFs extends AbstractFileSystem {
@Override
public boolean isValidName(String src) {
- // Prefix validated at mount time and rest of path validated by mount target.
+ // Prefix validated at mount time and rest of path validated by mount
+ // target.
return true;
}
@@ -714,8 +715,31 @@ public class ViewFs extends AbstractFileSystem {
fsState.resolve(getUriPath(path), true);
res.targetFileSystem.removeXAttr(res.remainingPath, name);
}
-
-
+
+ @Override
+ public Path createSnapshot(Path path, String snapshotName)
+ throws IOException {
+ InodeTree.ResolveResult<AbstractFileSystem> res = fsState.resolve(
+ getUriPath(path), true);
+ return res.targetFileSystem.createSnapshot(res.remainingPath, snapshotName);
+ }
+
+ @Override
+ public void renameSnapshot(Path path, String snapshotOldName,
+ String snapshotNewName) throws IOException {
+ InodeTree.ResolveResult<AbstractFileSystem> res = fsState.resolve(
+ getUriPath(path), true);
+ res.targetFileSystem.renameSnapshot(res.remainingPath, snapshotOldName,
+ snapshotNewName);
+ }
+
+ @Override
+ public void deleteSnapshot(Path path, String snapshotName) throws IOException {
+ InodeTree.ResolveResult<AbstractFileSystem> res = fsState.resolve(
+ getUriPath(path), true);
+ res.targetFileSystem.deleteSnapshot(res.remainingPath, snapshotName);
+ }
+
/*
* An instance of this class represents an internal dir of the viewFs
* ie internal dir of the mount table.
@@ -1025,5 +1049,26 @@ public class ViewFs extends AbstractFileSystem {
checkPathIsSlash(path);
throw readOnlyMountTable("removeXAttr", path);
}
+
+ @Override
+ public Path createSnapshot(Path path, String snapshotName)
+ throws IOException {
+ checkPathIsSlash(path);
+ throw readOnlyMountTable("createSnapshot", path);
+ }
+
+ @Override
+ public void renameSnapshot(Path path, String snapshotOldName,
+ String snapshotNewName) throws IOException {
+ checkPathIsSlash(path);
+ throw readOnlyMountTable("renameSnapshot", path);
+ }
+
+ @Override
+ public void deleteSnapshot(Path path, String snapshotName)
+ throws IOException {
+ checkPathIsSlash(path);
+ throw readOnlyMountTable("deleteSnapshot", path);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa082e1a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java
index a13ee8d..e4a1ac9 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java
@@ -416,4 +416,56 @@ public class TestChRootedFileSystem {
@Override
public void initialize(URI name, Configuration conf) throws IOException {}
}
+
+ @Test(timeout = 30000)
+ public void testCreateSnapshot() throws Exception {
+ Path snapRootPath = new Path("/snapPath");
+ Path chRootedSnapRootPath = new Path("/a/b/snapPath");
+
+ Configuration conf = new Configuration();
+ conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
+
+ URI chrootUri = URI.create("mockfs://foo/a/b");
+ ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf);
+ FileSystem mockFs = ((FilterFileSystem) chrootFs.getRawFileSystem())
+ .getRawFileSystem();
+
+ chrootFs.createSnapshot(snapRootPath, "snap1");
+ verify(mockFs).createSnapshot(chRootedSnapRootPath, "snap1");
+ }
+
+ @Test(timeout = 30000)
+ public void testDeleteSnapshot() throws Exception {
+ Path snapRootPath = new Path("/snapPath");
+ Path chRootedSnapRootPath = new Path("/a/b/snapPath");
+
+ Configuration conf = new Configuration();
+ conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
+
+ URI chrootUri = URI.create("mockfs://foo/a/b");
+ ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf);
+ FileSystem mockFs = ((FilterFileSystem) chrootFs.getRawFileSystem())
+ .getRawFileSystem();
+
+ chrootFs.deleteSnapshot(snapRootPath, "snap1");
+ verify(mockFs).deleteSnapshot(chRootedSnapRootPath, "snap1");
+ }
+
+ @Test(timeout = 30000)
+ public void testRenameSnapshot() throws Exception {
+ Path snapRootPath = new Path("/snapPath");
+ Path chRootedSnapRootPath = new Path("/a/b/snapPath");
+
+ Configuration conf = new Configuration();
+ conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
+
+ URI chrootUri = URI.create("mockfs://foo/a/b");
+ ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf);
+ FileSystem mockFs = ((FilterFileSystem) chrootFs.getRawFileSystem())
+ .getRawFileSystem();
+
+ chrootFs.renameSnapshot(snapRootPath, "snapOldName", "snapNewName");
+ verify(mockFs).renameSnapshot(chRootedSnapRootPath, "snapOldName",
+ "snapNewName");
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa082e1a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java
index e639291..20825e3 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java
@@ -327,4 +327,45 @@ public class TestChRootedFs {
Assert.assertFalse(chRootedFs.isValidName("/test"));
Mockito.verify(baseFs).isValidName("/chroot/test");
}
+
+ @Test(timeout = 30000)
+ public void testCreateSnapshot() throws Exception {
+ Path snapRootPath = new Path("/snapPath");
+ Path chRootedSnapRootPath = new Path(
+ Path.getPathWithoutSchemeAndAuthority(chrootedTo), "snapPath");
+ AbstractFileSystem baseFs = Mockito.spy(fc.getDefaultFileSystem());
+ ChRootedFs chRootedFs = new ChRootedFs(baseFs, chrootedTo);
+ Mockito.doReturn(snapRootPath).when(baseFs)
+ .createSnapshot(chRootedSnapRootPath, "snap1");
+ Assert.assertEquals(snapRootPath,
+ chRootedFs.createSnapshot(snapRootPath, "snap1"));
+ Mockito.verify(baseFs).createSnapshot(chRootedSnapRootPath, "snap1");
+ }
+
+ @Test(timeout = 30000)
+ public void testDeleteSnapshot() throws Exception {
+ Path snapRootPath = new Path("/snapPath");
+ Path chRootedSnapRootPath = new Path(
+ Path.getPathWithoutSchemeAndAuthority(chrootedTo), "snapPath");
+ AbstractFileSystem baseFs = Mockito.spy(fc.getDefaultFileSystem());
+ ChRootedFs chRootedFs = new ChRootedFs(baseFs, chrootedTo);
+ Mockito.doNothing().when(baseFs)
+ .deleteSnapshot(chRootedSnapRootPath, "snap1");
+ chRootedFs.deleteSnapshot(snapRootPath, "snap1");
+ Mockito.verify(baseFs).deleteSnapshot(chRootedSnapRootPath, "snap1");
+ }
+
+ @Test(timeout = 30000)
+ public void testRenameSnapshot() throws Exception {
+ Path snapRootPath = new Path("/snapPath");
+ Path chRootedSnapRootPath = new Path(
+ Path.getPathWithoutSchemeAndAuthority(chrootedTo), "snapPath");
+ AbstractFileSystem baseFs = Mockito.spy(fc.getDefaultFileSystem());
+ ChRootedFs chRootedFs = new ChRootedFs(baseFs, chrootedTo);
+ Mockito.doNothing().when(baseFs)
+ .renameSnapshot(chRootedSnapRootPath, "snapOldName", "snapNewName");
+ chRootedFs.renameSnapshot(snapRootPath, "snapOldName", "snapNewName");
+ Mockito.verify(baseFs).renameSnapshot(chRootedSnapRootPath, "snapOldName",
+ "snapNewName");
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa082e1a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
index 18769c2..7fad990 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
@@ -865,4 +865,24 @@ public class ViewFileSystemBaseTest {
fsView.removeXAttr(new Path("/internalDir"), "xattrName");
}
+ @Test(expected = AccessControlException.class)
+ public void testInternalCreateSnapshot1() throws IOException {
+ fsView.createSnapshot(new Path("/internalDir"));
+ }
+
+ @Test(expected = AccessControlException.class)
+ public void testInternalCreateSnapshot2() throws IOException {
+ fsView.createSnapshot(new Path("/internalDir"), "snap1");
+ }
+
+ @Test(expected = AccessControlException.class)
+ public void testInternalRenameSnapshot() throws IOException {
+ fsView.renameSnapshot(new Path("/internalDir"), "snapOldName",
+ "snapNewName");
+ }
+
+ @Test(expected = AccessControlException.class)
+ public void testInternalDeleteSnapshot() throws IOException {
+ fsView.deleteSnapshot(new Path("/internalDir"), "snap1");
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa082e1a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
index 035b280..d8ab539 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
@@ -777,4 +777,25 @@ public class ViewFsBaseTest {
public void testInternalRemoveXAttr() throws IOException {
fcView.removeXAttr(new Path("/internalDir"), "xattrName");
}
+
+ @Test(expected = AccessControlException.class)
+ public void testInternalCreateSnapshot1() throws IOException {
+ fcView.createSnapshot(new Path("/internalDir"));
+ }
+
+ @Test(expected = AccessControlException.class)
+ public void testInternalCreateSnapshot2() throws IOException {
+ fcView.createSnapshot(new Path("/internalDir"), "snap1");
+ }
+
+ @Test(expected = AccessControlException.class)
+ public void testInternalRenameSnapshot() throws IOException {
+ fcView.renameSnapshot(new Path("/internalDir"), "snapOldName",
+ "snapNewName");
+ }
+
+ @Test(expected = AccessControlException.class)
+ public void testInternalDeleteSnapshot() throws IOException {
+ fcView.deleteSnapshot(new Path("/internalDir"), "snap1");
+ }
}