You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2021/11/17 06:50:30 UTC
[hadoop] branch branch-2.10 updated: HADOOP-17999. No-op implementation of setWriteChecksum and setVerifyChecksum in ViewFileSystem. Contributed by Abhishek Das. (#3639)
This is an automated email from the ASF dual-hosted git repository.
shv pushed a commit to branch branch-2.10
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2.10 by this push:
new f9463e5 HADOOP-17999. No-op implementation of setWriteChecksum and setVerifyChecksum in ViewFileSystem. Contributed by Abhishek Das. (#3639)
f9463e5 is described below
commit f9463e511191cd100e9ed3f3b10bbad1d3989816
Author: Abhishek Das <ab...@gmail.com>
AuthorDate: Tue Nov 16 17:56:30 2021 -0800
HADOOP-17999. No-op implementation of setWriteChecksum and setVerifyChecksum in ViewFileSystem. Contributed by Abhishek Das. (#3639)
(cherry picked from commit 54a1d78e16533e286455de62a545ee75cbc1eff5)
---
.../apache/hadoop/fs/viewfs/ViewFileSystem.java | 20 +++-------
.../fs/viewfs/TestViewFileSystemDelegation.java | 12 ------
.../hadoop/fs/viewfs/ViewFileSystemBaseTest.java | 43 ++++++++++++++++++++++
3 files changed, 49 insertions(+), 26 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
index 5ee706d..c832cae 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
@@ -776,13 +776,9 @@ public class ViewFileSystem extends FileSystem {
}
@Override
- public void setVerifyChecksum(final boolean verifyChecksum) {
- List<InodeTree.MountPoint<FileSystem>> mountPoints =
- fsState.getMountPoints();
- Map<String, FileSystem> fsMap = initializeMountedFileSystems(mountPoints);
- for (InodeTree.MountPoint<FileSystem> mount : mountPoints) {
- fsMap.get(mount.src).setVerifyChecksum(verifyChecksum);
- }
+ public void setVerifyChecksum(final boolean verifyChecksum) {
+ // This is a file system level operations, however ViewFileSystem
+ // points to many file systems. Noop for ViewFileSystem.
}
/**
@@ -874,13 +870,9 @@ public class ViewFileSystem extends FileSystem {
}
@Override
- public void setWriteChecksum(final boolean writeChecksum) {
- List<InodeTree.MountPoint<FileSystem>> mountPoints =
- fsState.getMountPoints();
- Map<String, FileSystem> fsMap = initializeMountedFileSystems(mountPoints);
- for (InodeTree.MountPoint<FileSystem> mount : mountPoints) {
- fsMap.get(mount.src).setWriteChecksum(writeChecksum);
- }
+ public void setWriteChecksum(final boolean writeChecksum) {
+ // This is a file system level operations, however ViewFileSystem
+ // points to many file systems. Noop for ViewFileSystem.
}
@Override
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegation.java
index d8c39f7..3a60d6e 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegation.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegation.java
@@ -83,12 +83,6 @@ public class TestViewFileSystemDelegation { //extends ViewFileSystemTestSetup {
assertEquals(new URI("fs2:/").getAuthority(), fs2.getUri().getAuthority());
}
- @Test
- public void testVerifyChecksum() throws Exception {
- checkVerifyChecksum(false);
- checkVerifyChecksum(true);
- }
-
/**
* Tests that ViewFileSystem dispatches calls for every ACL method through the
* mount table to the correct underlying FileSystem with all Path arguments
@@ -144,12 +138,6 @@ public class TestViewFileSystemDelegation { //extends ViewFileSystemTestSetup {
verify(mockFs2).getAclStatus(mockFsPath2);
}
- void checkVerifyChecksum(boolean flag) {
- viewFs.setVerifyChecksum(flag);
- assertEquals(flag, fs1.getVerifyChecksum());
- assertEquals(flag, fs2.getVerifyChecksum());
- }
-
static class FakeFileSystem extends LocalFileSystem {
boolean verifyChecksum = true;
URI uri;
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
index d24d92a..1f01ea3 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
@@ -1129,4 +1129,47 @@ abstract public class ViewFileSystemBaseTest {
// viewfs inner cache is disabled
assertEquals(cacheSize + 2, TestFileUtil.getCacheSize());
}
+
+ @Test
+ public void testTargetFileSystemLazyInitializationForChecksumMethods()
+ throws Exception {
+ final String clusterName = "cluster" + new Random().nextInt();
+ Configuration config = new Configuration(conf);
+ config.setBoolean(CONFIG_VIEWFS_ENABLE_INNER_CACHE, false);
+ config.setClass("fs.othermockfs.impl",
+ TestChRootedFileSystem.MockFileSystem.class, FileSystem.class);
+ ConfigUtil.addLink(config, clusterName, "/user",
+ URI.create("othermockfs://mockauth1/mockpath"));
+ ConfigUtil.addLink(config, clusterName,
+ "/mock", URI.create("othermockfs://mockauth/mockpath"));
+
+ final int cacheSize = TestFileUtil.getCacheSize();
+ ViewFileSystem viewFs = (ViewFileSystem) FileSystem.get(
+ new URI("viewfs://" + clusterName + "/"), config);
+
+ // As no inner file system instance has been initialized,
+ // cache size will remain the same
+ // cache is disabled for viewfs scheme, so the viewfs:// instance won't
+ // go in the cache even after the initialization
+ assertEquals(cacheSize, TestFileUtil.getCacheSize());
+
+ // This is not going to initialize any filesystem instance
+ viewFs.setVerifyChecksum(true);
+
+ // Cache size will remain the same
+ assertEquals(cacheSize, TestFileUtil.getCacheSize());
+
+ // This resolve path will initialize the file system corresponding
+ // to the mount table entry of the path "/user"
+ viewFs.getFileChecksum(
+ new Path(String.format("viewfs://%s/%s", clusterName, "/user")));
+
+ // Cache size will increase by 1.
+ assertEquals(cacheSize + 1, TestFileUtil.getCacheSize());
+
+ viewFs.close();
+ // Initialized FileSystem instances will not be removed from cache as
+ // viewfs inner cache is disabled
+ assertEquals(cacheSize + 1, TestFileUtil.getCacheSize());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org